def run(self, builder, retry):
"""Run task on specified `builder`.
Args:
builder (obj): Builder object with the `func` to run.
retry (bool): number of retry.
Raises:
TaskDependenciesError: If the task has dependencies.
"""
time_start = time.time()
self.date_start = utils.strdate()
if self.require:
raise TaskDependenciesError("Task {} can't run. Requires {}"\
.format(self.tid, ', '.join(self.require)))
try:
self.results = getattr(builder, self.func)(retry=retry,
**self.inputs)
self.state = self.STATE_SUCCESS
except Exception as err: # pylint: disable=broad-except
trb = traceback.format_exc()
LOGGER.error(trb)
LOGGER.error(err)
self.results = str(err)
self.state = self.STATE_ERROR
duration = time.time() - time_start
self.time_duration = duration
self.time_duration_str = utils.strgmtime(time.gmtime(duration))
self.date_end = utils.strdate()
def run(self):
"""Run infinite while receive a marker var or exec something"""
while self.is_alive():
task, = self.queue.get()
if task is None:
LOGGER.debug("%s> getting end-of-queue markers", self.name)
# Indicate that a formerly enqueued task is complete
self.queue.task_done()
# reached end of queue
break
LOGGER.debug("%s> begin(%s) task.tid=%s", self.name,\
utils.strdate(), task)
# run the task
with self.lock:
current_task = self.tasks[task]
current_task.state = Task.STATE_RUNNING
self.tasks[task] = current_task
try:
for retry in range_type(1, self.max_retry + 1):
LOGGER.debug("running retry=%d task state %d", \
retry, self.tasks[task].state)
result = current_task.run(self.builder, retry=retry)
if current_task.state == Task.STATE_SUCCESS:
LOGGER.debug("%s> end(%s) task.tid=%s results=%s",\
self.name, utils.strdate(), task, result)
break
except TaskDependenciesError as err:
current_task.state = Task.STATE_WRONG
LOGGER.error("%s> %s", self.name, err)
except Exception as err: # pylint: disable=broad-except
current_task.state = Task.STATE_ERROR
trb = traceback.format_exc()
LOGGER.error("%s> You will see this error in prod: %s",\
self.name, err)
LOGGER.error(trb)
finally:
# write proxydict content
with self.lock:
self.tasks[task] = current_task
LOGGER.debug("update childs of %s", task)
# Update tasks depends on this task
for updated in current_task.update_childs(self.tasks):
for tasku_id, tasku in iteritems(updated):
# LOGGER.debug("update task of %s", tasku)
self.tasks[tasku_id] = tasku
self.queue.task_done()
else:
self.queue.task_done()
def __init__(self, tid, inputs, func, require=None):
self.tid = tid
self.inputs = inputs
self.func = func
self.require = require
if not self.require:
self.require = []
self.state = 0
self.results = None
self.date_created = utils.strdate()
self.date_start = None
self.date_end = None
self.time_duration_str = '00:00:00'
self.time_duration = 0.0
def start(self):
"""Start manager
Returns:
dict: run result in form.
>>> {
'date_end': '2018-08-29T20:02:54.640Z',
'date_start': '2018-08-29T20:02:39.606Z',
'elapsed': '00:00:15',
'exit_code': 1,
'results': {
'aborted': 0,
'deps': 2,
'failures': 1,
'nrun': 0,
'ready': 0,
'success': 3
},
'tasks': [
{
'date_created': '2018-08-29T20:02:39.605Z',
'date_end': None,
'date_start': None,
'func': 'builder_func_1',
'inputs': {'msg': 'task-1-msg'},
'require': ['task-id-2', 'task-id-4'],
'results': None,
'state': -2,
'tid': 'task-id-1',
'time_duration': 0.0,
'time_duration_str': '00:00:00'
},{
'date_created': '2018-08-29T20:02:39.605Z',
'date_end': '2018-08-29T20:02:41.617Z',
'date_start': '2018-08-29T20:02:40.615Z',
'func': 'builder_func_3',
'inputs': {'msg': 'task-3-msg'},
'require': [],
'results': 'ERROR builder_func_3',
'state': -1,
'tid': 'task-id-3',
'time_duration': 1.0019969940185547,
'time_duration_str': '00:00:01'
}
]
}
"""
time_start = time.time()
out = {
'elapsed': 0.0,
'date_end': None,
'date_start': utils.strdate(),
'results': {
'success': 0,
'failures': 0,
'deps': 0,
'nrun': 0,
'aborted': 0,
'ready': 0
},
'exit_code': 1,
'tasks': []
}
try:
# start all workers
LOGGER.debug("init %d workers", self.nb_workers)
for worker in self.workers:
worker.start()
LOGGER.debug("send resources to queues")
# generate exec graph
graph = Graph(self.tasks)
edges = list(graph.edges())
LOGGER.debug("Edges %s", edges)
# while we have non empty graph and don't reach timeout
while edges and time.time() < self.timeout:
# isolated vertice is task without deps
for task_id in graph.isolated_vertices():
LOGGER.debug("isolated vertex task(%s) with state(%d)", \
task_id, self.tasks[task_id].state)
if self.tasks[task_id].is_runnable():
LOGGER.debug("send task(%s)", task_id)
# update task status because put in queue != is running
# so to avoid multiple queue send, mark it as running
with self.lock:
task_new = self.tasks[task_id]
task_new.state = Task.STATE_READY
self.tasks[task_id] = task_new
self.queue.put((task_id,))
graph = Graph(self.tasks)
edges = list(graph.edges())
# LOGGER.debug("Task %s", self.tasks)
LOGGER.debug("Edges %s", edges)
if self.progress:
count = max(0, len([1 for task in self.tasks.values() \
if task.is_finished()]) - self.progress.n)
if count > 0:
self.progress.update(count)
time.sleep(self.sleep)
if self.progress:
count = max(0, len([1 for task in self.tasks.values() \
if task.is_finished()]) - self.progress.n)
if count > 0:
self.progress.update(count)
if time.time() > self.timeout:
raise TimeoutError('timeout error')
LOGGER.debug("add end-of-queue markers")
for _ in self.workers:
# True add the end to mark the end of queue
self.queue.put((None,))
LOGGER.debug("blocks until all items in the queue have been "\
"gotten and processed.")
self.queue.join()
if self.progress:
self.progress.close()
# catch all errors
# pylint: disable=broad-except
except Exception as err:
LOGGER.error("%s. Exiting...", err)
# stop workers
finally:
LOGGER.debug("stop all workers")
for worker in self.workers:
# send sigterm
worker.stop()
# wait end
worker.join()
LOGGER.debug("stop workers %s", worker.name)
out['date_end'] = utils.strdate()
# final message
for task in self.tasks.values():
out['tasks'].append(task.__dict__)
if task.state == Task.STATE_SUCCESS:
out['results']['success'] += 1
elif task.state == Task.STATE_ERROR:
out['results']['failures'] += 1
elif task.state == Task.STATE_DEPENDENCY:
out['results']['deps'] += 1
elif task.state == Task.STATE_INIT:
out['results']['nrun'] += 1
elif task.state == Task.STATE_RUNNING:
out['results']['aborted'] += 1
elif task.state == Task.STATE_READY:
out['results']['ready'] += 1
if len(out['tasks']) == out['results']['success']:
out['exit_code'] = 0
out['elapsed'] = utils.strgmtime(time.gmtime(time.time() - time_start))
return out
def test_strdate():
assert isinstance(utils.strdate(datetime.utcnow()), _py6.string_types)
assert isinstance(utils.strdate(), _py6.string_types)
with pytest.raises(ValueError):
utils.strdate("raise")