def run(self, builder, retry): """Run task on specified `builder`. Args: builder (obj): Builder object with the `func` to run. retry (bool): number of retry. Raises: TaskDependenciesError: If the task has dependencies. """ time_start = time.time() self.date_start = utils.strdate() if self.require: raise TaskDependenciesError("Task {} can't run. Requires {}"\ .format(self.tid, ', '.join(self.require))) try: self.results = getattr(builder, self.func)(retry=retry, **self.inputs) self.state = self.STATE_SUCCESS except Exception as err: # pylint: disable=broad-except trb = traceback.format_exc() LOGGER.error(trb) LOGGER.error(err) self.results = str(err) self.state = self.STATE_ERROR duration = time.time() - time_start self.time_duration = duration self.time_duration_str = utils.strgmtime(time.gmtime(duration)) self.date_end = utils.strdate()
def run(self): """Run infinite while receive a marker var or exec something""" while self.is_alive(): task, = self.queue.get() if task is None: LOGGER.debug("%s> getting end-of-queue markers", self.name) # Indicate that a formerly enqueued task is complete self.queue.task_done() # reached end of queue break LOGGER.debug("%s> begin(%s) task.tid=%s", self.name,\ utils.strdate(), task) # run the task with self.lock: current_task = self.tasks[task] current_task.state = Task.STATE_RUNNING self.tasks[task] = current_task try: for retry in range_type(1, self.max_retry + 1): LOGGER.debug("running retry=%d task state %d", \ retry, self.tasks[task].state) result = current_task.run(self.builder, retry=retry) if current_task.state == Task.STATE_SUCCESS: LOGGER.debug("%s> end(%s) task.tid=%s results=%s",\ self.name, utils.strdate(), task, result) break except TaskDependenciesError as err: current_task.state = Task.STATE_WRONG LOGGER.error("%s> %s", self.name, err) except Exception as err: # pylint: disable=broad-except current_task.state = Task.STATE_ERROR trb = traceback.format_exc() LOGGER.error("%s> You will see this error in prod: %s",\ self.name, err) LOGGER.error(trb) finally: # write proxydict content with self.lock: self.tasks[task] = current_task LOGGER.debug("update childs of %s", task) # Update tasks depends on this task for updated in current_task.update_childs(self.tasks): for tasku_id, tasku in iteritems(updated): # LOGGER.debug("update task of %s", tasku) self.tasks[tasku_id] = tasku self.queue.task_done() else: self.queue.task_done()
def __init__(self, tid, inputs, func, require=None): self.tid = tid self.inputs = inputs self.func = func self.require = require if not self.require: self.require = [] self.state = 0 self.results = None self.date_created = utils.strdate() self.date_start = None self.date_end = None self.time_duration_str = '00:00:00' self.time_duration = 0.0
def start(self): """Start manager Returns: dict: run result in form. >>> { 'date_end': '2018-08-29T20:02:54.640Z', 'date_start': '2018-08-29T20:02:39.606Z', 'elapsed': '00:00:15', 'exit_code': 1, 'results': { 'aborted': 0, 'deps': 2, 'failures': 1, 'nrun': 0, 'ready': 0, 'success': 3 }, 'tasks': [ { 'date_created': '2018-08-29T20:02:39.605Z', 'date_end': None, 'date_start': None, 'func': 'builder_func_1', 'inputs': {'msg': 'task-1-msg'}, 'require': ['task-id-2', 'task-id-4'], 'results': None, 'state': -2, 'tid': 'task-id-1', 'time_duration': 0.0, 'time_duration_str': '00:00:00' },{ 'date_created': '2018-08-29T20:02:39.605Z', 'date_end': '2018-08-29T20:02:41.617Z', 'date_start': '2018-08-29T20:02:40.615Z', 'func': 'builder_func_3', 'inputs': {'msg': 'task-3-msg'}, 'require': [], 'results': 'ERROR builder_func_3', 'state': -1, 'tid': 'task-id-3', 'time_duration': 1.0019969940185547, 'time_duration_str': '00:00:01' } ] } """ time_start = time.time() out = { 'elapsed': 0.0, 'date_end': None, 'date_start': utils.strdate(), 'results': { 'success': 0, 'failures': 0, 'deps': 0, 'nrun': 0, 'aborted': 0, 'ready': 0 }, 'exit_code': 1, 'tasks': [] } try: # start all workers LOGGER.debug("init %d workers", self.nb_workers) for worker in self.workers: worker.start() LOGGER.debug("send resources to queues") # generate exec graph graph = Graph(self.tasks) edges = list(graph.edges()) LOGGER.debug("Edges %s", edges) # while we have non empty graph and don't reach timeout while edges and time.time() < self.timeout: # isolated vertice is task without deps for task_id in graph.isolated_vertices(): LOGGER.debug("isolated vertex task(%s) with state(%d)", \ task_id, self.tasks[task_id].state) if self.tasks[task_id].is_runnable(): LOGGER.debug("send task(%s)", task_id) # update task status because put in queue != is running # so to avoid multiple queue send, mark it as running with self.lock: task_new = self.tasks[task_id] task_new.state = Task.STATE_READY self.tasks[task_id] = task_new self.queue.put((task_id,)) graph = Graph(self.tasks) edges = list(graph.edges()) # LOGGER.debug("Task %s", self.tasks) LOGGER.debug("Edges %s", edges) if self.progress: count = max(0, len([1 for task in self.tasks.values() \ if task.is_finished()]) - self.progress.n) if count > 0: self.progress.update(count) time.sleep(self.sleep) if self.progress: count = max(0, len([1 for task in self.tasks.values() \ if task.is_finished()]) - self.progress.n) if count > 0: self.progress.update(count) if time.time() > self.timeout: raise TimeoutError('timeout error') LOGGER.debug("add end-of-queue markers") for _ in self.workers: # True add the end to mark the end of queue self.queue.put((None,)) LOGGER.debug("blocks until all items in the queue have been "\ "gotten and processed.") self.queue.join() if self.progress: self.progress.close() # catch all errors # pylint: disable=broad-except except Exception as err: LOGGER.error("%s. Exiting...", err) # stop workers finally: LOGGER.debug("stop all workers") for worker in self.workers: # send sigterm worker.stop() # wait end worker.join() LOGGER.debug("stop workers %s", worker.name) out['date_end'] = utils.strdate() # final message for task in self.tasks.values(): out['tasks'].append(task.__dict__) if task.state == Task.STATE_SUCCESS: out['results']['success'] += 1 elif task.state == Task.STATE_ERROR: out['results']['failures'] += 1 elif task.state == Task.STATE_DEPENDENCY: out['results']['deps'] += 1 elif task.state == Task.STATE_INIT: out['results']['nrun'] += 1 elif task.state == Task.STATE_RUNNING: out['results']['aborted'] += 1 elif task.state == Task.STATE_READY: out['results']['ready'] += 1 if len(out['tasks']) == out['results']['success']: out['exit_code'] = 0 out['elapsed'] = utils.strgmtime(time.gmtime(time.time() - time_start)) return out
def test_strdate(): assert isinstance(utils.strdate(datetime.utcnow()), _py6.string_types) assert isinstance(utils.strdate(), _py6.string_types) with pytest.raises(ValueError): utils.strdate("raise")