def _refresh_task_state(task_ex_id): with db_api.transaction(): task_ex = db_api.load_task_execution(task_ex_id) if not task_ex: return if (states.is_completed(task_ex.state) or task_ex.state == states.RUNNING): return wf_ex = task_ex.workflow_execution if states.is_completed(wf_ex.state): return wf_spec = spec_parser.get_workflow_spec_by_execution_id( task_ex.workflow_execution_id ) wf_ctrl = wf_base.get_controller(wf_ex, wf_spec) with db_api.named_lock(task_ex.id): # NOTE: we have to use this lock to prevent two (or more) such # methods from changing task state and starting its action or # workflow. Checking task state outside of this section is a # performance optimization because locking is pretty expensive. db_api.refresh(task_ex) if (states.is_completed(task_ex.state) or task_ex.state == states.RUNNING): return log_state = wf_ctrl.get_logical_task_state(task_ex) state = log_state.state state_info = log_state.state_info # Update 'triggered_by' because it could have changed. task_ex.runtime_context['triggered_by'] = log_state.triggered_by if state == states.RUNNING: continue_task(task_ex) elif state == states.ERROR: complete_task(task_ex, state, state_info) elif state == states.WAITING: LOG.info( "Task execution is still in WAITING state" " [task_ex_id=%s, task_name=%s]", task_ex_id, task_ex.name ) else: # Must never get here. raise RuntimeError( 'Unexpected logical task state [task_ex_id=%s, ' 'task_name=%s, state=%s]' % (task_ex_id, task_ex.name, state) )
def _refresh_task_state(task_ex_id): with db_api.transaction(): task_ex = db_api.load_task_execution(task_ex_id) if not task_ex: return wf_ex = task_ex.workflow_execution if states.is_completed(wf_ex.state): return wf_spec = spec_parser.get_workflow_spec_by_execution_id( task_ex.workflow_execution_id) wf_ctrl = wf_base.get_controller(wf_ex, wf_spec) with db_api.named_lock(task_ex.id): db_api.refresh(task_ex) if (states.is_completed(task_ex.state) or task_ex.state == states.RUNNING): return log_state = wf_ctrl.get_logical_task_state(task_ex) state = log_state.state state_info = log_state.state_info # Update 'triggered_by' because it could have changed. task_ex.runtime_context['triggered_by'] = log_state.triggered_by if state == states.RUNNING: continue_task(task_ex) elif state == states.ERROR: complete_task(task_ex, state, state_info) elif state == states.WAITING: LOG.info( "Task execution is still in WAITING state" " [task_ex_id=%s, task_name=%s]", task_ex_id, task_ex.name) else: # Must never get here. raise RuntimeError( 'Unexpected logical task state [task_ex_id=%s, ' 'task_name=%s, state=%s]' % (task_ex_id, task_ex.name, state))
def defer(self): """Defers task. This method puts task to a waiting state. """ with db_api.named_lock(self.unique_key): if not self.task_ex: t_execs = db_api.get_task_executions( workflow_execution_id=self.wf_ex.id, unique_key=self.unique_key) self.task_ex = t_execs[0] if t_execs else None msg = 'Task is waiting.' if not self.task_ex: self._create_task_execution(state=states.WAITING, state_info=msg) elif self.task_ex.state != states.WAITING: self.set_state(states.WAITING, msg)
def on_action_complete(self, action_ex): assert self.task_ex if (not self._get_concurrency() and not self.task_spec.get_policies().get_retry()): self._on_action_complete(action_ex) else: # If we need to control 'concurrency' we need to do atomic # reads/writes to task runtime context. Locking prevents us # from modifying runtime context simultaneously by multiple # transactions. with db_api.named_lock('with-items-%s' % self.task_ex.id): # NOTE: We need to refresh task execution object right # after the lock is acquired to make sure that we're # working with a fresh state of its runtime context. # Otherwise, SQLAlchemy session can contain a stale # cached version of it so that we don't modify actual # values (i.e. capacity). db_api.refresh(self.task_ex) self._on_action_complete(action_ex)
def defer(self): """Defers task. This method puts task to a waiting state. """ # NOTE(rakhmerov): using named locks may cause problems under load # with MySQL that raises a lot of deadlocks in case of high # parallelism so it makes sense to do a fast check if the object # already exists in DB outside of the lock. if not self.task_ex: t_execs = db_api.get_task_executions( workflow_execution_id=self.wf_ex.id, unique_key=self.unique_key, state=states.WAITING ) self.task_ex = t_execs[0] if t_execs else None if self.task_ex: return with db_api.named_lock(self.unique_key): if not self.task_ex: t_execs = db_api.get_task_executions( workflow_execution_id=self.wf_ex.id, unique_key=self.unique_key ) self.task_ex = t_execs[0] if t_execs else None msg = 'Task is waiting.' if not self.task_ex: self._create_task_execution( state=states.WAITING, state_info=msg ) elif self.task_ex.state != states.WAITING: self.set_state(states.WAITING, msg)
def on_action_complete(self, action_ex): assert self.task_ex with db_api.named_lock('with-items-%s' % self.task_ex.id): # NOTE: We need to refresh task execution object right # after the lock is acquired to make sure that we're # working with a fresh state of its runtime context. # Otherwise, SQLAlchemy session can contain a stale # cached version of it so that we don't modify actual # values (i.e. capacity). db_api.refresh(self.task_ex) if self.is_completed(): return self._increase_capacity() if self.is_with_items_completed(): state = self._get_final_state() # TODO(rakhmerov): Here we can define more informative messages # in cases when action is successful and when it's not. # For example, in state_info we can specify the cause action. # The use of action_ex.output.get('result') for state_info is # not accurate because there could be action executions that # had failed or was cancelled prior to this action execution. state_info = { states.SUCCESS: None, states.ERROR: 'One or more actions had failed.', states.CANCELLED: 'One or more actions was cancelled.' } self.complete(state, state_info[state]) return if self._has_more_iterations() and self._get_concurrency(): self._schedule_actions()
def defer(self): """Defers task. This method puts task to a waiting state. """ with db_api.named_lock(self.unique_key): if not self.task_ex: t_execs = db_api.get_task_executions( workflow_execution_id=self.wf_ex.id, unique_key=self.unique_key ) self.task_ex = t_execs[0] if t_execs else None msg = 'Task is waiting.' if not self.task_ex: self._create_task_execution( state=states.WAITING, state_info=msg ) elif self.task_ex.state != states.WAITING: self.set_state(states.WAITING, msg)