def send_user_action(job_id, job_state, user_action): """ request a user action """ if not user_action: raise Exception("User action is missing") job = Job.objects.get(pk=job_id) #check whether user action is valid current_state = JobState.get_jobstate(job.state) if not current_state.outcome_cls.is_manual_outcome(user_action): raise Exception( "The action '{0}' is not a valid user action.".format( user_action)) required_state_name = JobState.get_jobstate( job_state).name if job_state else None #if action is not a cancel action, job state should be equal with the requested state. if user_action == JobStateOutcome.cancelled_by_custodian or ( job.state and job.state == required_state_name): job.user_action = user_action job.save(update_fields=["user_action"]) else: raise Exception( "Job is on the state {0} instead of required state {1}".format( job.state, required_state_name))
def job_action(self, o): job_state = JobState.get_jobstate(o.state) if not job_state or job_state.is_end_state: return "" value = "" if job_state.is_interactive_state: if o.user_action and o.user_action == JobStateOutcome.approved_by_custodian: value = "Approved" elif not o.user_action or o.user_action != JobStateOutcome.cancelled_by_custodian: value = "<a href='/job/" + str( o.id) + "/approve?job_state=" + o.state + "'>Approve</a>" if job_state.cancellable: if o.user_action and o.user_action == JobStateOutcome.cancelled_by_custodian: if value: value += " | Cancelled" else: value = "Cancelled" else: if value: value += " | <a href='/job/" + str( o.id) + "/cancel?job_state=" + o.state + "'>Cancel</a>" else: value = "<a href='/job/" + str( o.id) + "/cancel?job_state=" + o.state + "'>Cancel</a>" return value
def send_user_action(job_id,job_state,user_action): """ request a user action """ if not user_action: raise Exception("User action is missing") job = Job.objects.get(pk = job_id) #check whether user action is valid current_state = JobState.get_jobstate(job.state) if not current_state.outcome_cls.is_manual_outcome(user_action): raise Exception("The action '{0}' is not a valid user action.".format(user_action)) required_state_name = JobState.get_jobstate(job_state).name if job_state else None #if action is not a cancel action, job state should be equal with the requested state. if user_action == JobStateOutcome.cancelled_by_custodian or (job.state and job.state == required_state_name): job.user_action = user_action job.save(update_fields=["user_action"]) else: raise Exception("Job is on the state {0} instead of required state {1}".format(job.state, required_state_name))
def _job_status(self,o): if o.job_id: try: j = Job.objects.get(pk=o.job_id) state = JobState.get_jobstate(j.state) if state.is_end_state: return state.name elif state.is_error_state: return "Waiting approve" if state.is_interactive_state else "Error" else: return "running" except: return "" else: return ""
def _pre_delete(sender, instance, **args): if instance.state and not JobState.get_jobstate(instance.state).is_end_state: raise Exception("Unfinished job can not be deleted.") #remove the dump file if exist dump_dir = instance.dump_dir if dump_dir and os.path.exists(dump_dir): existed_files = 0 for f in os.listdir(dump_dir): if f.startswith(instance.publish.table_name + "."): #the file belongs to the job,remove it os.remove(os.path.join(dump_dir,f)) else: existed_files += 1 if not existed_files: os.rmdir(dump_dir)
def _pre_delete(sender, instance, **args): if instance.state and not JobState.get_jobstate( instance.state).is_end_state: raise Exception("Unfinished job can not be deleted.") #remove the dump file if exist dump_dir = instance.dump_dir if dump_dir and os.path.exists(dump_dir): existed_files = 0 for f in os.listdir(dump_dir): if f.startswith(instance.publish.table_name + "."): #the file belongs to the job,remove it os.remove(os.path.join(dump_dir, f)) else: existed_files += 1 if not existed_files: os.rmdir(dump_dir)
def _pre_delete(sender, instance, **args): if instance.state and not JobState.get_jobstate(instance.state).is_end_state: raise Exception("Unfinished job can not be deleted.") #remove the dump file if exist if instance.pgdump_file: dump_path = instance.pgdump_file.path if os.path.isfile(dump_path): os.remove(dump_path) #remove the style file if exist if instance.style_file: style_path = instance.style_file.path if os.path.isfile(style_path): os.remove(style_path) #remove the folder, if empty folder = os.path.dirname(dump_path) if os.path.exists(folder) and len(os.listdir(folder)) == 0: os.rmdir(folder)
def _pre_delete(sender, instance, **args): if instance.state and not JobState.get_jobstate( instance.state).is_end_state: raise Exception("Unfinished job can not be deleted.") #remove the dump file if exist if instance.pgdump_file: dump_path = instance.pgdump_file.path if os.path.isfile(dump_path): os.remove(dump_path) #remove the style file if exist if instance.style_file: style_path = instance.style_file.path if os.path.isfile(style_path): os.remove(style_path) #remove the folder, if empty folder = os.path.dirname(dump_path) if os.path.exists(folder) and len(os.listdir(folder)) == 0: os.rmdir(folder)
def job_action(self,o): job_state = JobState.get_jobstate(o.state); if not job_state or job_state.is_end_state: return "" value = ""; if job_state.is_interactive_state : if o.user_action and o.user_action == JobStateOutcome.approved_by_custodian: value = "Approved" elif not o.user_action or o.user_action != JobStateOutcome.cancelled_by_custodian: value = "<a href='/job/" + str(o.id) +"/approve?job_state=" + o.state + "'>Approve</a>" if job_state.cancellable: if o.user_action and o.user_action == JobStateOutcome.cancelled_by_custodian: if value: value += " | Cancelled" else: value = "Cancelled" else: if value: value += " | <a href='/job/" + str(o.id) +"/cancel?job_state=" + o.state + "'>Cancel</a>" else: value = "<a href='/job/" + str(o.id) +"/cancel?job_state=" + o.state + "'>Cancel</a>" return value
def run(job, first_run=True, step=False): current_state = JobState.get_jobstate(job.state) previous_state = None if job.previous_state: previous_state = JobState.get_jobstate(job.previous_state) log = None while True: start_time = timezone.now() logger.debug( "job(id={0},name={1}) begins to execute state ({2})".format( job.id, job.publish.name, job.state)) if current_state.is_end_state: #current job is already finished. logger.debug( "job(id={0},name={1},state={2}) is finished".format( job.id, job.publish.name, job.state)) return elif job.user_action and ( (job.user_action == JobStateOutcome.cancelled_by_custodian and current_state.cancellable) or (job.user_action != JobStateOutcome.cancelled_by_custodian)): #have a pending user action. if not current_state.outcome_cls.is_manual_outcome( job.user_action): #a invalid user action next_state = current_state state_result = ( JobStateOutcome.internal_error, "The action '{0}' is not a valid user action.".format( job.user_action)) else: try: next_state = current_state.next_state(job.user_action) state_result = (job.user_action, job.user_action) #user action is processed, clear it job.user_action = None except: #can not apply the user action on the current state, stay in the same state next_state = current_state state_result = ( JobStateOutcome.internal_error, "The action '{0}' can not apply on the current state({1})." .format(job.user_action, current_state.name)) elif current_state.is_interactive_state: #job is at interactive state, but no user action is requested, return and wait user action. return elif current_state.is_error_state: #wait the configured interval before continue try: if not first_run and job.last_execution_end_time and timezone.now( ) < job.last_execution_end_time + timedelta( seconds=BorgConfiguration.RETRY_INTERVAL): #early than the next execution time. can not run this time return else: state_result = current_state.execute( job, previous_state) except: #can not find the last execution time. run it state_result = current_state.execute(job, previous_state) next_state = current_state.next_state(state_result[0]) else: try: #normal state without pending user action state_result = current_state.execute(job, previous_state) if not state_result: raise Exception( "The outcome of state '{0}' is null.".format( current_state.name)) except: #run into a unexpected exception state_result = (JobStateOutcome.internal_error, JobState.get_exception_message()) try: next_state = current_state.next_state(state_result[0]) except: #can not find a transition from the current state with the state_result, stay at the current state. next_state = current_state if state_result[1]: state_result = ( state_result[0], "{0}\n=======================\n{1}".format( JobState.get_exception_message(), state_result[1])) else: state_result = (state_result[0], JobState.get_exception_message()) end_time = timezone.now() logger.debug( "job(id={0},name={1}) ends to execute state ({2}) with result ({3});" .format(job.id, job.publish.name, job.state, state_result)) #set the retry times if current_state == next_state: #stay at the same state, runs into some exception job.retry_times += 1 elif current_state.is_interactive_state: #current state is interactive state. job.retry_times = 0 elif current_state.is_error_state: #current state is a error state if next_state.is_error_state: #stay at the same state, run into a exception job.retry_times += 1 elif isinstance(next_state, current_state._normal_state): #a normal transition from current state to its associated normal state pass else: #a transition from current state to other normal state, reset retry_times. job.retry_times = 0 elif next_state.is_end_state: #reach the end state or a volative state job.retry_times = 0 else: #current state is a normal state if not next_state.is_error_state: #a successful transition from a normal state to another normal state. job.retry_times = 0 else: #a failed transition from a normal state to a failed state job.retry_times += 1 log = JobLog(job_id=job.id, state=job.state, outcome=state_result[0], message=state_result[1], next_state=next_state, start_time=start_time, end_time=end_time) if current_state.is_error_state and not current_state.is_interactive_state and not job.user_action and isinstance( next_state, current_state._normal_state): #normal transition from error state to normal state, ignore the log log = None elif next_state.is_error_state or current_state == next_state: #failed, last_log = JobLog.objects.filter( job_id=job.id).order_by("-pk").first() if last_log and last_log.state == job.state and last_log.outcome == state_result[ 0] and last_log.message == state_result[ 1] and last_log.next_state == next_state.name: #same execption occur last_log.start_time = start_time last_log.end_time = end_time log = last_log if current_state != next_state: #job move to a new state, change the job state job.previous_state = job.state job.state = next_state.name else: #some bad thing happens,job stays at the same state, leave the job's state untouched pass job.last_execution_end_time = timezone.now() job.message = state_result[1] job.save(update_fields=[ 'previous_state', 'state', 'message', 'retry_times', 'last_execution_end_time', 'user_action' ]) if log: log.save() #import ipdb; ipdb.set_trace() if current_state == next_state: #stay in the same state, stop execution logger.debug( "job(id={0},name={1},state={2}) stay in the same state.". format(job.id, job.publish.name, job.state)) return elif next_state.is_error_state: return elif step: return #set previous_state to the current state, current_state to the next_state previous_state = current_state current_state = next_state
def run(job,first_run=True,step=False): current_state = JobState.get_jobstate(job.state) previous_state = None if job.previous_state: previous_state = JobState.get_jobstate(job.previous_state) log = None while True: start_time = timezone.now() logger.debug("job(id={0},name={1}) begins to execute state ({2})".format(job.id,job.publish.name,job.state)) if current_state.is_end_state: #current job is already finished. logger.debug("job(id={0},name={1},state={2}) is finished".format(job.id,job.publish.name,job.state)) return elif job.user_action and ((job.user_action == JobStateOutcome.cancelled_by_custodian and current_state.cancellable) or (job.user_action != JobStateOutcome.cancelled_by_custodian)): #have a pending user action. if not current_state.outcome_cls.is_manual_outcome(job.user_action): #a invalid user action next_state = current_state state_result = (JobStateOutcome.internal_error, "The action '{0}' is not a valid user action.".format(job.user_action)) else: try: next_state = current_state.next_state(job.user_action) state_result = (job.user_action,job.user_action) #user action is processed, clear it job.user_action = None except: #can not apply the user action on the current state, stay in the same state next_state = current_state state_result = (JobStateOutcome.internal_error, "The action '{0}' can not apply on the current state({1}).".format(job.user_action,current_state.name)) elif current_state.is_interactive_state: #job is at interactive state, but no user action is requested, return and wait user action. return elif current_state.is_error_state: #wait the configured interval before continue try: if not first_run and job.last_execution_end_time and timezone.now() < job.last_execution_end_time + timedelta(seconds=BorgConfiguration.RETRY_INTERVAL): #early than the next execution time. can not run this time return else: state_result = current_state.execute(job,previous_state) except: #can not find the last execution time. run it state_result = current_state.execute(job,previous_state) next_state = current_state.next_state(state_result[0]) else: try: #normal state without pending user action state_result = current_state.execute(job,previous_state) if not state_result: raise Exception("The outcome of state '{0}' is null.".format(current_state.name)) except: #run into a unexpected exception state_result = (JobStateOutcome.internal_error, JobState.get_exception_message()) try: next_state = current_state.next_state(state_result[0]) except: #can not find a transition from the current state with the state_result, stay at the current state. next_state = current_state if state_result[1]: state_result = (state_result[0],"{0}\n=======================\n{1}".format(JobState.get_exception_message(),state_result[1])) else: state_result = (state_result[0],JobState.get_exception_message()) end_time = timezone.now() logger.debug("job(id={0},name={1}) ends to execute state ({2}) with result ({3});".format(job.id,job.publish.name,job.state,state_result)) #set the retry times if current_state == next_state: #stay at the same state, runs into some exception job.retry_times += 1 elif current_state.is_interactive_state: #current state is interactive state. job.retry_times = 0 elif current_state.is_error_state: #current state is a error state if next_state.is_error_state: #stay at the same state, run into a exception job.retry_times += 1 elif isinstance(next_state,current_state._normal_state): #a normal transition from current state to its associated normal state pass else: #a transition from current state to other normal state, reset retry_times. job.retry_times = 0 elif next_state.is_end_state : #reach the end state or a volative state job.retry_times = 0 else: #current state is a normal state if not next_state.is_error_state: #a successful transition from a normal state to another normal state. job.retry_times = 0 else: #a failed transition from a normal state to a failed state job.retry_times += 1 log = JobLog( job_id = job.id, state = job.state, outcome = state_result[0], message = state_result[1], next_state = next_state, start_time = start_time, end_time = end_time) if current_state.is_error_state and not current_state.is_interactive_state and not job.user_action and isinstance(next_state,current_state._normal_state): #normal transition from error state to normal state, ignore the log log = None elif next_state.is_error_state or current_state == next_state: #failed, last_log = JobLog.objects.filter(job_id = job.id).order_by("-pk").first() if last_log and last_log.state == job.state and last_log.outcome == state_result[0] and last_log.message == state_result[1] and last_log.next_state == next_state.name: #same execption occur last_log.start_time = start_time last_log.end_time = end_time log = last_log if current_state != next_state: #job move to a new state, change the job state job.previous_state = job.state job.state = next_state.name else: #some bad thing happens,job stays at the same state, leave the job's state untouched pass job.last_execution_end_time = timezone.now() job.message = state_result[1] job.save(update_fields=['previous_state','state','message','retry_times','last_execution_end_time','user_action']) if log: log.save() #import ipdb; ipdb.set_trace() if current_state == next_state: #stay in the same state, stop execution logger.debug("job(id={0},name={1},state={2}) stay in the same state.".format(job.id,job.publish.name,job.state)) return elif next_state.is_error_state: return elif step: return #set previous_state to the current state, current_state to the next_state previous_state = current_state current_state = next_state
def jobstate(self): if self.state: return JobState.get_jobstate(self.state) return None