def send_user_action(job_id, job_state, user_action):
"""
request a user action
"""
if not user_action:
raise Exception("User action is missing")
job = Job.objects.get(pk=job_id)
#check whether user action is valid
current_state = JobState.get_jobstate(job.state)
if not current_state.outcome_cls.is_manual_outcome(user_action):
raise Exception(
"The action '{0}' is not a valid user action.".format(
user_action))
required_state_name = JobState.get_jobstate(
job_state).name if job_state else None
#if action is not a cancel action, job state should be equal with the requested state.
if user_action == JobStateOutcome.cancelled_by_custodian or (
job.state and job.state == required_state_name):
job.user_action = user_action
job.save(update_fields=["user_action"])
else:
raise Exception(
"Job is on the state {0} instead of required state {1}".format(
job.state, required_state_name))
def job_action(self, o):
job_state = JobState.get_jobstate(o.state)
if not job_state or job_state.is_end_state:
return ""
value = ""
if job_state.is_interactive_state:
if o.user_action and o.user_action == JobStateOutcome.approved_by_custodian:
value = "Approved"
elif not o.user_action or o.user_action != JobStateOutcome.cancelled_by_custodian:
value = "<a href='/job/" + str(
o.id) + "/approve?job_state=" + o.state + "'>Approve</a>"
if job_state.cancellable:
if o.user_action and o.user_action == JobStateOutcome.cancelled_by_custodian:
if value:
value += " | Cancelled"
else:
value = "Cancelled"
else:
if value:
value += " | <a href='/job/" + str(
o.id) + "/cancel?job_state=" + o.state + "'>Cancel</a>"
else:
value = "<a href='/job/" + str(
o.id) + "/cancel?job_state=" + o.state + "'>Cancel</a>"
return value
def run_all_jobs(first_run=True):
"""
run all jobs sequentially
"""
succeed_jobs = 0
failed_jobs = 0
ignored_jobs = 0
error_jobs = 0
for j in Job.objects.exclude(
state__in=[Failed.instance().name,
Completed.instance().name]).order_by('id'):
try:
JobStatemachine.run(j, first_run)
if j.state == "Completed":
if j.launched is None:
ignored_jobs += 1
else:
succeed_jobs += 1
elif j.state == "Failed":
failed_jobs += 1
else:
error_jobs += 1
except:
logger.error(
"job(id={0},name={1}) runs into a exception{2}".format(
j.id, j.publish.name,
JobState.get_exception_message()))
error_jobs += 1
return (succeed_jobs, failed_jobs, ignored_jobs, error_jobs)
def send_user_action(job_id,job_state,user_action):
"""
request a user action
"""
if not user_action:
raise Exception("User action is missing")
job = Job.objects.get(pk = job_id)
#check whether user action is valid
current_state = JobState.get_jobstate(job.state)
if not current_state.outcome_cls.is_manual_outcome(user_action):
raise Exception("The action '{0}' is not a valid user action.".format(user_action))
required_state_name = JobState.get_jobstate(job_state).name if job_state else None
#if action is not a cancel action, job state should be equal with the requested state.
if user_action == JobStateOutcome.cancelled_by_custodian or (job.state and job.state == required_state_name):
job.user_action = user_action
job.save(update_fields=["user_action"])
else:
raise Exception("Job is on the state {0} instead of required state {1}".format(job.state, required_state_name))
def run_all_jobs(first_run=True):
"""
run all jobs sequentially
"""
for j in Job.objects.exclude(
state__in=[Failed.instance().name,
Completed.instance().name]).order_by('id'):
try:
JobStatemachine.run(j, first_run)
except:
logger.error(
"job(id={0},name={1}) runs into a exception{2}".format(
j.id, j.publish.name,
JobState.get_exception_message()))
def _job_status(self,o):
if o.job_id:
try:
j = Job.objects.get(pk=o.job_id)
state = JobState.get_jobstate(j.state)
if state.is_end_state:
return state.name
elif state.is_error_state:
return "Waiting approve" if state.is_interactive_state else "Error"
else:
return "running"
except:
return ""
else:
return ""
def _job_status(self,o):
if o.job_id:
try:
j = Job.objects.get(pk=o.job_id)
state = JobState.get_jobstate(j.state)
if state.is_end_state:
return state.name
elif state.is_error_state:
return "Waiting approve" if state.is_interactive_state else "Error"
else:
return "running"
except:
return ""
else:
return ""
def _pre_delete(sender, instance, **args):
if instance.state and not JobState.get_jobstate(instance.state).is_end_state:
raise Exception("Unfinished job can not be deleted.")
#remove the dump file if exist
dump_dir = instance.dump_dir
if dump_dir and os.path.exists(dump_dir):
existed_files = 0
for f in os.listdir(dump_dir):
if f.startswith(instance.publish.table_name + "."):
#the file belongs to the job,remove it
os.remove(os.path.join(dump_dir,f))
else:
existed_files += 1
if not existed_files:
os.rmdir(dump_dir)
def _pre_delete(sender, instance, **args):
if instance.state and not JobState.get_jobstate(instance.state).is_end_state:
raise Exception("Unfinished job can not be deleted.")
#remove the dump file if exist
if instance.pgdump_file:
dump_path = instance.pgdump_file.path
if os.path.isfile(dump_path):
os.remove(dump_path)
#remove the style file if exist
if instance.style_file:
style_path = instance.style_file.path
if os.path.isfile(style_path):
os.remove(style_path)
#remove the folder, if empty
folder = os.path.dirname(dump_path)
if os.path.exists(folder) and len(os.listdir(folder)) == 0:
os.rmdir(folder)
def _pre_delete(sender, instance, **args):
if instance.state and not JobState.get_jobstate(
instance.state).is_end_state:
raise Exception("Unfinished job can not be deleted.")
#remove the dump file if exist
dump_dir = instance.dump_dir
if dump_dir and os.path.exists(dump_dir):
existed_files = 0
for f in os.listdir(dump_dir):
if f.startswith(instance.publish.table_name + "."):
#the file belongs to the job,remove it
os.remove(os.path.join(dump_dir, f))
else:
existed_files += 1
if not existed_files:
os.rmdir(dump_dir)
def _pre_delete(sender, instance, **args):
if instance.state and not JobState.get_jobstate(
instance.state).is_end_state:
raise Exception("Unfinished job can not be deleted.")
#remove the dump file if exist
if instance.pgdump_file:
dump_path = instance.pgdump_file.path
if os.path.isfile(dump_path):
os.remove(dump_path)
#remove the style file if exist
if instance.style_file:
style_path = instance.style_file.path
if os.path.isfile(style_path):
os.remove(style_path)
#remove the folder, if empty
folder = os.path.dirname(dump_path)
if os.path.exists(folder) and len(os.listdir(folder)) == 0:
os.rmdir(folder)
def job_action(self,o):
job_state = JobState.get_jobstate(o.state);
if not job_state or job_state.is_end_state:
return ""
value = "";
if job_state.is_interactive_state :
if o.user_action and o.user_action == JobStateOutcome.approved_by_custodian:
value = "Approved"
elif not o.user_action or o.user_action != JobStateOutcome.cancelled_by_custodian:
value = "<a href='/job/" + str(o.id) +"/approve?job_state=" + o.state + "'>Approve</a>"
if job_state.cancellable:
if o.user_action and o.user_action == JobStateOutcome.cancelled_by_custodian:
if value:
value += " | Cancelled"
else:
value = "Cancelled"
else:
if value:
value += " | <a href='/job/" + str(o.id) +"/cancel?job_state=" + o.state + "'>Cancel</a>"
else:
value = "<a href='/job/" + str(o.id) +"/cancel?job_state=" + o.state + "'>Cancel</a>"
return value
def run(job, first_run=True, step=False):
current_state = JobState.get_jobstate(job.state)
previous_state = None
if job.previous_state:
previous_state = JobState.get_jobstate(job.previous_state)
log = None
while True:
start_time = timezone.now()
logger.debug(
"job(id={0},name={1}) begins to execute state ({2})".format(
job.id, job.publish.name, job.state))
if current_state.is_end_state:
#current job is already finished.
logger.debug(
"job(id={0},name={1},state={2}) is finished".format(
job.id, job.publish.name, job.state))
return
elif job.user_action and (
(job.user_action == JobStateOutcome.cancelled_by_custodian
and current_state.cancellable) or
(job.user_action != JobStateOutcome.cancelled_by_custodian)):
#have a pending user action.
if not current_state.outcome_cls.is_manual_outcome(
job.user_action):
#a invalid user action
next_state = current_state
state_result = (
JobStateOutcome.internal_error,
"The action '{0}' is not a valid user action.".format(
job.user_action))
else:
try:
next_state = current_state.next_state(job.user_action)
state_result = (job.user_action, job.user_action)
#user action is processed, clear it
job.user_action = None
except:
#can not apply the user action on the current state, stay in the same state
next_state = current_state
state_result = (
JobStateOutcome.internal_error,
"The action '{0}' can not apply on the current state({1})."
.format(job.user_action, current_state.name))
elif current_state.is_interactive_state:
#job is at interactive state, but no user action is requested, return and wait user action.
return
elif current_state.is_error_state:
#wait the configured interval before continue
try:
if not first_run and job.last_execution_end_time and timezone.now(
) < job.last_execution_end_time + timedelta(
seconds=BorgConfiguration.RETRY_INTERVAL):
#early than the next execution time. can not run this time
return
else:
state_result = current_state.execute(
job, previous_state)
except:
#can not find the last execution time. run it
state_result = current_state.execute(job, previous_state)
next_state = current_state.next_state(state_result[0])
else:
try:
#normal state without pending user action
state_result = current_state.execute(job, previous_state)
if not state_result:
raise Exception(
"The outcome of state '{0}' is null.".format(
current_state.name))
except:
#run into a unexpected exception
state_result = (JobStateOutcome.internal_error,
JobState.get_exception_message())
try:
next_state = current_state.next_state(state_result[0])
except:
#can not find a transition from the current state with the state_result, stay at the current state.
next_state = current_state
if state_result[1]:
state_result = (
state_result[0],
"{0}\n=======================\n{1}".format(
JobState.get_exception_message(),
state_result[1]))
else:
state_result = (state_result[0],
JobState.get_exception_message())
end_time = timezone.now()
logger.debug(
"job(id={0},name={1}) ends to execute state ({2}) with result ({3});"
.format(job.id, job.publish.name, job.state, state_result))
#set the retry times
if current_state == next_state:
#stay at the same state, runs into some exception
job.retry_times += 1
elif current_state.is_interactive_state:
#current state is interactive state.
job.retry_times = 0
elif current_state.is_error_state:
#current state is a error state
if next_state.is_error_state:
#stay at the same state, run into a exception
job.retry_times += 1
elif isinstance(next_state, current_state._normal_state):
#a normal transition from current state to its associated normal state
pass
else:
#a transition from current state to other normal state, reset retry_times.
job.retry_times = 0
elif next_state.is_end_state:
#reach the end state or a volative state
job.retry_times = 0
else:
#current state is a normal state
if not next_state.is_error_state:
#a successful transition from a normal state to another normal state.
job.retry_times = 0
else:
#a failed transition from a normal state to a failed state
job.retry_times += 1
log = JobLog(job_id=job.id,
state=job.state,
outcome=state_result[0],
message=state_result[1],
next_state=next_state,
start_time=start_time,
end_time=end_time)
if current_state.is_error_state and not current_state.is_interactive_state and not job.user_action and isinstance(
next_state, current_state._normal_state):
#normal transition from error state to normal state, ignore the log
log = None
elif next_state.is_error_state or current_state == next_state:
#failed,
last_log = JobLog.objects.filter(
job_id=job.id).order_by("-pk").first()
if last_log and last_log.state == job.state and last_log.outcome == state_result[
0] and last_log.message == state_result[
1] and last_log.next_state == next_state.name:
#same execption occur
last_log.start_time = start_time
last_log.end_time = end_time
log = last_log
if current_state != next_state:
#job move to a new state, change the job state
job.previous_state = job.state
job.state = next_state.name
else:
#some bad thing happens,job stays at the same state, leave the job's state untouched
pass
job.last_execution_end_time = timezone.now()
job.message = state_result[1]
job.save(update_fields=[
'previous_state', 'state', 'message', 'retry_times',
'last_execution_end_time', 'user_action'
])
if log:
log.save()
#import ipdb; ipdb.set_trace()
if current_state == next_state:
#stay in the same state, stop execution
logger.debug(
"job(id={0},name={1},state={2}) stay in the same state.".
format(job.id, job.publish.name, job.state))
return
elif next_state.is_error_state:
return
elif step:
return
#set previous_state to the current state, current_state to the next_state
previous_state = current_state
current_state = next_state
def run_all_jobs(first_run=True):
"""
run all jobs sequentially
"""
succeed_jobs = 0
failed_jobs = 0
ignored_jobs = 0
error_jobs = 0
for j in Job.objects.exclude(state__in = [Failed.instance().name,Completed.instance().name]).order_by('id'):
try:
JobStatemachine.run(j,first_run)
if j.state == "Completed":
if j.launched is None:
ignored_jobs += 1
else:
succeed_jobs += 1
elif j.state == "Failed":
failed_jobs += 1
else:
error_jobs += 1
except:
logger.error("job(id={0},name={1}) runs into a exception{2}".format(j.id,j.publish.name,JobState.get_exception_message()))
error_jobs += 1
return (succeed_jobs,failed_jobs,ignored_jobs,error_jobs)
def jobstate(self):
if self.state:
return JobState.get_jobstate(self.state)
return None
def run_all_jobs(first_run=True):
"""
run all jobs sequentially
"""
for j in Job.objects.exclude(state__in = [Failed.instance().name,Completed.instance().name]).order_by('id'):
try:
JobStatemachine.run(j,first_run)
except:
logger.error("job(id={0},name={1}) runs into a exception{2}".format(j.id,j.publish.name,JobState.get_exception_message()))
def jobstate(self):
if self.state:
return JobState.get_jobstate(self.state)
return None
def run(job,first_run=True,step=False):
current_state = JobState.get_jobstate(job.state)
previous_state = None
if job.previous_state:
previous_state = JobState.get_jobstate(job.previous_state)
log = None
while True:
start_time = timezone.now()
logger.debug("job(id={0},name={1}) begins to execute state ({2})".format(job.id,job.publish.name,job.state))
if current_state.is_end_state:
#current job is already finished.
logger.debug("job(id={0},name={1},state={2}) is finished".format(job.id,job.publish.name,job.state))
return
elif job.user_action and ((job.user_action == JobStateOutcome.cancelled_by_custodian and current_state.cancellable) or (job.user_action != JobStateOutcome.cancelled_by_custodian)):
#have a pending user action.
if not current_state.outcome_cls.is_manual_outcome(job.user_action):
#a invalid user action
next_state = current_state
state_result = (JobStateOutcome.internal_error, "The action '{0}' is not a valid user action.".format(job.user_action))
else:
try:
next_state = current_state.next_state(job.user_action)
state_result = (job.user_action,job.user_action)
#user action is processed, clear it
job.user_action = None
except:
#can not apply the user action on the current state, stay in the same state
next_state = current_state
state_result = (JobStateOutcome.internal_error, "The action '{0}' can not apply on the current state({1}).".format(job.user_action,current_state.name))
elif current_state.is_interactive_state:
#job is at interactive state, but no user action is requested, return and wait user action.
return
elif current_state.is_error_state:
#wait the configured interval before continue
try:
if not first_run and job.last_execution_end_time and timezone.now() < job.last_execution_end_time + timedelta(seconds=BorgConfiguration.RETRY_INTERVAL):
#early than the next execution time. can not run this time
return
else:
state_result = current_state.execute(job,previous_state)
except:
#can not find the last execution time. run it
state_result = current_state.execute(job,previous_state)
next_state = current_state.next_state(state_result[0])
else:
try:
#normal state without pending user action
state_result = current_state.execute(job,previous_state)
if not state_result:
raise Exception("The outcome of state '{0}' is null.".format(current_state.name))
except:
#run into a unexpected exception
state_result = (JobStateOutcome.internal_error, JobState.get_exception_message())
try:
next_state = current_state.next_state(state_result[0])
except:
#can not find a transition from the current state with the state_result, stay at the current state.
next_state = current_state
if state_result[1]:
state_result = (state_result[0],"{0}\n=======================\n{1}".format(JobState.get_exception_message(),state_result[1]))
else:
state_result = (state_result[0],JobState.get_exception_message())
end_time = timezone.now()
logger.debug("job(id={0},name={1}) ends to execute state ({2}) with result ({3});".format(job.id,job.publish.name,job.state,state_result))
#set the retry times
if current_state == next_state:
#stay at the same state, runs into some exception
job.retry_times += 1
elif current_state.is_interactive_state:
#current state is interactive state.
job.retry_times = 0
elif current_state.is_error_state:
#current state is a error state
if next_state.is_error_state:
#stay at the same state, run into a exception
job.retry_times += 1
elif isinstance(next_state,current_state._normal_state):
#a normal transition from current state to its associated normal state
pass
else:
#a transition from current state to other normal state, reset retry_times.
job.retry_times = 0
elif next_state.is_end_state :
#reach the end state or a volative state
job.retry_times = 0
else:
#current state is a normal state
if not next_state.is_error_state:
#a successful transition from a normal state to another normal state.
job.retry_times = 0
else:
#a failed transition from a normal state to a failed state
job.retry_times += 1
log = JobLog(
job_id = job.id,
state = job.state,
outcome = state_result[0],
message = state_result[1],
next_state = next_state,
start_time = start_time,
end_time = end_time)
if current_state.is_error_state and not current_state.is_interactive_state and not job.user_action and isinstance(next_state,current_state._normal_state):
#normal transition from error state to normal state, ignore the log
log = None
elif next_state.is_error_state or current_state == next_state:
#failed,
last_log = JobLog.objects.filter(job_id = job.id).order_by("-pk").first()
if last_log and last_log.state == job.state and last_log.outcome == state_result[0] and last_log.message == state_result[1] and last_log.next_state == next_state.name:
#same execption occur
last_log.start_time = start_time
last_log.end_time = end_time
log = last_log
if current_state != next_state:
#job move to a new state, change the job state
job.previous_state = job.state
job.state = next_state.name
else:
#some bad thing happens,job stays at the same state, leave the job's state untouched
pass
job.last_execution_end_time = timezone.now()
job.message = state_result[1]
job.save(update_fields=['previous_state','state','message','retry_times','last_execution_end_time','user_action'])
if log:
log.save()
#import ipdb; ipdb.set_trace()
if current_state == next_state:
#stay in the same state, stop execution
logger.debug("job(id={0},name={1},state={2}) stay in the same state.".format(job.id,job.publish.name,job.state))
return
elif next_state.is_error_state:
return
elif step:
return
#set previous_state to the current state, current_state to the next_state
previous_state = current_state
current_state = next_state
