def delete_task(self, task):
"""
Delete a task from the task scheduler
"""
DLOG.debug("Pool %s: Delete Task, name=%s." %
(self._task_worker_pool.name, task.name))
for timer_id, timer_owner in self._task_timers.items():
if timer_owner.id == task.id:
timers.timers_delete_timer(timer_id)
del self._task_timers[timer_id]
for timer_id, timer_owner in self._task_work_timers.items():
if timer_owner.task_id == task.id:
timers.timers_delete_timer(timer_id)
del self._task_work_timers[timer_id]
for select_obj, select_obj_owner in self._task_read_selobjs.items():
if select_obj_owner.id == task.id:
selobj.selobj_del_read_obj(select_obj)
del self._task_read_selobjs[select_obj]
for select_obj, select_obj_owner in self._task_write_selobjs.items():
if select_obj_owner.id == task.id:
selobj.selobj_del_write_obj(select_obj)
del self._task_write_selobjs[select_obj]
del self._tasks[task.id]
def task_work_complete(self):
"""
A task worker has completed it's assigned work
"""
while True:
select_obj = (yield)
worker = self._workers_selobj.get(select_obj, None)
if worker is not None:
self._task_worker_pool.release_worker(worker)
selobj.selobj_del_read_obj(worker.selobj)
del self._workers_selobj[worker.selobj]
task_work = worker.get_task_work_result()
if task_work is not None:
for timer_id, timer_owner in self._task_work_timers.items(
):
if timer_owner.id == task_work.id:
timers.timers_delete_timer(timer_id)
del self._task_work_timers[timer_id]
del self._workers_timer[timer_id]
task = self._tasks.get(task_work.task_id, None)
if task is not None:
self._running_task = task
try:
task.task_work_complete(task_work)
except StopIteration:
self.delete_task(task)
if self._task_worker_pool.available_workers():
self.schedule_task_work()
def finalize(self):
"""
Finalize the Alarm Worker
"""
if self._handlers is not None:
self._handlers.finalize()
if self._alarm_audit_timer_id is not None:
timers.timers_delete_timer(self._alarm_audit_timer_id)
def stop(self, max_wait_in_seconds):
"""
Stop the Thread
"""
self._work_queue.put([Thread.ACTION_STOP, None])
self._process.join(max_wait_in_seconds)
if self._process.is_alive():
self._process.terminate()
if self._check_timer_id is not None:
timers.timers_delete_timer(self._check_timer_id)
self._work_queue.close()
def refresh_timeouts(self):
"""
Phase Refresh Timeouts
"""
if not self._inprogress:
# No need to refresh phase timer, phase not started
return
if self._timer_id is not None:
timers.timers_delete_timer(self._timer_id)
self._timer_id = None
# Calculate overall phase timeout
timeout_in_secs = 0
for idx in range(self._current_stage, self._stop_at_stage, 1):
stage = self._stages[idx]
timeout_in_secs += stage.timeout_in_secs
if 0 == timeout_in_secs:
# No need to refresh phase timer, phase not inprogress
return
self._timeout_in_secs = timeout_in_secs + 1
# Re-start phase timer
self._timer_id = timers.timers_create_timer(self._name,
self._timeout_in_secs,
self._timeout_in_secs,
self._timeout)
DLOG.verbose(
"Started overall strategy phase timer, timeout_in_sec=%s" %
self._timeout_in_secs)
if self._stage_timer_id is not None:
timers.timers_delete_timer(self._stage_timer_id)
self._stage_timer_id = None
if len(self._stages) <= self._current_stage:
# No need to refresh strategy stage timer, no current stage being
# applied
return
# Re-start stage timer
stage = self._stages[self._current_stage]
if 0 < stage.timeout_in_secs:
self._stage_timer_id = timers.timers_create_timer(
stage.name, stage.timeout_in_secs, stage.timeout_in_secs,
self._stage_timeout)
DLOG.verbose("Started strategy stage timer, timeout_in_sec=%s" %
stage.timeout_in_secs)
stage.refresh_timeouts()
def _cleanup(self):
"""
State Task Cleanup
"""
if self._timer_id is not None:
timers.timers_delete_timer(self._timer_id)
self._timer_id = None
if self._task_work_timer_id is not None:
timers.timers_delete_timer(self._task_work_timer_id)
self._task_work_timer_id = None
def _cleanup(self):
"""
Stage Cleanup
"""
DLOG.info("Stage (%s) cleanup called" % self._name)
if self._timer_id is not None:
timers.timers_delete_timer(self._timer_id)
self._timer_id = None
if self._step_timer_id is not None:
timers.timers_delete_timer(self._step_timer_id)
self._step_timer_id = None
def refresh_timeouts(self):
"""
Stage Refresh Timeouts
"""
if not self.is_inprogress():
# No need to refresh stage timer, stage not started
return
if self._timer_id is not None:
timers.timers_delete_timer(self._timer_id)
self._timer_id = None
# Calculate overall stage timeout
self._timeout_in_secs = 0
for step in self._steps:
self._timeout_in_secs += step.timeout_in_secs
if 0 < self._timeout_in_secs:
self._timeout_in_secs += 1
# Re-start stage timer
self._timer_id = timers.timers_create_timer(self._name,
self._timeout_in_secs,
self._timeout_in_secs,
self._timeout)
DLOG.verbose(
"Started overall strategy stage timer, timeout_in_sec=%s" %
self._timeout_in_secs)
if self._step_timer_id is not None:
timers.timers_delete_timer(self._step_timer_id)
self._step_timer_id = None
if len(self._steps) <= self._current_step:
# No need to refresh step timer, no current step being applied
return
# Re-start step timer
step = self._steps[self._current_step]
if 0 < step.timeout_in_secs:
self._step_timer_id = timers.timers_create_timer(
step.name, step.timeout_in_secs, step.timeout_in_secs,
self._step_timeout)
DLOG.verbose("Started strategy step timer, timeout_in_sec=%s" %
step.timeout_in_secs)
def _run(self):
"""
State Task Run
"""
if not self._task_inprogress:
DLOG.debug("Task (%s) not inprogress." % self._name)
return
for idx in range(self._current_task_work, len(self._task_work_list), 1):
task_work = self._task_work_list[idx]
if self._task_work_timer_id is not None:
timers.timers_delete_timer(self._task_work_timer_id)
self._task_work_timer_id = None
DLOG.debug("Task %s running %s work." % (self._name,
task_work.name))
task_work_result, task_work_result_reason = task_work.run()
self._current_task_work = idx
if STATE_TASK_WORK_RESULT.WAIT == task_work_result:
if 0 < task_work.timeout_in_secs:
self._task_work_timer_id = timers.timers_create_timer(
task_work.name, task_work.timeout_in_secs,
task_work.timeout_in_secs, self._task_work_timeout)
DLOG.debug("Task (%s) is waiting for work (%s) to complete, "
"timeout_in_secs=%s." % (self._name, task_work.name,
task_work.timeout_in_secs))
break
else:
self._task_result, self._task_result_reason = \
state_task_result_update(
self._task_result, self._task_result_reason,
task_work_result, task_work_result_reason)
if STATE_TASK_RESULT.FAILED == self._task_result \
or STATE_TASK_RESULT.ABORTED == self._task_result \
or STATE_TASK_RESULT.TIMED_OUT == self._task_result:
self._abort()
self._complete(self._task_result, self._task_result_reason)
break
else:
DLOG.debug("Task (%s) done running." % self._name)
self._task_inprogress = False
self._cleanup()
self._complete(self._task_result, self._task_result_reason)
def strategy_apply(self, strategy_uuid, stage_id):
"""
Apply a software update strategy
"""
success = False
if self.strategy is None:
reason = "strategy not created"
elif strategy_uuid != self.strategy.uuid:
reason = "strategy does not exist"
else:
event_log.sw_update_issue_log(
self.event_id(SW_UPDATE_EVENT_IDS.APPLY_START))
success, reason = self.strategy.apply(stage_id)
if success:
if self._alarms:
alarm.clear_sw_update_alarm(self._alarms)
self._alarms = \
alarm.raise_sw_update_alarm(
self.alarm_type(SW_UPDATE_ALARM_TYPES.APPLY_INPROGRESS))
event_log.sw_update_issue_log(
self.event_id(SW_UPDATE_EVENT_IDS.APPLY_INPROGRESS))
if self._nfvi_timer_id is not None:
timers.timers_delete_timer(self._nfvi_timer_id)
self._nfvi_timer_id = None
self._nfvi_timer_id = \
timers.timers_create_timer(self._nfvi_timer_name, 30, 30,
self.nfvi_audit)
else:
event_log.sw_update_issue_log(self.event_id(
SW_UPDATE_EVENT_IDS.APPLY_REJECTED),
reason=reason)
return success, reason
def refresh_timeouts(self):
"""
State Task Refresh Timeouts
"""
if self._timer_id is None:
# No need to refresh task timer, task not started
return
timers.timers_delete_timer(self._timer_id)
self._timer_id = None
# Calculate overall task timeout
self._timeout_in_secs = 0
for task_work in self._task_work_list:
self._timeout_in_secs += task_work.timeout_in_secs
if 0 < self._timeout_in_secs:
self._timeout_in_secs += 1
# Re-start task timer
self._timer_id = timers.timers_create_timer(self._name,
self._timeout_in_secs,
self._timeout_in_secs,
self._timeout)
if self._task_work_timer_id is None:
# No need to refresh task work timer, no task work running
return
timers.timers_delete_timer(self._task_work_timer_id)
self._task_work_timer_id = None
if len(self._task_work_list) <= self._current_task_work:
# No need to refresh task work timer, no current task work running
return
# Re-start task work timer
task_work = self._task_work_list[self._current_task_work]
if 0 < task_work.timeout_in_secs:
self._task_work_timer_id = timers.timers_create_timer(
task_work.name, task_work.timeout_in_secs,
task_work.timeout_in_secs, self._task_work_timeout)
def strategy_delete(self, strategy_uuid, force):
"""
Delete a software update strategy
"""
if self.strategy is None:
reason = "strategy not created"
return False, reason
if strategy_uuid != self.strategy.uuid:
reason = "strategy does not exist"
return False, reason
if not force:
if self.strategy.is_building():
reason = "strategy is being built, can't delete"
return False, reason
if self.strategy.is_applying():
reason = "strategy is being applied, can't delete"
return False, reason
if self.strategy.is_aborting():
reason = "strategy is being aborted, can't delete"
return False, reason
if self._alarms:
alarm.clear_sw_update_alarm(self._alarms)
if self._nfvi_timer_id is not None:
timers.timers_delete_timer(self._nfvi_timer_id)
self._nfvi_timer_id = None
del self._strategy
self._strategy = None
self._persist()
return True, ''
def _apply(self):
"""
Stage Apply
"""
if not self._inprogress:
if 0 == self._current_step:
self._cleanup()
self._inprogress = True
self._result = STRATEGY_STAGE_RESULT.INPROGRESS
self._result_reason = ''
self._start_date_time = datetime.now().strftime(
"%Y-%m-%d %H:%M:%S")
if 0 < self.timeout_in_secs:
self._timer_id = timers.timers_create_timer(
self._name, self._timeout_in_secs,
self._timeout_in_secs, self._timeout)
else:
DLOG.debug("Stage (%s) not inprogress." % self._name)
return self._result, self._result_reason
for idx in range(self._current_step, len(self._steps), 1):
step = self._steps[idx]
if self._step_timer_id is not None:
timers.timers_delete_timer(self._step_timer_id)
self._step_timer_id = None
step.start_date_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
step.result, step.result_reason = step.apply()
self._current_step = idx
if STRATEGY_STEP_RESULT.WAIT == step.result:
if 0 < step.timeout_in_secs:
self._step_timer_id = timers.timers_create_timer(
step.name, step.timeout_in_secs, step.timeout_in_secs,
self._step_timeout)
DLOG.debug("Stage (%s) is waiting for step (%s) to complete, "
"timeout_in_secs=%s." %
(self.name, step.name, step.timeout_in_secs))
self._save()
return STRATEGY_STAGE_RESULT.WAIT, ''
else:
DLOG.debug(
"Stage (%s) step (%s) complete, result=%s, reason=%s." %
(self._name, step.name, step.result, step.result_reason))
step.end_date_time = datetime.now().strftime(
"%Y-%m-%d %H:%M:%S")
self._result, self._result_reason = \
strategy_stage_result_update(self._result,
self._result_reason,
step.result, step.result_reason)
if STRATEGY_STAGE_RESULT.FAILED == self._result or \
STRATEGY_STAGE_RESULT.ABORTED == self._result or \
STRATEGY_STAGE_RESULT.TIMED_OUT == self._result:
return self._complete(self._result, self._result_reason)
else:
self._save()
else:
# Check for a stage with no steps
if 0 == self._current_step:
self._result = STRATEGY_STAGE_RESULT.SUCCESS
self._result_reason = ''
DLOG.debug("Stage (%s) done running, result=%s, reason=%s." %
(self._name, self._result, self._result_reason))
return self._complete(self._result, self._result_reason)
def _apply(self):
"""
Phase Apply
"""
if not self._inprogress:
if 0 == self._current_stage:
self._cleanup()
self._current_stage = 0
self._inprogress = True
self._result = STRATEGY_PHASE_RESULT.INPROGRESS
self._result_reason = ''
self._start_date_time = datetime.now().strftime(
"%Y-%m-%d %H:%M:%S")
else:
DLOG.debug("Phase (%s) not inprogress." % self._name)
return self._result, self._result_reason
if self._timer_id is None:
timeout_in_secs = 0
for idx in range(self._current_stage, self._stop_at_stage, 1):
stage = self._stages[idx]
timeout_in_secs += stage.timeout_in_secs
if 0 < timeout_in_secs:
self._timeout_in_secs = timeout_in_secs + 1
self._timer_id = timers.timers_create_timer(
self._name, self._timeout_in_secs, self._timeout_in_secs,
self._timeout)
for idx in range(self._current_stage, self._stop_at_stage, 1):
stage = self._stages[idx]
if self._stage_timer_id is not None:
timers.timers_delete_timer(self._stage_timer_id)
self._stage_timer_id = None
DLOG.info("Phase %s running %s stage." % (self._name, stage.name))
stage_result, stage_result_reason = stage.apply()
self._current_stage = idx
if STRATEGY_STAGE_RESULT.WAIT == stage_result:
if 0 < stage.timeout_in_secs:
self._stage_timer_id = timers.timers_create_timer(
stage.name, stage.timeout_in_secs,
stage.timeout_in_secs, self._stage_timeout)
DLOG.debug("Phase (%s) is waiting for stage (%s) to complete, "
"timeout_in_secs=%s." %
(self._name, stage.name, stage.timeout_in_secs))
self._save()
return STRATEGY_PHASE_RESULT.WAIT, ''
else:
DLOG.debug(
"Phase (%s) stage (%s) complete, result=%s, reason=%s." %
(self._name, stage.name, stage_result,
stage_result_reason))
self._result, self._result_reason = \
strategy_phase_result_update(self._result,
self._result_reason,
stage_result, stage_result_reason)
if STRATEGY_PHASE_RESULT.FAILED == self._result \
or STRATEGY_PHASE_RESULT.ABORTED == self._result \
or STRATEGY_PHASE_RESULT.TIMED_OUT == self._result:
return self._complete(self._result, self._result_reason)
else:
self._save()
else:
# Check if this is an intermediate stop or the phase has been completed
if self._stop_at_stage == len(self._stages):
# Check for a phase with no stages
if 0 == self._current_stage:
self._result = STRATEGY_PHASE_RESULT.SUCCESS
self._result_reason = ''
DLOG.debug("Phase (%s) done running, result=%s, reason=%s." %
(self._name, self._result, self._result_reason))
return self._complete(self._result, self._result_reason)
else:
self._cleanup()
self._save()
return self._result, self._result_reason
def cancel_task_timer(self, timer_id, task):
timer_owner = self._task_timers.get(timer_id, None)
if timer_owner is not None:
if timer_owner.id == task.id:
timers.timers_delete_timer(timer_id)
del self._task_timers[timer_id]
