def dor_initialize():
"""
Initialize DOR handling
"""
global _minimum_hosts, _dor_process_uptime, _dor_stabilize_uptime
global _dor_complete_uptime, _dor_complete_percentage
global _system_state_get_inprogress, _system_state_gathered
if config.section_exists('dor-configuration'):
section = config.CONF['dor-configuration']
_minimum_hosts = int(section.get('minimum_hosts', 4))
_dor_process_uptime = int(section.get('dor_process_uptime', 60))
_dor_stabilize_uptime = int(section.get('dor_stabilize_uptime', 240))
_dor_complete_uptime = int(section.get('dor_complete_uptime', 1200))
_dor_complete_percentage = int(section.get('dor_complete_uptime', 50))
else:
_minimum_hosts = 4
_dor_process_uptime = 60
_dor_stabilize_uptime = 240
_dor_complete_uptime = 1200
_dor_complete_percentage = 50
_system_state_get_inprogress = False
_system_state_gathered = False
timers.timers_create_timer('dor', 1, 20, _dor_timer)
def nr_initialize():
"""
Initialize Network Rebalance handling
"""
global _L3Rebalance
_L3Rebalance.set_state(L3_REBALANCE_STATE.DONE)
if config.section_exists('l3agent-rebalance'):
section = config.CONF['l3agent-rebalance']
_nr_timer_interval = int(section.get('timer_interval', 10))
_L3Rebalance.router_diff_threshold = int(section.get('router_diff_threshold', 3))
_L3Rebalance.hold_off = int(section.get('hold_off', 3))
if _L3Rebalance.router_diff_threshold < 1:
DLOG.warn("Invalid setting for router_diff_threshold: %s, forcing to 1" %
_L3Rebalance.router_diff_threshold)
_L3Rebalance.router_diff_threshold = 1
if _nr_timer_interval < 1:
DLOG.warn("Invalid setting for timer_interval: %s, forcing to 1" %
_nr_timer_interval)
_nr_timer_interval = 1
else:
_nr_timer_interval = 10
_L3Rebalance.router_diff_threshold = 3
_L3Rebalance.hold_off = 3
timers.timers_create_timer('nr', 1, _nr_timer_interval, _nr_timer)
def refresh_timeouts(self):
"""
Phase Refresh Timeouts
"""
if not self._inprogress:
# No need to refresh phase timer, phase not started
return
if self._timer_id is not None:
timers.timers_delete_timer(self._timer_id)
self._timer_id = None
# Calculate overall phase timeout
timeout_in_secs = 0
for idx in range(self._current_stage, self._stop_at_stage, 1):
stage = self._stages[idx]
timeout_in_secs += stage.timeout_in_secs
if 0 == timeout_in_secs:
# No need to refresh phase timer, phase not inprogress
return
self._timeout_in_secs = timeout_in_secs + 1
# Re-start phase timer
self._timer_id = timers.timers_create_timer(self._name,
self._timeout_in_secs,
self._timeout_in_secs,
self._timeout)
DLOG.verbose(
"Started overall strategy phase timer, timeout_in_sec=%s" %
self._timeout_in_secs)
if self._stage_timer_id is not None:
timers.timers_delete_timer(self._stage_timer_id)
self._stage_timer_id = None
if len(self._stages) <= self._current_stage:
# No need to refresh strategy stage timer, no current stage being
# applied
return
# Re-start stage timer
stage = self._stages[self._current_stage]
if 0 < stage.timeout_in_secs:
self._stage_timer_id = timers.timers_create_timer(
stage.name, stage.timeout_in_secs, stage.timeout_in_secs,
self._stage_timeout)
DLOG.verbose("Started strategy stage timer, timeout_in_sec=%s" %
stage.timeout_in_secs)
stage.refresh_timeouts()
def __init__(self,
sw_update_type,
sw_update_uuid=None,
strategy_data=None):
from nfv_vim import strategy
super(SwUpdate, self).__init__('1.0.0')
self._sw_update_type = sw_update_type
if sw_update_uuid is None:
self._uuid = str(uuid.uuid4())
else:
self._uuid = sw_update_uuid
if strategy_data is None:
self._strategy = None
else:
self._strategy = strategy.strategy_rebuild_from_dict(strategy_data)
if self._strategy is not None:
self._strategy.sw_update_obj = self
self._strategy.refresh_timeouts()
self._alarms = list()
self._nfvi_alarms = list()
self._nfvi_timer_name = sw_update_type + ' nfvi audit'
self._nfvi_timer_id = \
timers.timers_create_timer(self._nfvi_timer_name, 30, 30,
self.nfvi_audit)
self._nfvi_audit_inprogress = False
def schedule_task_work(self, task_work=None):
"""
Schedule task work to one of the task workers if available
"""
if task_work is not None:
self._wait_queue.appendleft(task_work)
if 0 == len(self._wait_queue):
return False
worker = self._task_worker_pool.claim_worker()
if worker is not None:
task_work = self._wait_queue.pop()
DLOG.verbose("Pool %s: Task worker available to run TaskWork, "
"name=%s." %
(self._task_worker_pool.name, task_work.name))
selobj.selobj_add_read_obj(worker.selobj, self.task_work_complete)
self._workers_selobj[worker.selobj] = worker
worker.submit_task_work(task_work)
if task_work.timeout_in_secs is not None:
timer_id = timers.timers_create_timer(
task_work.name, task_work.timeout_in_secs,
task_work.timeout_in_secs, self.task_work_timeout)
self._task_work_timers[timer_id] = task_work
self._workers_timer[timer_id] = worker
return True
else:
DLOG.verbose("Pool %s: No task worker available to run TaskWork." %
self._task_worker_pool.name)
return False
def refresh_timeouts(self):
"""
Stage Refresh Timeouts
"""
if not self.is_inprogress():
# No need to refresh stage timer, stage not started
return
if self._timer_id is not None:
timers.timers_delete_timer(self._timer_id)
self._timer_id = None
# Calculate overall stage timeout
self._timeout_in_secs = 0
for step in self._steps:
self._timeout_in_secs += step.timeout_in_secs
if 0 < self._timeout_in_secs:
self._timeout_in_secs += 1
# Re-start stage timer
self._timer_id = timers.timers_create_timer(self._name,
self._timeout_in_secs,
self._timeout_in_secs,
self._timeout)
DLOG.verbose(
"Started overall strategy stage timer, timeout_in_sec=%s" %
self._timeout_in_secs)
if self._step_timer_id is not None:
timers.timers_delete_timer(self._step_timer_id)
self._step_timer_id = None
if len(self._steps) <= self._current_step:
# No need to refresh step timer, no current step being applied
return
# Re-start step timer
step = self._steps[self._current_step]
if 0 < step.timeout_in_secs:
self._step_timer_id = timers.timers_create_timer(
step.name, step.timeout_in_secs, step.timeout_in_secs,
self._step_timeout)
DLOG.verbose("Started strategy step timer, timeout_in_sec=%s" %
step.timeout_in_secs)
def start(self):
"""
Start the Thread
"""
self._process.start()
if self._check_timer_id is None:
self._check_timer_id = timers.timers_create_timer(
self._name, self._check_interval_in_secs,
self._check_interval_in_secs, self.do_check)
def add_task_timer(self, name, interval_secs, task):
"""
Add timer for a task
"""
timer_id = timers.timers_create_timer(name, interval_secs,
interval_secs,
self.task_timer_timeout)
self._task_timers[timer_id] = task
return timer_id
def initialize(self):
"""
Initialize the Alarm Worker
"""
self._handlers = AlarmHandlers(self._config['namespace'],
self._config['handlers'])
self._handlers.initialize(self._config['config_file'])
self._alarm_audit_timer_id = timers.timers_create_timer(
'alarm_audit', int(self._config['audit_interval']),
int(self._config['audit_interval']), self._alarm_audit)
def refresh_timeouts(self):
"""
State Task Refresh Timeouts
"""
if self._timer_id is None:
# No need to refresh task timer, task not started
return
timers.timers_delete_timer(self._timer_id)
self._timer_id = None
# Calculate overall task timeout
self._timeout_in_secs = 0
for task_work in self._task_work_list:
self._timeout_in_secs += task_work.timeout_in_secs
if 0 < self._timeout_in_secs:
self._timeout_in_secs += 1
# Re-start task timer
self._timer_id = timers.timers_create_timer(self._name,
self._timeout_in_secs,
self._timeout_in_secs,
self._timeout)
if self._task_work_timer_id is None:
# No need to refresh task work timer, no task work running
return
timers.timers_delete_timer(self._task_work_timer_id)
self._task_work_timer_id = None
if len(self._task_work_list) <= self._current_task_work:
# No need to refresh task work timer, no current task work running
return
# Re-start task work timer
task_work = self._task_work_list[self._current_task_work]
if 0 < task_work.timeout_in_secs:
self._task_work_timer_id = timers.timers_create_timer(
task_work.name, task_work.timeout_in_secs,
task_work.timeout_in_secs, self._task_work_timeout)
def commit(self):
if self._commit_inline:
start_ms = timers.get_monotonic_timestamp_in_ms()
self._session.commit()
elapsed_ms = timers.get_monotonic_timestamp_in_ms() - start_ms
histogram.add_histogram_data("database-commits (inline)",
elapsed_ms / 100, "decisecond")
else:
if self._commit_timer_id is None:
self._commit_timer_id \
= timers.timers_create_timer('db-auto-commit', 1, 1,
self.auto_commit)
def start(self):
"""
State Task Start
"""
self._cleanup()
self._current_task_work = 0
self._task_inprogress = True
self._task_result = STATE_TASK_RESULT.SUCCESS
self._task_result_reason = 'success'
self._timer_id = timers.timers_create_timer(self._name,
self._timeout_in_secs,
self._timeout_in_secs,
self._timeout)
self._run()
def _run(self):
"""
State Task Run
"""
if not self._task_inprogress:
DLOG.debug("Task (%s) not inprogress." % self._name)
return
for idx in range(self._current_task_work, len(self._task_work_list), 1):
task_work = self._task_work_list[idx]
if self._task_work_timer_id is not None:
timers.timers_delete_timer(self._task_work_timer_id)
self._task_work_timer_id = None
DLOG.debug("Task %s running %s work." % (self._name,
task_work.name))
task_work_result, task_work_result_reason = task_work.run()
self._current_task_work = idx
if STATE_TASK_WORK_RESULT.WAIT == task_work_result:
if 0 < task_work.timeout_in_secs:
self._task_work_timer_id = timers.timers_create_timer(
task_work.name, task_work.timeout_in_secs,
task_work.timeout_in_secs, self._task_work_timeout)
DLOG.debug("Task (%s) is waiting for work (%s) to complete, "
"timeout_in_secs=%s." % (self._name, task_work.name,
task_work.timeout_in_secs))
break
else:
self._task_result, self._task_result_reason = \
state_task_result_update(
self._task_result, self._task_result_reason,
task_work_result, task_work_result_reason)
if STATE_TASK_RESULT.FAILED == self._task_result \
or STATE_TASK_RESULT.ABORTED == self._task_result \
or STATE_TASK_RESULT.TIMED_OUT == self._task_result:
self._abort()
self._complete(self._task_result, self._task_result_reason)
break
else:
DLOG.debug("Task (%s) done running." % self._name)
self._task_inprogress = False
self._cleanup()
self._complete(self._task_result, self._task_result_reason)
def strategy_apply(self, strategy_uuid, stage_id):
"""
Apply a software update strategy
"""
success = False
if self.strategy is None:
reason = "strategy not created"
elif strategy_uuid != self.strategy.uuid:
reason = "strategy does not exist"
else:
event_log.sw_update_issue_log(
self.event_id(SW_UPDATE_EVENT_IDS.APPLY_START))
success, reason = self.strategy.apply(stage_id)
if success:
if self._alarms:
alarm.clear_sw_update_alarm(self._alarms)
self._alarms = \
alarm.raise_sw_update_alarm(
self.alarm_type(SW_UPDATE_ALARM_TYPES.APPLY_INPROGRESS))
event_log.sw_update_issue_log(
self.event_id(SW_UPDATE_EVENT_IDS.APPLY_INPROGRESS))
if self._nfvi_timer_id is not None:
timers.timers_delete_timer(self._nfvi_timer_id)
self._nfvi_timer_id = None
self._nfvi_timer_id = \
timers.timers_create_timer(self._nfvi_timer_name, 30, 30,
self.nfvi_audit)
else:
event_log.sw_update_issue_log(self.event_id(
SW_UPDATE_EVENT_IDS.APPLY_REJECTED),
reason=reason)
return success, reason
def _apply(self):
"""
Stage Apply
"""
if not self._inprogress:
if 0 == self._current_step:
self._cleanup()
self._inprogress = True
self._result = STRATEGY_STAGE_RESULT.INPROGRESS
self._result_reason = ''
self._start_date_time = datetime.now().strftime(
"%Y-%m-%d %H:%M:%S")
if 0 < self.timeout_in_secs:
self._timer_id = timers.timers_create_timer(
self._name, self._timeout_in_secs,
self._timeout_in_secs, self._timeout)
else:
DLOG.debug("Stage (%s) not inprogress." % self._name)
return self._result, self._result_reason
for idx in range(self._current_step, len(self._steps), 1):
step = self._steps[idx]
if self._step_timer_id is not None:
timers.timers_delete_timer(self._step_timer_id)
self._step_timer_id = None
step.start_date_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
step.result, step.result_reason = step.apply()
self._current_step = idx
if STRATEGY_STEP_RESULT.WAIT == step.result:
if 0 < step.timeout_in_secs:
self._step_timer_id = timers.timers_create_timer(
step.name, step.timeout_in_secs, step.timeout_in_secs,
self._step_timeout)
DLOG.debug("Stage (%s) is waiting for step (%s) to complete, "
"timeout_in_secs=%s." %
(self.name, step.name, step.timeout_in_secs))
self._save()
return STRATEGY_STAGE_RESULT.WAIT, ''
else:
DLOG.debug(
"Stage (%s) step (%s) complete, result=%s, reason=%s." %
(self._name, step.name, step.result, step.result_reason))
step.end_date_time = datetime.now().strftime(
"%Y-%m-%d %H:%M:%S")
self._result, self._result_reason = \
strategy_stage_result_update(self._result,
self._result_reason,
step.result, step.result_reason)
if STRATEGY_STAGE_RESULT.FAILED == self._result or \
STRATEGY_STAGE_RESULT.ABORTED == self._result or \
STRATEGY_STAGE_RESULT.TIMED_OUT == self._result:
return self._complete(self._result, self._result_reason)
else:
self._save()
else:
# Check for a stage with no steps
if 0 == self._current_step:
self._result = STRATEGY_STAGE_RESULT.SUCCESS
self._result_reason = ''
DLOG.debug("Stage (%s) done running, result=%s, reason=%s." %
(self._name, self._result, self._result_reason))
return self._complete(self._result, self._result_reason)
def _apply(self):
"""
Phase Apply
"""
if not self._inprogress:
if 0 == self._current_stage:
self._cleanup()
self._current_stage = 0
self._inprogress = True
self._result = STRATEGY_PHASE_RESULT.INPROGRESS
self._result_reason = ''
self._start_date_time = datetime.now().strftime(
"%Y-%m-%d %H:%M:%S")
else:
DLOG.debug("Phase (%s) not inprogress." % self._name)
return self._result, self._result_reason
if self._timer_id is None:
timeout_in_secs = 0
for idx in range(self._current_stage, self._stop_at_stage, 1):
stage = self._stages[idx]
timeout_in_secs += stage.timeout_in_secs
if 0 < timeout_in_secs:
self._timeout_in_secs = timeout_in_secs + 1
self._timer_id = timers.timers_create_timer(
self._name, self._timeout_in_secs, self._timeout_in_secs,
self._timeout)
for idx in range(self._current_stage, self._stop_at_stage, 1):
stage = self._stages[idx]
if self._stage_timer_id is not None:
timers.timers_delete_timer(self._stage_timer_id)
self._stage_timer_id = None
DLOG.info("Phase %s running %s stage." % (self._name, stage.name))
stage_result, stage_result_reason = stage.apply()
self._current_stage = idx
if STRATEGY_STAGE_RESULT.WAIT == stage_result:
if 0 < stage.timeout_in_secs:
self._stage_timer_id = timers.timers_create_timer(
stage.name, stage.timeout_in_secs,
stage.timeout_in_secs, self._stage_timeout)
DLOG.debug("Phase (%s) is waiting for stage (%s) to complete, "
"timeout_in_secs=%s." %
(self._name, stage.name, stage.timeout_in_secs))
self._save()
return STRATEGY_PHASE_RESULT.WAIT, ''
else:
DLOG.debug(
"Phase (%s) stage (%s) complete, result=%s, reason=%s." %
(self._name, stage.name, stage_result,
stage_result_reason))
self._result, self._result_reason = \
strategy_phase_result_update(self._result,
self._result_reason,
stage_result, stage_result_reason)
if STRATEGY_PHASE_RESULT.FAILED == self._result \
or STRATEGY_PHASE_RESULT.ABORTED == self._result \
or STRATEGY_PHASE_RESULT.TIMED_OUT == self._result:
return self._complete(self._result, self._result_reason)
else:
self._save()
else:
# Check if this is an intermediate stop or the phase has been completed
if self._stop_at_stage == len(self._stages):
# Check for a phase with no stages
if 0 == self._current_stage:
self._result = STRATEGY_PHASE_RESULT.SUCCESS
self._result_reason = ''
DLOG.debug("Phase (%s) done running, result=%s, reason=%s." %
(self._name, self._result, self._result_reason))
return self._complete(self._result, self._result_reason)
else:
self._cleanup()
self._save()
return self._result, self._result_reason