def dor_initialize(): """ Initialize DOR handling """ global _minimum_hosts, _dor_process_uptime, _dor_stabilize_uptime global _dor_complete_uptime, _dor_complete_percentage global _system_state_get_inprogress, _system_state_gathered if config.section_exists('dor-configuration'): section = config.CONF['dor-configuration'] _minimum_hosts = int(section.get('minimum_hosts', 4)) _dor_process_uptime = int(section.get('dor_process_uptime', 60)) _dor_stabilize_uptime = int(section.get('dor_stabilize_uptime', 240)) _dor_complete_uptime = int(section.get('dor_complete_uptime', 1200)) _dor_complete_percentage = int(section.get('dor_complete_uptime', 50)) else: _minimum_hosts = 4 _dor_process_uptime = 60 _dor_stabilize_uptime = 240 _dor_complete_uptime = 1200 _dor_complete_percentage = 50 _system_state_get_inprogress = False _system_state_gathered = False timers.timers_create_timer('dor', 1, 20, _dor_timer)
def nr_initialize(): """ Initialize Network Rebalance handling """ global _L3Rebalance _L3Rebalance.set_state(L3_REBALANCE_STATE.DONE) if config.section_exists('l3agent-rebalance'): section = config.CONF['l3agent-rebalance'] _nr_timer_interval = int(section.get('timer_interval', 10)) _L3Rebalance.router_diff_threshold = int(section.get('router_diff_threshold', 3)) _L3Rebalance.hold_off = int(section.get('hold_off', 3)) if _L3Rebalance.router_diff_threshold < 1: DLOG.warn("Invalid setting for router_diff_threshold: %s, forcing to 1" % _L3Rebalance.router_diff_threshold) _L3Rebalance.router_diff_threshold = 1 if _nr_timer_interval < 1: DLOG.warn("Invalid setting for timer_interval: %s, forcing to 1" % _nr_timer_interval) _nr_timer_interval = 1 else: _nr_timer_interval = 10 _L3Rebalance.router_diff_threshold = 3 _L3Rebalance.hold_off = 3 timers.timers_create_timer('nr', 1, _nr_timer_interval, _nr_timer)
def refresh_timeouts(self): """ Phase Refresh Timeouts """ if not self._inprogress: # No need to refresh phase timer, phase not started return if self._timer_id is not None: timers.timers_delete_timer(self._timer_id) self._timer_id = None # Calculate overall phase timeout timeout_in_secs = 0 for idx in range(self._current_stage, self._stop_at_stage, 1): stage = self._stages[idx] timeout_in_secs += stage.timeout_in_secs if 0 == timeout_in_secs: # No need to refresh phase timer, phase not inprogress return self._timeout_in_secs = timeout_in_secs + 1 # Re-start phase timer self._timer_id = timers.timers_create_timer(self._name, self._timeout_in_secs, self._timeout_in_secs, self._timeout) DLOG.verbose( "Started overall strategy phase timer, timeout_in_sec=%s" % self._timeout_in_secs) if self._stage_timer_id is not None: timers.timers_delete_timer(self._stage_timer_id) self._stage_timer_id = None if len(self._stages) <= self._current_stage: # No need to refresh strategy stage timer, no current stage being # applied return # Re-start stage timer stage = self._stages[self._current_stage] if 0 < stage.timeout_in_secs: self._stage_timer_id = timers.timers_create_timer( stage.name, stage.timeout_in_secs, stage.timeout_in_secs, self._stage_timeout) DLOG.verbose("Started strategy stage timer, timeout_in_sec=%s" % stage.timeout_in_secs) stage.refresh_timeouts()
def __init__(self, sw_update_type, sw_update_uuid=None, strategy_data=None): from nfv_vim import strategy super(SwUpdate, self).__init__('1.0.0') self._sw_update_type = sw_update_type if sw_update_uuid is None: self._uuid = str(uuid.uuid4()) else: self._uuid = sw_update_uuid if strategy_data is None: self._strategy = None else: self._strategy = strategy.strategy_rebuild_from_dict(strategy_data) if self._strategy is not None: self._strategy.sw_update_obj = self self._strategy.refresh_timeouts() self._alarms = list() self._nfvi_alarms = list() self._nfvi_timer_name = sw_update_type + ' nfvi audit' self._nfvi_timer_id = \ timers.timers_create_timer(self._nfvi_timer_name, 30, 30, self.nfvi_audit) self._nfvi_audit_inprogress = False
def schedule_task_work(self, task_work=None): """ Schedule task work to one of the task workers if available """ if task_work is not None: self._wait_queue.appendleft(task_work) if 0 == len(self._wait_queue): return False worker = self._task_worker_pool.claim_worker() if worker is not None: task_work = self._wait_queue.pop() DLOG.verbose("Pool %s: Task worker available to run TaskWork, " "name=%s." % (self._task_worker_pool.name, task_work.name)) selobj.selobj_add_read_obj(worker.selobj, self.task_work_complete) self._workers_selobj[worker.selobj] = worker worker.submit_task_work(task_work) if task_work.timeout_in_secs is not None: timer_id = timers.timers_create_timer( task_work.name, task_work.timeout_in_secs, task_work.timeout_in_secs, self.task_work_timeout) self._task_work_timers[timer_id] = task_work self._workers_timer[timer_id] = worker return True else: DLOG.verbose("Pool %s: No task worker available to run TaskWork." % self._task_worker_pool.name) return False
def refresh_timeouts(self): """ Stage Refresh Timeouts """ if not self.is_inprogress(): # No need to refresh stage timer, stage not started return if self._timer_id is not None: timers.timers_delete_timer(self._timer_id) self._timer_id = None # Calculate overall stage timeout self._timeout_in_secs = 0 for step in self._steps: self._timeout_in_secs += step.timeout_in_secs if 0 < self._timeout_in_secs: self._timeout_in_secs += 1 # Re-start stage timer self._timer_id = timers.timers_create_timer(self._name, self._timeout_in_secs, self._timeout_in_secs, self._timeout) DLOG.verbose( "Started overall strategy stage timer, timeout_in_sec=%s" % self._timeout_in_secs) if self._step_timer_id is not None: timers.timers_delete_timer(self._step_timer_id) self._step_timer_id = None if len(self._steps) <= self._current_step: # No need to refresh step timer, no current step being applied return # Re-start step timer step = self._steps[self._current_step] if 0 < step.timeout_in_secs: self._step_timer_id = timers.timers_create_timer( step.name, step.timeout_in_secs, step.timeout_in_secs, self._step_timeout) DLOG.verbose("Started strategy step timer, timeout_in_sec=%s" % step.timeout_in_secs)
def start(self): """ Start the Thread """ self._process.start() if self._check_timer_id is None: self._check_timer_id = timers.timers_create_timer( self._name, self._check_interval_in_secs, self._check_interval_in_secs, self.do_check)
def add_task_timer(self, name, interval_secs, task): """ Add timer for a task """ timer_id = timers.timers_create_timer(name, interval_secs, interval_secs, self.task_timer_timeout) self._task_timers[timer_id] = task return timer_id
def initialize(self): """ Initialize the Alarm Worker """ self._handlers = AlarmHandlers(self._config['namespace'], self._config['handlers']) self._handlers.initialize(self._config['config_file']) self._alarm_audit_timer_id = timers.timers_create_timer( 'alarm_audit', int(self._config['audit_interval']), int(self._config['audit_interval']), self._alarm_audit)
def refresh_timeouts(self): """ State Task Refresh Timeouts """ if self._timer_id is None: # No need to refresh task timer, task not started return timers.timers_delete_timer(self._timer_id) self._timer_id = None # Calculate overall task timeout self._timeout_in_secs = 0 for task_work in self._task_work_list: self._timeout_in_secs += task_work.timeout_in_secs if 0 < self._timeout_in_secs: self._timeout_in_secs += 1 # Re-start task timer self._timer_id = timers.timers_create_timer(self._name, self._timeout_in_secs, self._timeout_in_secs, self._timeout) if self._task_work_timer_id is None: # No need to refresh task work timer, no task work running return timers.timers_delete_timer(self._task_work_timer_id) self._task_work_timer_id = None if len(self._task_work_list) <= self._current_task_work: # No need to refresh task work timer, no current task work running return # Re-start task work timer task_work = self._task_work_list[self._current_task_work] if 0 < task_work.timeout_in_secs: self._task_work_timer_id = timers.timers_create_timer( task_work.name, task_work.timeout_in_secs, task_work.timeout_in_secs, self._task_work_timeout)
def commit(self): if self._commit_inline: start_ms = timers.get_monotonic_timestamp_in_ms() self._session.commit() elapsed_ms = timers.get_monotonic_timestamp_in_ms() - start_ms histogram.add_histogram_data("database-commits (inline)", elapsed_ms / 100, "decisecond") else: if self._commit_timer_id is None: self._commit_timer_id \ = timers.timers_create_timer('db-auto-commit', 1, 1, self.auto_commit)
def start(self): """ State Task Start """ self._cleanup() self._current_task_work = 0 self._task_inprogress = True self._task_result = STATE_TASK_RESULT.SUCCESS self._task_result_reason = 'success' self._timer_id = timers.timers_create_timer(self._name, self._timeout_in_secs, self._timeout_in_secs, self._timeout) self._run()
def _run(self): """ State Task Run """ if not self._task_inprogress: DLOG.debug("Task (%s) not inprogress." % self._name) return for idx in range(self._current_task_work, len(self._task_work_list), 1): task_work = self._task_work_list[idx] if self._task_work_timer_id is not None: timers.timers_delete_timer(self._task_work_timer_id) self._task_work_timer_id = None DLOG.debug("Task %s running %s work." % (self._name, task_work.name)) task_work_result, task_work_result_reason = task_work.run() self._current_task_work = idx if STATE_TASK_WORK_RESULT.WAIT == task_work_result: if 0 < task_work.timeout_in_secs: self._task_work_timer_id = timers.timers_create_timer( task_work.name, task_work.timeout_in_secs, task_work.timeout_in_secs, self._task_work_timeout) DLOG.debug("Task (%s) is waiting for work (%s) to complete, " "timeout_in_secs=%s." % (self._name, task_work.name, task_work.timeout_in_secs)) break else: self._task_result, self._task_result_reason = \ state_task_result_update( self._task_result, self._task_result_reason, task_work_result, task_work_result_reason) if STATE_TASK_RESULT.FAILED == self._task_result \ or STATE_TASK_RESULT.ABORTED == self._task_result \ or STATE_TASK_RESULT.TIMED_OUT == self._task_result: self._abort() self._complete(self._task_result, self._task_result_reason) break else: DLOG.debug("Task (%s) done running." % self._name) self._task_inprogress = False self._cleanup() self._complete(self._task_result, self._task_result_reason)
def strategy_apply(self, strategy_uuid, stage_id): """ Apply a software update strategy """ success = False if self.strategy is None: reason = "strategy not created" elif strategy_uuid != self.strategy.uuid: reason = "strategy does not exist" else: event_log.sw_update_issue_log( self.event_id(SW_UPDATE_EVENT_IDS.APPLY_START)) success, reason = self.strategy.apply(stage_id) if success: if self._alarms: alarm.clear_sw_update_alarm(self._alarms) self._alarms = \ alarm.raise_sw_update_alarm( self.alarm_type(SW_UPDATE_ALARM_TYPES.APPLY_INPROGRESS)) event_log.sw_update_issue_log( self.event_id(SW_UPDATE_EVENT_IDS.APPLY_INPROGRESS)) if self._nfvi_timer_id is not None: timers.timers_delete_timer(self._nfvi_timer_id) self._nfvi_timer_id = None self._nfvi_timer_id = \ timers.timers_create_timer(self._nfvi_timer_name, 30, 30, self.nfvi_audit) else: event_log.sw_update_issue_log(self.event_id( SW_UPDATE_EVENT_IDS.APPLY_REJECTED), reason=reason) return success, reason
def _apply(self): """ Stage Apply """ if not self._inprogress: if 0 == self._current_step: self._cleanup() self._inprogress = True self._result = STRATEGY_STAGE_RESULT.INPROGRESS self._result_reason = '' self._start_date_time = datetime.now().strftime( "%Y-%m-%d %H:%M:%S") if 0 < self.timeout_in_secs: self._timer_id = timers.timers_create_timer( self._name, self._timeout_in_secs, self._timeout_in_secs, self._timeout) else: DLOG.debug("Stage (%s) not inprogress." % self._name) return self._result, self._result_reason for idx in range(self._current_step, len(self._steps), 1): step = self._steps[idx] if self._step_timer_id is not None: timers.timers_delete_timer(self._step_timer_id) self._step_timer_id = None step.start_date_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") step.result, step.result_reason = step.apply() self._current_step = idx if STRATEGY_STEP_RESULT.WAIT == step.result: if 0 < step.timeout_in_secs: self._step_timer_id = timers.timers_create_timer( step.name, step.timeout_in_secs, step.timeout_in_secs, self._step_timeout) DLOG.debug("Stage (%s) is waiting for step (%s) to complete, " "timeout_in_secs=%s." % (self.name, step.name, step.timeout_in_secs)) self._save() return STRATEGY_STAGE_RESULT.WAIT, '' else: DLOG.debug( "Stage (%s) step (%s) complete, result=%s, reason=%s." % (self._name, step.name, step.result, step.result_reason)) step.end_date_time = datetime.now().strftime( "%Y-%m-%d %H:%M:%S") self._result, self._result_reason = \ strategy_stage_result_update(self._result, self._result_reason, step.result, step.result_reason) if STRATEGY_STAGE_RESULT.FAILED == self._result or \ STRATEGY_STAGE_RESULT.ABORTED == self._result or \ STRATEGY_STAGE_RESULT.TIMED_OUT == self._result: return self._complete(self._result, self._result_reason) else: self._save() else: # Check for a stage with no steps if 0 == self._current_step: self._result = STRATEGY_STAGE_RESULT.SUCCESS self._result_reason = '' DLOG.debug("Stage (%s) done running, result=%s, reason=%s." % (self._name, self._result, self._result_reason)) return self._complete(self._result, self._result_reason)
def _apply(self): """ Phase Apply """ if not self._inprogress: if 0 == self._current_stage: self._cleanup() self._current_stage = 0 self._inprogress = True self._result = STRATEGY_PHASE_RESULT.INPROGRESS self._result_reason = '' self._start_date_time = datetime.now().strftime( "%Y-%m-%d %H:%M:%S") else: DLOG.debug("Phase (%s) not inprogress." % self._name) return self._result, self._result_reason if self._timer_id is None: timeout_in_secs = 0 for idx in range(self._current_stage, self._stop_at_stage, 1): stage = self._stages[idx] timeout_in_secs += stage.timeout_in_secs if 0 < timeout_in_secs: self._timeout_in_secs = timeout_in_secs + 1 self._timer_id = timers.timers_create_timer( self._name, self._timeout_in_secs, self._timeout_in_secs, self._timeout) for idx in range(self._current_stage, self._stop_at_stage, 1): stage = self._stages[idx] if self._stage_timer_id is not None: timers.timers_delete_timer(self._stage_timer_id) self._stage_timer_id = None DLOG.info("Phase %s running %s stage." % (self._name, stage.name)) stage_result, stage_result_reason = stage.apply() self._current_stage = idx if STRATEGY_STAGE_RESULT.WAIT == stage_result: if 0 < stage.timeout_in_secs: self._stage_timer_id = timers.timers_create_timer( stage.name, stage.timeout_in_secs, stage.timeout_in_secs, self._stage_timeout) DLOG.debug("Phase (%s) is waiting for stage (%s) to complete, " "timeout_in_secs=%s." % (self._name, stage.name, stage.timeout_in_secs)) self._save() return STRATEGY_PHASE_RESULT.WAIT, '' else: DLOG.debug( "Phase (%s) stage (%s) complete, result=%s, reason=%s." % (self._name, stage.name, stage_result, stage_result_reason)) self._result, self._result_reason = \ strategy_phase_result_update(self._result, self._result_reason, stage_result, stage_result_reason) if STRATEGY_PHASE_RESULT.FAILED == self._result \ or STRATEGY_PHASE_RESULT.ABORTED == self._result \ or STRATEGY_PHASE_RESULT.TIMED_OUT == self._result: return self._complete(self._result, self._result_reason) else: self._save() else: # Check if this is an intermediate stop or the phase has been completed if self._stop_at_stage == len(self._stages): # Check for a phase with no stages if 0 == self._current_stage: self._result = STRATEGY_PHASE_RESULT.SUCCESS self._result_reason = '' DLOG.debug("Phase (%s) done running, result=%s, reason=%s." % (self._name, self._result, self._result_reason)) return self._complete(self._result, self._result_reason) else: self._cleanup() self._save() return self._result, self._result_reason