def run_periodic_tasks(self, context, raise_on_error=False): """Tasks to be run at a periodic interval.""" idle_for = DEFAULT_INTERVAL for task_name, task in self._periodic_tasks: full_task_name = '.'.join([self.__class__.__name__, task_name]) spacing = self._periodic_spacing[task_name] last_run = self._periodic_last_run[task_name] # If a periodic task is _nearly_ due, then we'll run it early idle_for = min(idle_for, spacing) if last_run is not None: delta = last_run + spacing - time.time() if delta > 0.2: idle_for = min(idle_for, delta) continue LOG.debug("Running periodic task %(full_task_name)s", {"full_task_name": full_task_name}) self._periodic_last_run[task_name] = time.time() try: task(self, context) except Exception as e: if raise_on_error: raise LOG.exception(_LE("Error during %(full_task_name)s: %(e)s"), { "full_task_name": full_task_name, "e": e }) time.sleep(0) return idle_for
def _wrap(self, *args, **kwargs): try: assert issubclass(self.__class__, sqlalchemy.orm.session.Session), ( "_wrap_db_error() can only be applied to methods of " "subclasses of sqlalchemy.orm.session.Session." ) return f(self, *args, **kwargs) except UnicodeEncodeError: raise exception.DBInvalidUnicodeParameter() except sqla_exc.OperationalError as e: _raise_if_db_connection_lost(e, self.bind) _raise_if_deadlock_error(e, self.bind.dialect.name) # NOTE(comstud): A lot of code is checking for OperationalError # so let's not wrap it for now. raise # note(boris-42): We should catch unique constraint violation and # wrap it by our own DBDuplicateEntry exception. Unique constraint # violation is wrapped by IntegrityError. except sqla_exc.IntegrityError as e: # note(boris-42): SqlAlchemy doesn't unify errors from different # DBs so we must do this. Also in some tables (for example # instance_types) there are more than one unique constraint. This # means we should get names of columns, which values violate # unique constraint, from error message. _raise_if_duplicate_entry_error(e, self.bind.dialect.name) raise exception.DBError(e) except Exception as e: LOG.exception(_LE("DB exception wrapped.")) raise exception.DBError(e)
def inner_func(*args, **kwargs): last_log_time = 0 last_exc_message = None exc_count = 0 while True: try: return infunc(*args, **kwargs) except Exception as exc: this_exc_message = six.u(str(exc)) if this_exc_message == last_exc_message: exc_count += 1 else: exc_count = 1 # Do not log any more frequently than once a minute unless # the exception message changes cur_time = int(time.time()) if (cur_time - last_log_time > 60 or this_exc_message != last_exc_message): logging.exception( _LE('Unexpected exception occurred %d time(s)... ' 'retrying.') % exc_count) last_log_time = cur_time last_exc_message = this_exc_message exc_count = 0 # This should be a very rare event. In case it isn't, do # a sleep. time.sleep(1)
def run_periodic_tasks(self, context, raise_on_error=False): """Tasks to be run at a periodic interval.""" idle_for = DEFAULT_INTERVAL for task_name, task in self._periodic_tasks: full_task_name = '.'.join([self.__class__.__name__, task_name]) spacing = self._periodic_spacing[task_name] last_run = self._periodic_last_run[task_name] # If a periodic task is _nearly_ due, then we'll run it early idle_for = min(idle_for, spacing) if last_run is not None: delta = last_run + spacing - time.time() if delta > 0.2: idle_for = min(idle_for, delta) continue LOG.debug("Running periodic task %(full_task_name)s", {"full_task_name": full_task_name}) self._periodic_last_run[task_name] = time.time() try: task(self, context) except Exception as e: if raise_on_error: raise LOG.exception(_LE("Error during %(full_task_name)s: %(e)s"), {"full_task_name": full_task_name, "e": e}) time.sleep(0) return idle_for
def _wait_for_exit_or_signal(self, ready_callback=None): status = None signo = 0 LOG.debug("Full set of CONF:") CONF.log_opt_values(LOG, std_logging.DEBUG) try: if ready_callback: ready_callback() super(ServiceLauncher, self).wait() except SignalExit as exc: signame = _signo_to_signame(exc.signo) LOG.info(_LI("Caught %s, exiting"), signame) status = exc.code signo = exc.signo except SystemExit as exc: status = exc.code finally: self.stop() if rpc: try: rpc.cleanup() except Exception: # We're shutting down, so it doesn't matter at this point. LOG.exception(_LE("Exception during rpc cleanup.")) return status, signo
def _wrap(self, *args, **kwargs): try: assert issubclass( self.__class__, sqlalchemy.orm.session.Session), ( '_wrap_db_error() can only be applied to methods of ' 'subclasses of sqlalchemy.orm.session.Session.') return f(self, *args, **kwargs) except UnicodeEncodeError: raise exception.DBInvalidUnicodeParameter() except sqla_exc.OperationalError as e: _raise_if_db_connection_lost(e, self.bind) _raise_if_deadlock_error(e, self.bind.dialect.name) # NOTE(comstud): A lot of code is checking for OperationalError # so let's not wrap it for now. raise # note(boris-42): We should catch unique constraint violation and # wrap it by our own DBDuplicateEntry exception. Unique constraint # violation is wrapped by IntegrityError. except sqla_exc.IntegrityError as e: # note(boris-42): SqlAlchemy doesn't unify errors from different # DBs so we must do this. Also in some tables (for example # instance_types) there are more than one unique constraint. This # means we should get names of columns, which values violate # unique constraint, from error message. _raise_if_duplicate_entry_error(e, self.bind.dialect.name) raise exception.DBError(e) except Exception as e: LOG.exception(_LE('DB exception wrapped.')) raise exception.DBError(e)
def _wait_for_exit_or_signal(self, ready_callback=None): status = None signo = 0 LOG.debug('Full set of CONF:') CONF.log_opt_values(LOG, std_logging.DEBUG) try: if ready_callback: ready_callback() super(ServiceLauncher, self).wait() except SignalExit as exc: signame = _signo_to_signame(exc.signo) LOG.info(_LI('Caught %s, exiting'), signame) status = exc.code signo = exc.signo except SystemExit as exc: status = exc.code finally: self.stop() if rpc: try: rpc.cleanup() except Exception: # We're shutting down, so it doesn't matter at this point. LOG.exception(_LE('Exception during rpc cleanup.')) return status, signo
def wrapped(*args, **kwargs): try: return fn(*args, **kwargs) except Exception as e: LOG.exception( _LE('An exception occurred processing ' 'the API call: %s '), e)
def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is not None: if self.reraise: logging.error(_LE('Original exception being dropped: %s'), traceback.format_exception(self.type_, self.value, self.tb)) return False if self.reraise: six.reraise(self.type_, self.value, self.tb)
def wrapper(*args, **kwargs): next_interval = self.retry_interval remaining = self.max_retries while True: try: return f(*args, **kwargs) except exception.DBConnectionError as e: if remaining == 0: LOG.exception(_LE('DB exceeded retry limit.')) raise exception.DBError(e) if remaining != -1: remaining -= 1 LOG.exception(_LE('DB connection error.')) # NOTE(vsergeyev): We are using patched time module, so # this effectively yields the execution # context to another green thread. time.sleep(next_interval) if self.inc_retry_interval: next_interval = min(next_interval * 2, self.max_retry_interval)
def wrapper(*args, **kwargs): next_interval = self.retry_interval remaining = self.max_retries while True: try: return f(*args, **kwargs) except exception.DBConnectionError as e: if remaining == 0: LOG.exception(_LE('DB exceeded retry limit.')) raise exception.DBError(e) if remaining != -1: remaining -= 1 LOG.exception(_LE('DB connection error.')) # NOTE(vsergeyev): We are using patched time module, so # this effectively yields the execution # context to another green thread. time.sleep(next_interval) if self.inc_retry_interval: next_interval = min( next_interval * 2, self.max_retry_interval )
def _create_device_wait(self, context, device_dict): driver_name = self._infra_driver_name(device_dict) device_id = device_dict['id'] instance_id = self._instance_id(device_dict) create_failed = False try: self._device_manager.invoke(driver_name, 'create_wait', plugin=self, context=context, device_dict=device_dict, device_id=instance_id) except vnfm.DeviceCreateWaitFailed: LOG.error(_LE("VNF Create failed for vnf_id %s"), device_id) create_failed = True device_dict['status'] = constants.ERROR if instance_id is None or create_failed: mgmt_url = None else: # mgmt_url = self.mgmt_url(context, device_dict) # FIXME(yamahata): mgmt_url = device_dict['mgmt_url'] self._create_device_post(context, device_id, instance_id, mgmt_url, device_dict) self.mgmt_create_post(context, device_dict) if instance_id is None or create_failed: return device_dict['mgmt_url'] = mgmt_url kwargs = { mgmt_constants.KEY_ACTION: mgmt_constants.ACTION_CREATE_DEVICE, mgmt_constants.KEY_KWARGS: { 'device': device_dict }, } new_status = constants.ACTIVE try: self.mgmt_call(context, device_dict, kwargs) except Exception: LOG.exception(_('create_device_wait')) new_status = constants.ERROR device_dict['status'] = new_status self._create_device_status(context, device_id, new_status)
def _create_device_wait(self, context, device_dict): driver_name = self._infra_driver_name(device_dict) device_id = device_dict['id'] instance_id = self._instance_id(device_dict) create_failed = False try: self._device_manager.invoke( driver_name, 'create_wait', plugin=self, context=context, device_dict=device_dict, device_id=instance_id) except vnfm.DeviceCreateWaitFailed: LOG.error(_LE("VNF Create failed for vnf_id %s"), device_id) create_failed = True device_dict['status'] = constants.ERROR if instance_id is None or create_failed: mgmt_url = None else: # mgmt_url = self.mgmt_url(context, device_dict) # FIXME(yamahata): mgmt_url = device_dict['mgmt_url'] self._create_device_post( context, device_id, instance_id, mgmt_url, device_dict) self.mgmt_create_post(context, device_dict) if instance_id is None or create_failed: return device_dict['mgmt_url'] = mgmt_url kwargs = { mgmt_constants.KEY_ACTION: mgmt_constants.ACTION_CREATE_DEVICE, mgmt_constants.KEY_KWARGS: {'device': device_dict}, } new_status = constants.ACTIVE try: self.mgmt_call(context, device_dict, kwargs) except Exception: LOG.exception(_('create_device_wait')) new_status = constants.ERROR device_dict['status'] = new_status self._create_device_status(context, device_id, new_status)
def _child_wait_for_exit_or_signal(self, launcher): status = 0 signo = 0 # NOTE(johannes): All exceptions are caught to ensure this # doesn't fallback into the loop spawning children. It would # be bad for a child to spawn more children. try: launcher.wait() except SignalExit as exc: signame = _signo_to_signame(exc.signo) LOG.info(_LI("Child caught %s, exiting"), signame) status = exc.code signo = exc.signo except SystemExit as exc: status = exc.code except BaseException: LOG.exception(_LE("Unhandled exception")) status = 2 finally: launcher.stop() return status, signo
def _child_wait_for_exit_or_signal(self, launcher): status = 0 signo = 0 # NOTE(johannes): All exceptions are caught to ensure this # doesn't fallback into the loop spawning children. It would # be bad for a child to spawn more children. try: launcher.wait() except SignalExit as exc: signame = _signo_to_signame(exc.signo) LOG.info(_LI('Child caught %s, exiting'), signame) status = exc.code signo = exc.signo except SystemExit as exc: status = exc.code except BaseException: LOG.exception(_LE('Unhandled exception')) status = 2 finally: launcher.stop() return status, signo
def wrapped(*args, **kwargs): try: return fn(*args, **kwargs) except Exception as e: LOG.exception(_LE('An exception occurred processing ' 'the API call: %s '), e)