def run_periodic_tasks(self, context, raise_on_error=False):
"""Tasks to be run at a periodic interval."""
idle_for = DEFAULT_INTERVAL
for task_name, task in self._periodic_tasks:
full_task_name = '.'.join([self.__class__.__name__, task_name])
spacing = self._periodic_spacing[task_name]
last_run = self._periodic_last_run[task_name]
# If a periodic task is _nearly_ due, then we'll run it early
idle_for = min(idle_for, spacing)
if last_run is not None:
delta = last_run + spacing - time.time()
if delta > 0.2:
idle_for = min(idle_for, delta)
continue
LOG.debug("Running periodic task %(full_task_name)s",
{"full_task_name": full_task_name})
self._periodic_last_run[task_name] = time.time()
try:
task(self, context)
except Exception as e:
if raise_on_error:
raise
LOG.exception(_LE("Error during %(full_task_name)s: %(e)s"), {
"full_task_name": full_task_name,
"e": e
})
time.sleep(0)
return idle_for
def _wrap(self, *args, **kwargs):
try:
assert issubclass(self.__class__, sqlalchemy.orm.session.Session), (
"_wrap_db_error() can only be applied to methods of " "subclasses of sqlalchemy.orm.session.Session."
)
return f(self, *args, **kwargs)
except UnicodeEncodeError:
raise exception.DBInvalidUnicodeParameter()
except sqla_exc.OperationalError as e:
_raise_if_db_connection_lost(e, self.bind)
_raise_if_deadlock_error(e, self.bind.dialect.name)
# NOTE(comstud): A lot of code is checking for OperationalError
# so let's not wrap it for now.
raise
# note(boris-42): We should catch unique constraint violation and
# wrap it by our own DBDuplicateEntry exception. Unique constraint
# violation is wrapped by IntegrityError.
except sqla_exc.IntegrityError as e:
# note(boris-42): SqlAlchemy doesn't unify errors from different
# DBs so we must do this. Also in some tables (for example
# instance_types) there are more than one unique constraint. This
# means we should get names of columns, which values violate
# unique constraint, from error message.
_raise_if_duplicate_entry_error(e, self.bind.dialect.name)
raise exception.DBError(e)
except Exception as e:
LOG.exception(_LE("DB exception wrapped."))
raise exception.DBError(e)
def inner_func(*args, **kwargs):
last_log_time = 0
last_exc_message = None
exc_count = 0
while True:
try:
return infunc(*args, **kwargs)
except Exception as exc:
this_exc_message = six.u(str(exc))
if this_exc_message == last_exc_message:
exc_count += 1
else:
exc_count = 1
# Do not log any more frequently than once a minute unless
# the exception message changes
cur_time = int(time.time())
if (cur_time - last_log_time > 60 or
this_exc_message != last_exc_message):
logging.exception(
_LE('Unexpected exception occurred %d time(s)... '
'retrying.') % exc_count)
last_log_time = cur_time
last_exc_message = this_exc_message
exc_count = 0
# This should be a very rare event. In case it isn't, do
# a sleep.
time.sleep(1)
def run_periodic_tasks(self, context, raise_on_error=False):
"""Tasks to be run at a periodic interval."""
idle_for = DEFAULT_INTERVAL
for task_name, task in self._periodic_tasks:
full_task_name = '.'.join([self.__class__.__name__, task_name])
spacing = self._periodic_spacing[task_name]
last_run = self._periodic_last_run[task_name]
# If a periodic task is _nearly_ due, then we'll run it early
idle_for = min(idle_for, spacing)
if last_run is not None:
delta = last_run + spacing - time.time()
if delta > 0.2:
idle_for = min(idle_for, delta)
continue
LOG.debug("Running periodic task %(full_task_name)s",
{"full_task_name": full_task_name})
self._periodic_last_run[task_name] = time.time()
try:
task(self, context)
except Exception as e:
if raise_on_error:
raise
LOG.exception(_LE("Error during %(full_task_name)s: %(e)s"),
{"full_task_name": full_task_name, "e": e})
time.sleep(0)
return idle_for
def _wait_for_exit_or_signal(self, ready_callback=None):
status = None
signo = 0
LOG.debug("Full set of CONF:")
CONF.log_opt_values(LOG, std_logging.DEBUG)
try:
if ready_callback:
ready_callback()
super(ServiceLauncher, self).wait()
except SignalExit as exc:
signame = _signo_to_signame(exc.signo)
LOG.info(_LI("Caught %s, exiting"), signame)
status = exc.code
signo = exc.signo
except SystemExit as exc:
status = exc.code
finally:
self.stop()
if rpc:
try:
rpc.cleanup()
except Exception:
# We're shutting down, so it doesn't matter at this point.
LOG.exception(_LE("Exception during rpc cleanup."))
return status, signo
def _wrap(self, *args, **kwargs):
try:
assert issubclass(
self.__class__, sqlalchemy.orm.session.Session), (
'_wrap_db_error() can only be applied to methods of '
'subclasses of sqlalchemy.orm.session.Session.')
return f(self, *args, **kwargs)
except UnicodeEncodeError:
raise exception.DBInvalidUnicodeParameter()
except sqla_exc.OperationalError as e:
_raise_if_db_connection_lost(e, self.bind)
_raise_if_deadlock_error(e, self.bind.dialect.name)
# NOTE(comstud): A lot of code is checking for OperationalError
# so let's not wrap it for now.
raise
# note(boris-42): We should catch unique constraint violation and
# wrap it by our own DBDuplicateEntry exception. Unique constraint
# violation is wrapped by IntegrityError.
except sqla_exc.IntegrityError as e:
# note(boris-42): SqlAlchemy doesn't unify errors from different
# DBs so we must do this. Also in some tables (for example
# instance_types) there are more than one unique constraint. This
# means we should get names of columns, which values violate
# unique constraint, from error message.
_raise_if_duplicate_entry_error(e, self.bind.dialect.name)
raise exception.DBError(e)
except Exception as e:
LOG.exception(_LE('DB exception wrapped.'))
raise exception.DBError(e)
def _wait_for_exit_or_signal(self, ready_callback=None):
status = None
signo = 0
LOG.debug('Full set of CONF:')
CONF.log_opt_values(LOG, std_logging.DEBUG)
try:
if ready_callback:
ready_callback()
super(ServiceLauncher, self).wait()
except SignalExit as exc:
signame = _signo_to_signame(exc.signo)
LOG.info(_LI('Caught %s, exiting'), signame)
status = exc.code
signo = exc.signo
except SystemExit as exc:
status = exc.code
finally:
self.stop()
if rpc:
try:
rpc.cleanup()
except Exception:
# We're shutting down, so it doesn't matter at this point.
LOG.exception(_LE('Exception during rpc cleanup.'))
return status, signo
def wrapped(*args, **kwargs):
try:
return fn(*args, **kwargs)
except Exception as e:
LOG.exception(
_LE('An exception occurred processing '
'the API call: %s '), e)
def __exit__(self, exc_type, exc_val, exc_tb):
if exc_type is not None:
if self.reraise:
logging.error(_LE('Original exception being dropped: %s'),
traceback.format_exception(self.type_,
self.value,
self.tb))
return False
if self.reraise:
six.reraise(self.type_, self.value, self.tb)
def wrapper(*args, **kwargs):
next_interval = self.retry_interval
remaining = self.max_retries
while True:
try:
return f(*args, **kwargs)
except exception.DBConnectionError as e:
if remaining == 0:
LOG.exception(_LE('DB exceeded retry limit.'))
raise exception.DBError(e)
if remaining != -1:
remaining -= 1
LOG.exception(_LE('DB connection error.'))
# NOTE(vsergeyev): We are using patched time module, so
# this effectively yields the execution
# context to another green thread.
time.sleep(next_interval)
if self.inc_retry_interval:
next_interval = min(next_interval * 2,
self.max_retry_interval)
def wrapper(*args, **kwargs):
next_interval = self.retry_interval
remaining = self.max_retries
while True:
try:
return f(*args, **kwargs)
except exception.DBConnectionError as e:
if remaining == 0:
LOG.exception(_LE('DB exceeded retry limit.'))
raise exception.DBError(e)
if remaining != -1:
remaining -= 1
LOG.exception(_LE('DB connection error.'))
# NOTE(vsergeyev): We are using patched time module, so
# this effectively yields the execution
# context to another green thread.
time.sleep(next_interval)
if self.inc_retry_interval:
next_interval = min(
next_interval * 2,
self.max_retry_interval
)
def _create_device_wait(self, context, device_dict):
driver_name = self._infra_driver_name(device_dict)
device_id = device_dict['id']
instance_id = self._instance_id(device_dict)
create_failed = False
try:
self._device_manager.invoke(driver_name,
'create_wait',
plugin=self,
context=context,
device_dict=device_dict,
device_id=instance_id)
except vnfm.DeviceCreateWaitFailed:
LOG.error(_LE("VNF Create failed for vnf_id %s"), device_id)
create_failed = True
device_dict['status'] = constants.ERROR
if instance_id is None or create_failed:
mgmt_url = None
else:
# mgmt_url = self.mgmt_url(context, device_dict)
# FIXME(yamahata):
mgmt_url = device_dict['mgmt_url']
self._create_device_post(context, device_id, instance_id, mgmt_url,
device_dict)
self.mgmt_create_post(context, device_dict)
if instance_id is None or create_failed:
return
device_dict['mgmt_url'] = mgmt_url
kwargs = {
mgmt_constants.KEY_ACTION: mgmt_constants.ACTION_CREATE_DEVICE,
mgmt_constants.KEY_KWARGS: {
'device': device_dict
},
}
new_status = constants.ACTIVE
try:
self.mgmt_call(context, device_dict, kwargs)
except Exception:
LOG.exception(_('create_device_wait'))
new_status = constants.ERROR
device_dict['status'] = new_status
self._create_device_status(context, device_id, new_status)
def _create_device_wait(self, context, device_dict):
driver_name = self._infra_driver_name(device_dict)
device_id = device_dict['id']
instance_id = self._instance_id(device_dict)
create_failed = False
try:
self._device_manager.invoke(
driver_name, 'create_wait', plugin=self, context=context,
device_dict=device_dict, device_id=instance_id)
except vnfm.DeviceCreateWaitFailed:
LOG.error(_LE("VNF Create failed for vnf_id %s"), device_id)
create_failed = True
device_dict['status'] = constants.ERROR
if instance_id is None or create_failed:
mgmt_url = None
else:
# mgmt_url = self.mgmt_url(context, device_dict)
# FIXME(yamahata):
mgmt_url = device_dict['mgmt_url']
self._create_device_post(
context, device_id, instance_id, mgmt_url, device_dict)
self.mgmt_create_post(context, device_dict)
if instance_id is None or create_failed:
return
device_dict['mgmt_url'] = mgmt_url
kwargs = {
mgmt_constants.KEY_ACTION: mgmt_constants.ACTION_CREATE_DEVICE,
mgmt_constants.KEY_KWARGS: {'device': device_dict},
}
new_status = constants.ACTIVE
try:
self.mgmt_call(context, device_dict, kwargs)
except Exception:
LOG.exception(_('create_device_wait'))
new_status = constants.ERROR
device_dict['status'] = new_status
self._create_device_status(context, device_id, new_status)
def _child_wait_for_exit_or_signal(self, launcher):
status = 0
signo = 0
# NOTE(johannes): All exceptions are caught to ensure this
# doesn't fallback into the loop spawning children. It would
# be bad for a child to spawn more children.
try:
launcher.wait()
except SignalExit as exc:
signame = _signo_to_signame(exc.signo)
LOG.info(_LI("Child caught %s, exiting"), signame)
status = exc.code
signo = exc.signo
except SystemExit as exc:
status = exc.code
except BaseException:
LOG.exception(_LE("Unhandled exception"))
status = 2
finally:
launcher.stop()
return status, signo
def _child_wait_for_exit_or_signal(self, launcher):
status = 0
signo = 0
# NOTE(johannes): All exceptions are caught to ensure this
# doesn't fallback into the loop spawning children. It would
# be bad for a child to spawn more children.
try:
launcher.wait()
except SignalExit as exc:
signame = _signo_to_signame(exc.signo)
LOG.info(_LI('Child caught %s, exiting'), signame)
status = exc.code
signo = exc.signo
except SystemExit as exc:
status = exc.code
except BaseException:
LOG.exception(_LE('Unhandled exception'))
status = 2
finally:
launcher.stop()
return status, signo
def wrapped(*args, **kwargs):
try:
return fn(*args, **kwargs)
except Exception as e:
LOG.exception(_LE('An exception occurred processing '
'the API call: %s '), e)