def _run_with_log(self, func, timeout, *args, **kwargs):
start_time = time.time()
try:
with e_timeout.Timeout(timeout, ex.TimeoutException(timeout)):
return self._run(func, *args, **kwargs)
finally:
self._log_command('%s took %.1f seconds to complete' % (
func.__name__, time.time() - start_time))
def poll(get_status,
kwargs=None,
args=None,
operation_name=None,
timeout_name=None,
timeout=DEFAULT_TIMEOUT,
sleep=DEFAULT_SLEEP_TIME,
exception_strategy='raise'):
"""This util poll status of object obj during some timeout.
:param get_status: function, which return current status of polling
as Boolean
:param kwargs: keyword arguments of function get_status
:param operation_name: name of polling process
:param timeout_name: name of timeout option
:param timeout: value of timeout in seconds. By default, it equals to
3 hours
:param sleep: duration between two consecutive executions of
get_status function
:param exception_strategy: possible values ('raise', 'mark_as_true',
'mark_as_false'). If exception_strategy is 'raise' exception would be
raised. If exception_strategy is 'mark_as_true', return value of
get_status would marked as True, and in case of 'mark_as_false' - False.
By default it's 'raise'.
"""
start_time = timeutils.utcnow()
# We shouldn't raise TimeoutException if incorrect timeout specified and
# status is ok now. In such way we should execute get_status at least once.
at_least_once = True
if not kwargs:
kwargs = {}
if not args:
args = ()
while at_least_once or _get_consumed(start_time) < timeout:
at_least_once = False
try:
status = get_status(*args, **kwargs)
except BaseException:
if exception_strategy == 'raise':
raise
elif exception_strategy == 'mark_as_true':
status = True
else:
status = False
if status:
operation = "Operation"
if operation_name:
operation = "Operation with name {op_name}".format(
op_name=operation_name)
LOG.debug('{operation_desc} was executed successfully in timeout '
'{timeout}'.format(operation_desc=operation,
timeout=timeout))
return
context.sleep(sleep)
raise ex.TimeoutException(timeout, operation_name, timeout_name)
def _run_with_log(self, func, timeout, description, *args, **kwargs):
start_time = time.time()
try:
with e_timeout.Timeout(
timeout, ex.TimeoutException(timeout,
op_name=description)):
return self._run(func, *args, **kwargs)
finally:
self._log_command('"%s" took %.1f seconds to complete' %
(description, time.time() - start_time))
def handler(*args, **kwargs):
start_time = timeutils.utcnow()
cluster = get_obj_in_args(check_object, *args, **kwargs)
while _get_consumed(start_time) < timeout:
consumed = _get_consumed(start_time)
if func(*args, **kwargs):
LOG.info(
_LI("Operation %(op_name)s was successfully executed "
"in seconds: %(sec)s"), {'op_name': op_name,
'sec': consumed})
return
if not check_cluster_exists(cluster):
return
context.sleep(sleeping_time)
raise e.TimeoutException(timeout, op_name)
def execute(self):
timeout = CONF.cluster_verifications.verification_timeout
try:
with e_timeout.Timeout(timeout, ex.TimeoutException(timeout)):
if not self.is_available():
return
self._indicate_start()
try:
result = self.check_health()
status = common.HEALTH_STATUS_GREEN
except Exception as exc:
result = six.text_type(exc)
if isinstance(exc, BaseHealthError):
status = exc.status
else:
status = common.HEALTH_STATUS_RED
except ex.TimeoutException:
result = _("Health check timed out")
status = common.HEALTH_STATUS_YELLOW
self._write_result(status, result)