def test_repr(self):
# just verify these don't crash
tm = timeout.Timeout(1)
greenthread.sleep(0)
repr(tm)
str(tm)
tm.cancel()
tm = timeout.Timeout(None, RuntimeError)
repr(tm)
str(tm)
tm = timeout.Timeout(None, False)
repr(tm)
str(tm)
def _start_transfer(read_handle, write_handle, timeout_secs):
# read_handle/write_handle could be an NFC lease, so we need to
# periodically update its progress
read_updater = _create_progress_updater(read_handle)
write_updater = _create_progress_updater(write_handle)
timer = timeout.Timeout(timeout_secs)
try:
while True:
data = read_handle.read(CHUNK_SIZE)
if not data:
break
write_handle.write(data)
except timeout.Timeout as excep:
msg = (_('Timeout, read_handle: "%(src)s", write_handle: "%(dest)s"') %
{'src': read_handle,
'dest': write_handle})
LOG.exception(msg)
raise exceptions.ImageTransferException(msg, excep)
except Exception as excep:
msg = (_('Error, read_handle: "%(src)s", write_handle: "%(dest)s"') %
{'src': read_handle,
'dest': write_handle})
LOG.exception(msg)
raise exceptions.ImageTransferException(msg, excep)
finally:
timer.cancel()
if read_updater:
read_updater.stop()
if write_updater:
write_updater.stop()
read_handle.close()
write_handle.close()
def do_receive(tp):
timeout.Timeout(0, RuntimeError())
try:
t = tp.get()
self.fail("Shouldn't have recieved anything from the pool")
except RuntimeError:
return 'timed out'
def _login_with_password(self, user, pw, session):
login_exception = XenAPI.Failure(
_("Unable to log in to XenAPI "
"(is the Dom0 disk full?)"))
with timeout.Timeout(self.timeout, login_exception):
session.login_with_password(user, pw, self.PLUGIN_REQUIRED_VERSION,
self.originator)
def _poll_for_lpar_status(self, instance_name, status, operation,
timeout=constants.POWERVM_LPAR_OPERATION_TIMEOUT):
"""Polls until the LPAR with the given name reaches the given status.
:param instance_name: LPAR instance name
:param status: Poll until the given LPAR status is reached
:param operation: The operation being performed, e.g. 'stop_lpar'
:param timeout: The number of seconds to wait.
:raises: PowerVMLPARInstanceNotFound
:raises: PowerVMLPAROperationTimeout
:raises: InvalidParameterValue
"""
# make sure it's a valid status
if (status == constants.POWERVM_NOSTATE or
not status in constants.POWERVM_POWER_STATE):
msg = _("Invalid LPAR state: %s") % status
raise n_exc.InvalidParameterValue(err=msg)
# raise the given timeout exception if the loop call doesn't complete
# in the specified timeout
timeout_exception = exception.PowerVMLPAROperationTimeout(
operation=operation,
instance_name=instance_name)
with eventlet_timeout.Timeout(timeout, timeout_exception):
def _wait_for_lpar_status(instance_name, status):
"""Called at an interval until the status is reached."""
lpar_obj = self.get_lpar(instance_name)
if lpar_obj['state'] == status:
raise loopingcall.LoopingCallDone()
timer = loopingcall.FixedIntervalLoopingCall(_wait_for_lpar_status,
instance_name, status)
timer.start(interval=1).wait()
def _populate_session_pool(self, url, user, pw, exception):
for i in xrange(CONF.xenserver.connection_concurrent - 1):
session = self._create_session(url)
with timeout.Timeout(CONF.xenserver.login_timeout, exception):
session.login_with_password(user, pw, self.nova_version,
'OpenStack')
self._sessions.put(session)
def test_direct_raise_instance(self):
tm = timeout.Timeout()
try:
raise tm
except timeout.Timeout, t:
assert tm is t, (tm, t)
assert not t.pending, repr(t)
def _start_transfer(read_handle, write_handle, timeout_secs):
# write_handle could be an NFC lease, so we need to periodically
# update its progress
update_cb = getattr(write_handle, 'update_progress', lambda: None)
updater = loopingcall.FixedIntervalLoopingCall(update_cb)
timer = timeout.Timeout(timeout_secs)
try:
updater.start(interval=NFC_LEASE_UPDATE_PERIOD)
while True:
data = read_handle.read(CHUNK_SIZE)
if not data:
break
write_handle.write(data)
except timeout.Timeout as excep:
msg = (_('Timeout, read_handle: "%(src)s", write_handle: "%(dest)s"') %
{'src': read_handle,
'dest': write_handle})
LOG.exception(msg)
raise exceptions.ImageTransferException(msg, excep)
except Exception as excep:
msg = (_('Error, read_handle: "%(src)s", write_handle: "%(dest)s"') %
{'src': read_handle,
'dest': write_handle})
LOG.exception(msg)
raise exceptions.ImageTransferException(msg, excep)
finally:
timer.cancel()
updater.stop()
read_handle.close()
write_handle.close()
def provider_run(aliases_dict, tiid, method_name, provider_name):
provider = ProviderFactory.get_provider(provider_name)
# logger.info(u"in provider_run for {provider}".format(
# provider=provider.provider_name))
(success, estimated_wait_seconds) = rate.acquire(provider_name, block=False)
# add up to random 2 seconds to spread it out
estimated_wait_seconds += random.random() * 3
if not success:
logger.warning(u"RATE LIMIT HIT in provider_run for {provider} {method_name} {tiid}, retrying".format(
provider=provider.provider_name, method_name=method_name, tiid=tiid))
provider_run.retry(args=[aliases_dict, tiid, method_name, provider_name],
countdown=estimated_wait_seconds,
max_retries=10)
timeout_seconds = 30
try:
with timeout.Timeout(timeout_seconds):
response = provider_method_wrapper(tiid, aliases_dict, provider, method_name)
except timeout.Timeout:
msg = u"TIMEOUT in provider_run for {provider} {method_name} {tiid} after {timeout_seconds} seconds".format(
provider=provider.provider_name, method_name=method_name, tiid=tiid, timeout_seconds=timeout_seconds)
# logger.warning(msg) # message is written elsewhere
raise ProviderTimeout(msg)
return response
def __init__(self, url, user, pw):
self.XenAPI = self.get_imported_xenapi()
self._session = self._create_session(url)
exception = self.XenAPI.Failure(_("Unable to log in to XenAPI "
"(is the Dom0 disk full?)"))
with timeout.Timeout(FLAGS.xenapi_login_timeout, exception):
self._session.login_with_password(user, pw)
def start_transfer(context, timeout_secs, read_file_handle, max_data_size,
write_file_handle=None, image_service=None, image_id=None,
image_meta=None):
"""Start the data transfer from the reader to the writer.
Reader writes to the pipe and the writer reads from the pipe. This means
that the total transfer time boils down to the slower of the read/write
and not the addition of the two times.
"""
if not image_meta:
image_meta = {}
# The pipe that acts as an intermediate store of data for reader to write
# to and writer to grab from.
thread_safe_pipe = io_util.ThreadSafePipe(QUEUE_BUFFER_SIZE, max_data_size)
# The read thread. In case of glance it is the instance of the
# GlanceFileRead class. The glance client read returns an iterator
# and this class wraps that iterator to provide datachunks in calls
# to read.
read_thread = io_util.IOThread(read_file_handle, thread_safe_pipe)
# In case of Glance - VMware transfer, we just need a handle to the
# HTTP Connection that is to send transfer data to the VMware datastore.
if write_file_handle:
write_thread = io_util.IOThread(thread_safe_pipe, write_file_handle)
# In case of VMware - Glance transfer, we relinquish VMware HTTP file read
# handle to Glance Client instance, but to be sure of the transfer we need
# to be sure of the status of the image on glance changing to active.
# The GlanceWriteThread handles the same for us.
elif image_service and image_id:
write_thread = io_util.GlanceWriteThread(context, thread_safe_pipe,
image_service, image_id,
image_meta)
# Start the read and write threads.
read_event = read_thread.start()
write_event = write_thread.start()
timer = timeout.Timeout(timeout_secs)
try:
# Wait on the read and write events to signal their end
read_event.wait()
write_event.wait()
except (timeout.Timeout, Exception) as exc:
# In case of any of the reads or writes raising an exception,
# stop the threads so that we un-necessarily don't keep the other one
# waiting.
read_thread.stop()
write_thread.stop()
# Log and raise the exception.
LOG.exception(exc)
raise exception.CinderException(exc)
finally:
timer.cancel()
# No matter what, try closing the read and write handles, if it so
# applies.
read_file_handle.close()
if write_file_handle:
write_file_handle.close()
def _run_with_log(self, func, timeout, *args, **kwargs):
start_time = time.time()
try:
with e_timeout.Timeout(timeout, ex.TimeoutException(timeout)):
return self._run(func, *args, **kwargs)
finally:
self._log_command('%s took %.1f seconds to complete' % (
func.__name__, time.time() - start_time))
def connect(cls, db_module, connect_timeout, *args, **kw):
t = timeout.Timeout(connect_timeout, ConnectTimeout())
try:
from eventlet import tpool
conn = tpool.execute(db_module.connect, *args, **kw)
return tpool.Proxy(conn, autowrap_names=('cursor', ))
finally:
t.cancel()
def connect(cls, db_driver, conn_timeout, *args, **kwargs):
"Override ``Connection.Pool`` connect to use a failover connect."
from eventlet import timeout, tpool
t = timeout.Timeout(conn_timeout, db_pool.ConnectTimeout())
try:
conn = tpool.execute(_get_connection_maker(db_driver, kwargs))
return tpool.Proxy(conn, autowrap_names=('cursor',))
finally:
t.cancel()
def test_wait_for_power_state_false(self, mock_with_timeout):
desired_power_state = mock.sentinel.power_state
mock_with_timeout.side_effect = etimeout.Timeout()
response = vmutils.wait_for_power_state(
self._instance, desired_power_state,
constants.SHUTDOWN_RETRY_INTERVAL)
self.assertFalse(response)
def _timeout(self, call):
with timeout.Timeout(self.timeout_seconds):
while True:
try:
return call()
except sqlite3.OperationalError as e:
if 'locked' not in str(e):
raise
sleep(0.05)
def __init__(self, url, user, pw):
self.XenAPI = self.get_imported_xenapi()
self._sessions = queue.Queue()
exception = self.XenAPI.Failure(_("Unable to log in to XenAPI "
"(is the Dom0 disk full?)"))
for i in xrange(FLAGS.xenapi_connection_concurrent):
session = self._create_session(url)
with timeout.Timeout(FLAGS.xenapi_login_timeout, exception):
session.login_with_password(user, pw)
self._sessions.put(session)
def _run(self):
if self._request_timeout:
# No timeout exception escapes the with block.
with timeout.Timeout(self._request_timeout, False):
return self._handle_request()
LOG.info('Request timeout handling request.')
self._request_error = Exception('Request timeout')
return None
else:
return self._handle_request()
def setUp(self):
super(SchedulerServiceTest, self).setUp()
self.timeout = timeout.Timeout(seconds=10)
self.queue = queue.Queue()
self.scheduler = scheduler.Scheduler(0, 1, None)
self.scheduler.start()
self.addCleanup(self.scheduler.stop, True)
self.addCleanup(self.timeout.cancel)
def _run_with_log(self, func, timeout, description, *args, **kwargs):
start_time = time.time()
try:
with e_timeout.Timeout(
timeout, ex.TimeoutException(timeout,
op_name=description)):
return self._run(func, *args, **kwargs)
finally:
self._log_command('"%s" took %.1f seconds to complete' %
(description, time.time() - start_time))
def _run(self):
'''Method executed within green thread.'''
if self._request_timeout:
# No timeout exception escapes the with block.
with timeout.Timeout(self._request_timeout, False):
return self._handle_request()
lg.info('[%d] Request timeout.' % self._rid())
self._request_error = Exception('Request timeout')
return None
else:
return self._handle_request()
def shutdown_subprocess(proc, cleanup_func):
try:
with e_timeout.Timeout(5):
# timeout would mean that our single-threaded subprocess
# is hung on previous task which blocks _finish to complete
run_in_subprocess(proc, _finish, (cleanup_func, ))
except BaseException:
# exception could be caused by either timeout, or
# successful shutdown, ignoring anyway
pass
finally:
kill_subprocess(proc)
def assert_pool_has_free(self, pool, num_free):
def wait_long_time(e):
e.wait()
timer = timeout.Timeout(1, api.TimeoutError)
try:
evt = _event.Event()
for x in six.moves.range(num_free):
pool.execute(wait_long_time, evt)
# if the pool has fewer free than we expect,
# then we'll hit the timeout error
finally:
timer.cancel()
# if the runtime error is not raised it means the pool had
# some unexpected free items
timer = timeout.Timeout(0, RuntimeError)
self.assertRaises(RuntimeError, pool.execute, wait_long_time, evt)
# clean up by causing all the wait_long_time functions to return
evt.send(None)
api.sleep(0)
api.sleep(0)
def _create_first_session(self, url, user, pw, exception):
try:
session = self._create_session(url)
with timeout.Timeout(CONF.xenapi_login_timeout, exception):
session.login_with_password(user, pw)
except self.XenAPI.Failure, e:
# if user and pw of the master are different, we're doomed!
if e.details[0] == 'HOST_IS_SLAVE':
master = e.details[1]
url = pool.swap_xapi_host(url, master)
session = self.XenAPI.Session(url)
session.login_with_password(user, pw)
self.is_slave = True
else:
raise
def setUp(self):
super(LegacySchedulerTest, self).setUp()
self.timeout = timeout.Timeout(seconds=10)
self.queue = queue.Queue()
self.override_config('fixed_delay', 1, 'scheduler')
self.override_config('random_delay', 0, 'scheduler')
self.override_config('batch_size', 100, 'scheduler')
self.scheduler = legacy_scheduler.LegacyScheduler(CONF.scheduler)
self.scheduler.start()
self.addCleanup(self.scheduler.stop, True)
self.addCleanup(self.timeout.cancel)
def setUp(self):
super(SchedulerTest, self).setUp()
# This Timeout object is needed to raise an exception if the test took
# longer than a configured number of seconds.
self.timeout = timeout.Timeout(seconds=15)
# Synchronization primitives to control when a scheduled invoked
# method is allowed to enter the method and exit from it to perform
# all needed checks.
self.target_mtd_started = event.Event()
self.target_mtd_finished = event.Event()
self.target_mtd_lock = semaphore.Semaphore(0)
self.scheduler = default_scheduler.DefaultScheduler(1, 1, 100)
self.scheduler.start()
self.addCleanup(self.scheduler.stop, True)
self.addCleanup(self.timeout.cancel)
def execute(self):
timeout = CONF.cluster_verifications.verification_timeout
try:
with e_timeout.Timeout(timeout, ex.TimeoutException(timeout)):
if not self.is_available():
return
self._indicate_start()
try:
result = self.check_health()
status = common.HEALTH_STATUS_GREEN
except Exception as exc:
result = six.text_type(exc)
if isinstance(exc, BaseHealthError):
status = exc.status
else:
status = common.HEALTH_STATUS_RED
except ex.TimeoutException:
result = _("Health check timed out")
status = common.HEALTH_STATUS_YELLOW
self._write_result(status, result)
def get_api_session():
if not api:
raise ImportError('XenAPI not installed')
url = CONF.xenapi.connection_url
username = CONF.xenapi.connection_username
password = CONF.xenapi.connection_password
if not url or password is None:
raise XenapiException('Must specify connection_url, and '
'connection_password to use')
exception = api.Failure("Unable to log in to XenAPI "
"(is the Dom0 disk full?)")
try:
session = api.Session(url)
with timeout.Timeout(CONF.xenapi.login_timeout, exception):
session.login_with_password(username, password)
except api.Failure as e:
msg = "Could not connect to XenAPI: %s" % e.details[0]
raise XenapiException(msg)
return session
def test_action_timeout(self):
wf_text = """---
version: '2.0'
wf1:
tasks:
task1:
action: std.sleep seconds=10
timeout: 2
"""
wf_service.create_workflows(wf_text)
wf_ex = self.engine.start_workflow('wf1')
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
action_ex = task_ex.action_executions[0]
with timeout.Timeout(8):
self.await_workflow_error(wf_ex.id)
self.await_task_error(task_ex.id)
self.await_action_error(action_ex.id)
def test_stderr_raising(self):
# testing that really egregious errors in the error handling code
# (that prints tracebacks to stderr) don't cause the pool to lose
# any members
import sys
pool = self.klass(min_size=1, max_size=1)
def crash(*args, **kw):
raise RuntimeError("Whoa")
class FakeFile(object):
write = crash
# we're going to do this by causing the traceback.print_exc in
# safe_apply to raise an exception and thus exit _main_loop
normal_err = sys.stderr
try:
sys.stderr = FakeFile()
waiter = pool.execute(crash)
self.assertRaises(RuntimeError, waiter.wait)
# the pool should have something free at this point since the
# waiter returned
# pool.Pool change: if an exception is raised during execution of a link,
# the rest of the links are scheduled to be executed on the next hub iteration
# this introduces a delay in updating pool.sem which makes pool.free() report 0
# therefore, sleep:
api.sleep(0)
self.assertEqual(pool.free(), 1)
# shouldn't block when trying to get
t = timeout.Timeout(0.1)
try:
pool.execute(api.sleep, 1)
finally:
t.cancel()
finally:
sys.stderr = normal_err
