def _run(self):
"""
Processes all queued up requests for this RPM Controller.
Callers should first request_queue up atleast one request and if this
RPM Controller is not running then call run.
Caller can either simply call run but then they will be blocked or
can instantiate a new thread to process all queued up requests.
For example:
threading.Thread(target=rpm_controller.run).start()
Requests are in the format of:
[powerunit_info, new_state, condition_var, result]
Run will set the result with the correct value.
"""
while not self.request_queue.empty():
try:
result = multiprocessing.Value(ctypes.c_bool, False)
request = self.request_queue.get()
device_hostname = request['powerunit_info'].device_hostname
if (datetime.datetime.utcnow() >
(request['start_time'] +
datetime.timedelta(minutes=RPM_CALL_TIMEOUT_MINS))):
logging.error('The request was waited for too long to be '
"processed. It is timed out and won't be "
'processed.')
request['result_queue'].put(False)
continue
is_timeout = multiprocessing.Value(ctypes.c_bool, False)
process = multiprocessing.Process(target=self._process_request,
args=(request, result,
is_timeout))
process.start()
process.join(SET_POWER_STATE_TIMEOUT_SECONDS +
PROCESS_TIMEOUT_BUFFER)
if process.is_alive():
logging.debug(
'%s: process (%s) still running, will be '
'terminated!', device_hostname, process.pid)
process.terminate()
is_timeout.value = True
if is_timeout.value:
raise error.TimeoutException(
'Attempt to set power state is timed out after %s '
'seconds.' % SET_POWER_STATE_TIMEOUT_SECONDS)
if not result.value:
logging.error('Request to change %s to state %s failed.',
device_hostname, request['new_state'])
except Exception as e:
logging.error(
'Request to change %s to state %s failed: '
'Raised exception: %s', device_hostname,
request['new_state'], e)
result.value = False
# Put result inside the result Queue to allow the caller to resume.
request['result_queue'].put(result.value)
self._stop_processing_requests()
def urlopen_socket_timeout(url, data=None, timeout=5):
"""
Wrapper to urllib2.urlopen with a socket timeout.
This method will convert all socket timeouts to
TimeoutExceptions, so we can use it in conjunction
with the rpc retry decorator and continue to handle
other URLErrors as we see fit.
@param url: The url to open.
@param data: The data to send to the url (eg: the urlencoded dictionary
used with a POST call).
@param timeout: The timeout for this urlopen call.
@return: The response of the urlopen call.
@raises: error.TimeoutException when a socket timeout occurs.
urllib2.URLError for errors that not caused by timeout.
urllib2.HTTPError for errors like 404 url not found.
"""
old_timeout = socket.getdefaulttimeout()
socket.setdefaulttimeout(timeout)
try:
return urllib2.urlopen(url, data=data)
except urllib2.URLError as e:
if type(e.reason) is socket.timeout:
raise error.TimeoutException(str(e))
raise
finally:
socket.setdefaulttimeout(old_timeout)
def func_retry(*args, **kwargs):
"""
Used to cache exception to be raised later.
"""
exc_info = None
delayed_enabled = False
exception_tuple = () if blacklist is None else tuple(blacklist)
start_time = time.time()
remaining_time = timeout_min * 60
is_main_thread = isinstance(threading.current_thread(),
threading._MainThread)
if label:
details = 'label="%s"' % label
elif hasattr(func, '__name__'):
details = 'function="%s()"' % func.__name__
else:
details = 'unknown function'
exception_message = ('retry exception (%s), timeout = %ds' %
(details, timeout_min * 60))
while remaining_time > 0:
if delayed_enabled:
delay()
else:
delayed_enabled = True
try:
# Clear the cache
exc_info = None
if is_main_thread:
is_timeout, result = timeout(func, args, kwargs,
remaining_time)
if not is_timeout:
return result
else:
return func(*args, **kwargs)
except exception_tuple:
raise
except error.CrosDynamicSuiteException:
raise
except ExceptionToCheck as e:
logging.warning('%s(%s)', e.__class__, e)
# Cache the exception to be raised later.
exc_info = sys.exc_info()
remaining_time = int(timeout_min * 60 -
(time.time() - start_time))
# The call must have timed out or raised ExceptionToCheck.
if not exc_info:
if exception_to_raise:
raise exception_to_raise(exception_message)
else:
raise error.TimeoutException(exception_message)
# Raise the cached exception with original backtrace.
if exception_to_raise:
raise exception_to_raise('%s: %s' % (exc_info[0], exc_info[1]))
raise exc_info[0], exc_info[1], exc_info[2]
def sigalarm_wrapper(message):
"""
Raise a TimeoutException with the given message. Needed because the body
of a closure (lambda) can only be an expression, not a statement (such
as "raise") :P :P :P
@param message: the exception message.
"""
raise error.TimeoutException(message)
def func_retry(*args, **kwargs):
# Used to cache exception to be raised later.
exc_info = None
delayed_enabled = False
exception_tuple = () if blacklist is None else tuple(blacklist)
start_time = time.time()
remaining_time = timeout_min * 60
is_main_thread = isinstance(threading.current_thread(),
threading._MainThread)
while remaining_time > 0:
if delayed_enabled:
delay()
else:
delayed_enabled = True
try:
# Clear the cache
exc_info = None
if is_main_thread:
is_timeout, result = timeout(func, args, kwargs,
remaining_time)
if not is_timeout:
return result
else:
return func(*args, **kwargs)
except exception_tuple:
raise
except error.CrosDynamicSuiteException:
raise
except ExceptionToCheck as e:
logging.warning('%s(%s)', e.__class__, e)
# Cache the exception to be raised later.
exc_info = sys.exc_info()
remaining_time = int(timeout_min * 60 -
(time.time() - start_time))
# The call must have timed out or raised ExceptionToCheck.
if not exc_info:
raise error.TimeoutException('Call is timed out.')
# Raise the cached exception with original backtrace.
raise exc_info[0], exc_info[1], exc_info[2]
def handler(signum, frame):
"""
Register a handler for the timeout.
"""
raise error.TimeoutException('Call is timed out.')