def _process_request(self, request, result, is_timeout):
"""Process the request to change a device's outlet state.
The call of set_power_state is made in a new running process. If it
takes longer than SET_POWER_STATE_TIMEOUT_SECONDS, the request will be
timed out.
@param request: A request to change a device's outlet state.
@param result: A Value object passed to the new process for the caller
thread to retrieve the result.
@param is_timeout: A Value object passed to the new process for the
caller thread to retrieve the information about if
the set_power_state call timed out.
"""
try:
logging.getLogger().handlers = []
kwargs = {'use_log_server': True}
is_timeout_value, result_value = retry.timeout(
rpm_logging_config.set_up_logging,
args=(),
kwargs=kwargs,
timeout_sec=10)
if is_timeout_value:
raise Exception('Setup local log server handler timed out.')
except Exception as e:
# Fail over to log to a new file.
LOG_FILENAME_FORMAT = rpm_config.get('GENERAL',
'dispatcher_logname_format')
log_filename_format = LOG_FILENAME_FORMAT.replace(
'dispatcher', 'controller_%d' % os.getpid())
logging.getLogger().handlers = []
rpm_logging_config.set_up_logging(
log_filename_format=log_filename_format, use_log_server=False)
logging.info('Failed to set up logging through log server: %s', e)
kwargs = {
'powerunit_info': request['powerunit_info'],
'new_state': request['new_state']
}
try:
is_timeout_value, result_value = retry.timeout(
self.set_power_state,
args=(),
kwargs=kwargs,
timeout_sec=SET_POWER_STATE_TIMEOUT_SECONDS)
result.value = result_value
is_timeout.value = is_timeout_value
except Exception as e:
# This method runs in a subprocess. Must log the exception,
# otherwise exceptions raised in set_power_state just get lost.
# Need to convert e to a str type, because our logging server
# code doesn't handle the conversion very well.
logging.error(
'Request to change %s to state %s failed: '
'Raised exception: %s',
request['powerunit_info'].device_hostname,
request['new_state'], str(e))
raise e
def sigint_handler(signum, stack_frame):
#pylint: disable-msg=C0111
"""Handle SIGINT or SIGTERM to a local test_that run.
This handler sends a SIGINT to the running autoserv process,
if one is running, giving it up to 5 seconds to clean up and exit. After
the timeout elapses, autoserv is killed. In either case, after autoserv
exits then this process exits with status 1.
"""
# If multiple signals arrive before handler is unset, ignore duplicates
if not _sigint_handler_lock.acquire(False):
return
try:
# Ignore future signals by unsetting handler.
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
logging.warning('Received SIGINT or SIGTERM. Cleaning up and exiting.')
if _autoserv_proc:
logging.warning(
'Sending SIGINT to autoserv process. Waiting up '
'to %s seconds for cleanup.', _AUTOSERV_SIGINT_TIMEOUT_SECONDS)
_autoserv_proc.send_signal(signal.SIGINT)
timed_out, _ = retry.timeout(
_autoserv_proc.wait,
timeout_sec=_AUTOSERV_SIGINT_TIMEOUT_SECONDS)
if timed_out:
_autoserv_proc.kill()
logging.warning('Timed out waiting for autoserv to handle '
'SIGINT. Killed autoserv.')
finally:
_sigint_handler_lock.release() # this is not really necessary?
sys.exit(1)
def _set_power(args_tuple, timeout_mins=RPM_CALL_TIMEOUT_MINS):
"""Sends the power state change request to the RPM Infrastructure.
@param args_tuple: A args tuple for rpc call. See example below:
(hostname, powerunit_hostname, outlet, hydra_hostname, new_state)
"""
client = xmlrpclib.ServerProxy(RPM_FRONTEND_URI,
verbose=False,
allow_none=True)
timeout = None
result = None
endpoint = (client.set_power_via_poe
if len(args_tuple) == 2 else client.set_power_via_rpm)
try:
timeout, result = retry.timeout(endpoint,
args=args_tuple,
timeout_sec=timeout_mins * 60,
default_result=False)
except Exception as e:
logging.exception(e)
raise RemotePowerException('Client call exception: ' + str(e))
if timeout:
raise RemotePowerException('Call to RPM Infrastructure timed out.')
if not result:
error_msg = ('Failed to change outlet status for host: %s to '
'state: %s.' % (args_tuple[0], args_tuple[-1]))
logging.error(error_msg)
raise RemotePowerException(error_msg)
def find_and_generate_minidump_stacktraces(host_resultdir):
"""
Finds all minidump files and generates a stack trace for each.
Enumerates all files under the test results directory (recursively)
and generates a stack trace file for the minidumps. Minidump files are
identified as files with .dmp extension. The stack trace filename is
composed by appending the .txt extension to the minidump filename.
@param host_resultdir: Directory to walk looking for dmp files.
@returns The list of generated minidumps.
"""
minidumps = []
for dir, subdirs, files in os.walk(host_resultdir):
for file in files:
if not file.endswith('.dmp'):
continue
minidump = os.path.join(dir, file)
# First, try to symbolicate locally.
try:
generate_minidump_stacktrace(minidump)
logging.info('Generated stack trace for dump %s', minidump)
minidumps.append(minidump)
continue
except client_utils.error.CmdError as err:
logging.warning(
'Failed to generate stack trace locally for '
'dump %s (rc=%d):\n%r', minidump,
err.result_obj.exit_status, err)
# If that did not succeed, try to symbolicate using the dev server.
try:
logging.info('Generating stack trace for %s', minidump)
minidumps.append(minidump)
is_timeout, _ = retry.timeout(
symbolicate_minidump_with_devserver,
args=(minidump, host_resultdir),
timeout_sec=600)
if is_timeout:
logging.warn(
'Generating stack trace is timed out for dump '
'%s', minidump)
autotest_stats.Counter(SYMBOLICATE_TIMEDOUT).increment()
else:
logging.info('Generated stack trace for dump %s', minidump)
continue
except dev_server.DevServerException as e:
logging.warning(
'Failed to generate stack trace on devserver for '
'dump %s:\n%r', minidump, e)
return minidumps
def fork_waitfor(self, timeout=None):
if not timeout:
return self.wait()
else:
_, result = retry.timeout(self.wait, timeout_sec=timeout)
if result is None:
utils.nuke_pid(self.pid)
print "subcommand failed pid %d" % self.pid
print "%s" % (self.func, )
print "timeout after %ds" % timeout
print
result = self.wait()
return result
def connect_servo(self):
"""Establish a connection to the servod server on this host.
Initializes `self._servo` and then verifies that all network
connections are working. This will create an ssh tunnel if
it's required.
As a side effect of testing the connection, all signals on the
target servo are reset to default values, and the USB stick is
set to the neutral (off) position.
"""
servo_obj = servo.Servo(servo_host=self,
servo_serial=self.servo_serial)
timeout, _ = retry.timeout(
servo_obj.initialize_dut,
timeout_sec=self.INITIALIZE_SERVO_TIMEOUT_SECS)
if timeout:
raise hosts.AutoservVerifyError('Servo initialize timed out.')
self._servo = servo_obj
def generate_stacktrace_for_file(minidump, host_resultdir):
"""
Tries to generate a stack trace for the file located at |minidump|.
@param minidump: path to minidump file to generate the stacktrace for.
@param host_resultdir: server job's result directory.
"""
# First, try to symbolicate locally.
try:
logging.info('Trying to generate stack trace locally for %s', minidump)
generate_minidump_stacktrace(minidump)
logging.info('Generated stack trace for dump %s', minidump)
return
except client_utils.error.CmdError as err:
logging.info(
'Failed to generate stack trace locally for '
'dump %s (rc=%d):\n%r', minidump, err.result_obj.exit_status, err)
# If that did not succeed, try to symbolicate using the dev server.
try:
logging.info('Generating stack trace using devserver for %s', minidump)
crashserver_name = _resolve_crashserver()
args = (minidump, host_resultdir, crashserver_name)
is_timeout, _ = retry.timeout(_symbolicate_minidump_with_devserver,
args=args,
timeout_sec=600)
if is_timeout:
logging.info('Generating stack trace timed out for dump %s',
minidump)
metrics.Counter(
'chromeos/autotest/crashcollect/symbolicate_timed_out'
).increment(fields={'crash_server': crashserver_name})
else:
logging.info('Generated stack trace for dump %s', minidump)
return
except dev_server.DevServerException as e:
logging.info(
'Failed to generate stack trace on devserver for dump '
'%s:\n%r', minidump, e)
# Symbolicating failed.
logging.warning('Failed to generate stack trace for %s (see info logs)',
minidump)
def set_power_afe(hostname, new_state, timeout_mins=RPM_CALL_TIMEOUT_MINS):
"""Sends the power state change request to the RPM Infrastructure.
@param hostname: host who's power outlet we want to change.
@param new_state: State we want to set the power outlet to.
"""
client = xmlrpclib.ServerProxy(RPM_FRONTEND_URI, verbose=False)
timeout = None
result = None
try:
timeout, result = retry.timeout(client.queue_request,
args=(hostname, new_state),
timeout_sec=timeout_mins * 60,
default_result=False)
except Exception as e:
logging.exception(e)
raise RemotePowerException('Client call exception: ' + str(e))
if timeout:
raise RemotePowerException('Call to RPM Infrastructure timed out.')
if not result:
error_msg = ('Failed to change outlet status for host: %s to '
'state: %s.' % (hostname, new_state))
logging.error(error_msg)
raise RemotePowerException(error_msg)