def __init__(self, hostname, powerswitch=None):
username = rpm_config.get('WEBPOWERED', 'username')
password = rpm_config.get('WEBPOWERED', 'password')
# Call the constructor in RPMController. However since this is a web
# accessible device, there should not be a need to tunnel through a
# hydra serial concentrator.
super(WebPoweredRPMController, self).__init__(hostname)
self.hostname = '%s.%s' % (self.hostname, self._dns_zone)
if not powerswitch:
self._rpm = dli_urllib.Powerswitch(hostname=self.hostname,
userid=username,
password=password)
else:
# Should only be used in unit_testing
self._rpm = powerswitch
def __init__(self, hostname):
"""
Initialize controller class for a Cisco POE switch.
@param hostname: the Cisco POE switch host name.
"""
super(CiscoPOEController, self).__init__(hostname)
self._username = rpm_config.get('CiscoPOE', 'username')
self._password = rpm_config.get('CiscoPOE', 'password')
# For a switch, e.g. 'chromeos2-poe-switch8',
# the device prompt looks like 'chromeos2-poe-sw8#'.
short_hostname = self.hostname.replace('switch', 'sw')
self.poe_prompt = self.POE_PROMPT % short_hostname
self.config_prompt = self.CONFIG_PROMPT % short_hostname
self.config_if_prompt = self.CONFIG_IF_PROMPT % short_hostname
def __init__(self, address, port):
"""
RPMDispatcher constructor.
Initialized instance vars and registers this server with the frontend
server.
@param address: Address of this dispatcher server.
@param port: Port assigned to this dispatcher server.
@raise RPMInfrastructureException: Raised if the dispatch server is
unable to register with the frontend
server.
"""
self._address = address
self._port = port
self._lock = threading.Lock()
self._worker_dict = {}
self._frontend_server = rpm_config.get('RPM_INFRASTRUCTURE',
'frontend_uri')
logging.info(
'Registering this rpm dispatcher with the frontend '
'server at %s.', self._frontend_server)
client = xmlrpclib.ServerProxy(self._frontend_server)
# De-register with the frontend when the dispatcher exit's.
atexit.register(self._unregister)
try:
client.register_dispatcher(self._get_serveruri())
except socket.error as er:
err_msg = ('Unable to register with frontend server. Error: %s.' %
errno.errorcode[er.errno])
logging.error(err_msg)
raise RPMInfrastructureException(err_msg)
def _kill_previous_connection(self):
"""
In case the port to the RPM through the hydra serial concentrator is in
use, terminate the previous connection so we can log into the RPM.
It logs into the hydra serial concentrator over ssh, launches the CLI
command, gets the port number and then kills the current session.
"""
ssh = self._authenticate_with_hydra(admin_override=True)
if not ssh:
return
ssh.expect(RPMController.PASSWORD_PROMPT, timeout=60)
ssh.sendline(rpm_config.get('HYDRA', 'admin_password'))
ssh.expect(RPMController.HYDRA_PROMPT)
ssh.sendline(RPMController.CLI_CMD)
cli_prompt_re = re.compile(RPMController.CLI_PROMPT)
cli_held_re = re.compile(RPMController.CLI_HELD)
response = ssh.expect_list([cli_prompt_re, cli_held_re], timeout=60)
if response == 1:
# Need to kill the previous adminstator's session.
logging.error("Need to disconnect previous administrator's CLI "
"session to release the connection to RPM device %s.",
self.hostname)
ssh.sendline(RPMController.CLI_KILL_PREVIOUS)
ssh.expect(RPMController.CLI_PROMPT)
ssh.sendline(RPMController.PORT_STATUS_CMD)
ssh.expect(': %s' % self.hostname)
ports_status = ssh.before
port_number = ports_status.split(' ')[-1]
ssh.expect(RPMController.CLI_PROMPT)
ssh.sendline(RPMController.SESSION_KILL_CMD_FORMAT % port_number)
ssh.expect(RPMController.CLI_PROMPT)
self._logout(ssh, admin_logout=True)
def _hydra_login(self, ssh):
"""
Perform the extra steps required to log into a hydra serial
concentrator.
@param ssh: pexpect.spawn object used to communicate with the hydra
serial concentrator.
@return: True if the login procedure is successful. False if an error
occurred. The most common case would be if another user is
logged into the device.
"""
try:
response = ssh.expect_list([
re.compile(RPMController.PASSWORD_PROMPT),
re.compile(RPMController.HYDRA_CONN_HELD_MSG_FORMAT)
],
timeout=15)
except pexpect.TIMEOUT:
# If there was a timeout, this ssh tunnel could be set up to
# not require the hydra password.
ssh.sendline('')
try:
ssh.expect(re.compile(RPMController.USERNAME_PROMPT))
logging.debug('Connected to rpm through hydra. Logging in.')
return True
except pexpect.ExceptionPexpect:
return False
if response == 0:
try:
ssh.sendline(rpm_config.get('HYDRA', 'password'))
ssh.sendline('')
response = ssh.expect_list([
re.compile(RPMController.USERNAME_PROMPT),
re.compile(RPMController.HYDRA_CONN_HELD_MSG_FORMAT)
],
timeout=60)
except pexpect.EOF:
# Did not receive any of the expect responses, retry.
return False
except pexpect.TIMEOUT:
logging.debug('Timeout occurred logging in to hydra.')
return False
# Send the username that the subclass will have set in its
# construction.
if response == 1:
logging.debug('SSH Terminal most likely serving another'
' connection, retrying.')
# Kill the connection for the next connection attempt.
try:
self._kill_previous_connection()
except pexpect.ExceptionPexpect:
logging.error('Failed to disconnect previous connection, '
'retrying.')
raise
return False
logging.debug('Connected to rpm through hydra. Logging in.')
return True
def _process_request(self, request, result, is_timeout):
"""Process the request to change a device's outlet state.
The call of set_power_state is made in a new running process. If it
takes longer than SET_POWER_STATE_TIMEOUT_SECONDS, the request will be
timed out.
@param request: A request to change a device's outlet state.
@param result: A Value object passed to the new process for the caller
thread to retrieve the result.
@param is_timeout: A Value object passed to the new process for the
caller thread to retrieve the information about if
the set_power_state call timed out.
"""
try:
logging.getLogger().handlers = []
kwargs = {'use_log_server': True}
is_timeout_value, result_value = retry.timeout(
rpm_logging_config.set_up_logging,
args=(),
kwargs=kwargs,
timeout_sec=10)
if is_timeout_value:
raise Exception('Setup local log server handler timed out.')
except Exception as e:
# Fail over to log to a new file.
LOG_FILENAME_FORMAT = rpm_config.get('GENERAL',
'dispatcher_logname_format')
log_filename_format = LOG_FILENAME_FORMAT.replace(
'dispatcher', 'controller_%d' % os.getpid())
logging.getLogger().handlers = []
rpm_logging_config.set_up_logging(
log_filename_format=log_filename_format, use_log_server=False)
logging.info('Failed to set up logging through log server: %s', e)
kwargs = {
'powerunit_info': request['powerunit_info'],
'new_state': request['new_state']
}
try:
is_timeout_value, result_value = retry.timeout(
self.set_power_state,
args=(),
kwargs=kwargs,
timeout_sec=SET_POWER_STATE_TIMEOUT_SECONDS)
result.value = result_value
is_timeout.value = is_timeout_value
except Exception as e:
# This method runs in a subprocess. Must log the exception,
# otherwise exceptions raised in set_power_state just get lost.
# Need to convert e to a str type, because our logging server
# code doesn't handle the conversion very well.
logging.error(
'Request to change %s to state %s failed: '
'Raised exception: %s',
request['powerunit_info'].device_hostname,
request['new_state'], str(e))
raise e
def _authenticate_with_hydra(self, admin_override=False):
"""
Some RPM's are behind a hydra serial concentrator and require their ssh
connection to be tunneled through this device. This can fail if another
user is logged in; therefore this will retry multiple times.
This function also allows us to authenticate directly to the
administrator interface of the hydra device.
@param admin_override: Set to True if we are trying to access the
administrator interface rather than tunnel
through to the RPM.
@return: The connected pexpect.spawn instance if the login procedure is
successful. None if an error occurred. The most common case
would be if another user is logged into the device.
"""
if admin_override:
username = rpm_config.get('HYDRA', 'admin_username')
else:
username = '******' % (rpm_config.get('HYDRA',
'username'), self.hostname)
cmd = RPMController.SSH_LOGIN_CMD % (username, self.hydra_hostname)
num_attempts = 0
while num_attempts < RPMController.HYDRA_MAX_CONNECT_RETRIES:
try:
ssh = pexpect.spawn(cmd)
except pexpect.ExceptionPexpect:
return None
if admin_override:
return ssh
if self._hydra_login(ssh):
return ssh
# Authenticating with hydra failed. Sleep then retry.
time.sleep(RPMController.HYRDA_RETRY_SLEEP_SECS)
num_attempts += 1
logging.error(
'Failed to connect to the hydra serial concentrator after'
' %d attempts.', RPMController.HYDRA_MAX_CONNECT_RETRIES)
return None
def _setup_test_user(self, ssh):
"""Configure the test user for the RPM
@param ssh: Pexpect object to use to configure the RPM.
"""
# Create and configure the testing user profile.
testing_user = rpm_config.get('SENTRY','testing_user')
testing_password = rpm_config.get('SENTRY','testing_password')
ssh.sendline('create user %s' % testing_user)
response = ssh.expect_list([re.compile('not unique'),
re.compile(self.PASSWORD_PROMPT)])
if not response:
return
# Testing user is not set up yet.
ssh.sendline(testing_password)
ssh.expect('Verify Password:'******'add outlettouser all %s' % testing_user)
ssh.expect(self.SUCCESS_MSG)
ssh.expect(self.DEVICE_PROMPT)
def __init__(self, rpm_hostname, hydra_hostname=None):
"""
RPMController Constructor.
To be called by subclasses.
@param rpm_hostname: hostname of rpm device to be controlled.
"""
self._dns_zone = rpm_config.get('CROS', 'dns_zone')
self.hostname = rpm_hostname
self.request_queue = Queue.Queue()
self._running = False
self.is_running_lock = threading.Lock()
# If a hydra name is provided by the subclass then we know we are
# talking to an rpm behind a hydra device.
self.hydra_hostname = hydra_hostname if hydra_hostname else None
self.behind_hydra = hydra_hostname is not None
def __init__(self, hostname, hydra_hostname=None):
super(SentryRPMController, self).__init__(hostname, hydra_hostname)
self._username = rpm_config.get('SENTRY', 'username')
self._password = rpm_config.get('SENTRY', 'password')
import sys
import socket
import threading
import xmlrpclib
import rpm_controller
import rpm_logging_config
from config import rpm_config
from MultiThreadedXMLRPCServer import MultiThreadedXMLRPCServer
from rpm_infrastructure_exception import RPMInfrastructureException
import common
from autotest_lib.site_utils.rpm_control_system import utils
LOG_FILENAME_FORMAT = rpm_config.get('GENERAL', 'dispatcher_logname_format')
class RPMDispatcher(object):
"""
This class is the RPM dispatcher server and it is responsible for
communicating directly to the RPM devices to change a DUT's outlet status.
When an RPMDispatcher is initialized it registers itself with the frontend
server, who will field out outlet requests to this dispatcher.
Once a request is received the dispatcher looks up the RPMController
instance for the given DUT and then queues up the request and blocks until
it is processed.
@var _address: IP address or Hostname of this dispatcher server.
from config import rpm_config
from MultiThreadedXMLRPCServer import MultiThreadedXMLRPCServer
from rpm_infrastructure_exception import RPMInfrastructureException
import common
from autotest_lib.server import frontend
from autotest_lib.site_utils.rpm_control_system import utils
DEFAULT_RPM_COUNT = 0
TERMINATED = -1
# Indexes for accessing heap entries.
RPM_COUNT = 0
DISPATCHER_URI = 1
LOG_FILENAME_FORMAT = rpm_config.get('GENERAL', 'frontend_logname_format')
DEFAULT_RPM_ID = rpm_config.get('RPM_INFRASTRUCTURE', 'default_rpm_id')
# Valid state values.
VALID_STATE_VALUES = ['ON', 'OFF', 'CYCLE']
# Servo-interface mapping file
MAPPING_FILE = os.path.join(
os.path.dirname(__file__),
rpm_config.get('CiscoPOE', 'servo_interface_mapping_file'))
# Size of the LRU that holds power management unit information related
# to a device, e.g. rpm_hostname, outlet, hydra_hostname, etc.
LRU_SIZE = rpm_config.getint('RPM_INFRASTRUCTURE', 'lru_size')
import collections
import csv
import logging
import os
import time
import common
import rpm_infrastructure_exception
from config import rpm_config
from autotest_lib.client.common_lib import enum
MAPPING_FILE = os.path.join(
os.path.dirname(__file__),
rpm_config.get('CiscoPOE', 'servo_interface_mapping_file'))
POWERUNIT_HOSTNAME_KEY = 'powerunit_hostname'
POWERUNIT_OUTLET_KEY = 'powerunit_outlet'
HYDRA_HOSTNAME_KEY = 'hydra_hostname'
DEFAULT_EXPIRATION_SECS = 60 * 30
class PowerUnitInfo(object):
"""A class that wraps rpm/poe information of a device."""
POWERUNIT_TYPES = enum.Enum('POE', 'RPM', string_value=True)
def __init__(self,
device_hostname,
powerunit_type,
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import logging, sys
from config import rpm_config
import rpm_controller
LOGGING_FORMAT = rpm_config.get('GENERAL','logging_format')
oyster_rpm_name_format = 'chromeos1-rack%d-rpm1'
atlantis_rpm_name_format = 'chromeos2-row%d-rack%d-rpm1'
DEFAULT_OYSTERBAY_OUTLET_MAP = {
1 : 'host1',
2 : 'host2',
4 : 'host3',
5 : 'host4',
7 : 'host5',
8 : 'host6',
9 : 'host7',
10 : 'host8',
12 : 'host9',
13 : 'host10',
15 : 'host11',
16 : 'host12'
}
DEFAULT_ATLANTIS_OUTLET_MAP = {
1 : 'host1',
2 : 'host7',
4 : 'host2',
5 : 'host8',
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import mox
import unittest
from config import rpm_config
import rpm_dispatcher
DUT_SAME_RPM1 = 'chromeos-rack8e-hostbs1'
DUT_SAME_RPM2 = 'chromeos-rack8e-hostbs2'
RPM_HOSTNAME = 'chromeos-rack8e-rpm1'
DUT_DIFFERENT_RPM = 'chromeos-rack1-hostbs1'
FAKE_DISPATCHER_URI = 'fake-dispatcher'
FAKE_DISPATCHER_PORT = 9999
FRONT_END_URI = rpm_config.get('RPM_INFRASTRUCTURE', 'frontend_uri')
PROPER_URI_FORMAT = 'http://%s:%d'
class TestRPMDispatcher(mox.MoxTestBase):
"""
Simple unit tests to verify that the RPM Dispatcher properly registers with
the frontend server, and also initializes and reuses the same RPM Controller
for DUT requests on the same RPM.
queue_request is the only public method of RPM Dispatcher, however its logic
is simple and relies mostly on the private methods; therefore, I am testing
primarily RPMDispatcher initialization and _get_rpm_controller (which calls
_create_rpm_controller) to verify correct implementation.
"""
def setUp(self):
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import datetime
import logging
import logging.handlers
import os
import socket
import time
from config import rpm_config
import common
from autotest_lib.site_utils import log_socket_server
from autotest_lib.site_utils.rpm_control_system import rpm_infrastructure_exception
LOGGING_FORMAT = rpm_config.get('GENERAL', 'logging_format')
RECEIVERS = rpm_config.get('RPM_INFRASTRUCTURE',
'email_notification_recipients').split(',')
SUBJECT_LINE = (rpm_config.get('GENERAL', 'email_subject_line_format') %
socket.gethostname())
class SuspendableSMTPHandler(logging.handlers.SMTPHandler):
"""SMTPHandler that can have it's emails suspended."""
_suspend_start_time = datetime.datetime.now()
_suspend_time_hrs = 0
def suspend_emails(self, hours):
"""Suspend email notifications.
