def testAddRoleToPrimarySuccess(self):
"""Test manager can add a role to a primary server successfully.
Confirm that actions needs to be taken, e.g., restart scheduler for
new drone to be added.
"""
server_models.validate(role=server_models.ServerRole.ROLE.DRONE)
server_manager_utils.check_server(mox.IgnoreArg(),
mox.IgnoreArg()).AndReturn(True)
server_manager_utils.use_server_db().MultipleTimes().AndReturn(True)
self.mox.StubOutWithMock(self.PRIMARY_SCHEDULER, 'get_role_names')
self.PRIMARY_SCHEDULER.get_role_names().AndReturn(
[server_models.ServerRole.ROLE.SCHEDULER])
server_models.ServerRole.objects.create(
server=self.PRIMARY_SCHEDULER,
role=server_models.ServerRole.ROLE.DRONE).AndReturn(
self.DRONE_ROLE)
server_models.Server.objects.filter(
roles__role=server_models.ServerRole.ROLE.SCHEDULER,
status=server_models.Server.STATUS.PRIMARY).AndReturn(
[self.PRIMARY_SCHEDULER])
infra.execute_command(mox.IgnoreArg(), mox.IgnoreArg())
self.mox.ReplayAll()
server_manager._add_role(self.PRIMARY_SCHEDULER,
server_models.ServerRole.ROLE.DRONE,
action=True)
def testDeleteRoleFromPrimarySuccess(self):
"""Test manager can delete a role from a primary server successfully.
Confirm that database call is made, and actions are taken, e.g.,
restart scheduler to delete an existing drone.
"""
server_manager_utils.use_server_db().MultipleTimes().AndReturn(True)
server_models.validate(role=server_models.ServerRole.ROLE.DRONE)
self.mox.StubOutWithMock(self.PRIMARY_DRONE, 'get_role_names')
self.PRIMARY_DRONE.get_role_names().MultipleTimes().AndReturn(
[server_models.ServerRole.ROLE.DRONE])
self.mox.StubOutWithMock(self.PRIMARY_DRONE.roles, 'get')
self.PRIMARY_DRONE.roles.get(
role=server_models.ServerRole.ROLE.DRONE).AndReturn(
self.DRONE_ROLE)
server_models.Server.objects.filter(
roles__role=server_models.ServerRole.ROLE.SCHEDULER,
status=server_models.Server.STATUS.PRIMARY).AndReturn(
[self.PRIMARY_SCHEDULER])
server_manager.server_manager_utils.warn_missing_role(
server_models.ServerRole.ROLE.DRONE, self.PRIMARY_DRONE)
infra.execute_command(mox.IgnoreArg(), mox.IgnoreArg())
self.mox.ReplayAll()
server_manager._delete_role(self.PRIMARY_DRONE,
server_models.ServerRole.ROLE.DRONE,
action=True)
def apply(action):
"""Apply an given action.
It usually involves ssh to the server with specific role and run the
command, e.g., ssh to scheduler server and restart scheduler.
@param action: A tuple of (the role of which the command should be executed,
the command)
@raise ServerActionError: If the action can't be applied due to database
issue.
@param subprocess.CalledProcessError: If command is failed to be
executed.
"""
role = action[0]
command = action[1]
# Find the servers with role
servers = server_manager_utils.get_servers(
role=role, status=server_models.Server.STATUS.PRIMARY)
if not servers:
print >> sys.stderr, ('WARNING! Action %s failed to be applied. No '
'server with given role %s was found.' %
(action, role))
return
for server in servers:
print 'Run command `%s` on server %s' % (command, server.hostname)
try:
infra.execute_command(server.hostname, command)
except subprocess.CalledProcessError as e:
print >> sys.stderr, ('Failed to check server %s, error: %s' %
(server.hostname, e))
def testChangeStatusSuccess_PrimaryToRepairFailed(self):
"""Test manager can change the status of a primary server to
repair_required.
"""
server_models.validate(
status=server_models.Server.STATUS.REPAIR_REQUIRED)
self.mox.StubOutWithMock(self.PRIMARY_DRONE.roles, 'filter')
self.mox.StubOutWithMock(self.PRIMARY_DRONE, 'get_role_names')
self.PRIMARY_DRONE.get_role_names().MultipleTimes().AndReturn(
[server_models.ServerRole.ROLE.DRONE])
self.PRIMARY_DRONE.roles.filter(
role__in=server_models.ServerRole.ROLES_REQUIRE_UNIQUE_INSTANCE
).AndReturn(None)
server_manager_utils.use_server_db().MultipleTimes().AndReturn(True)
server_manager_utils.warn_missing_role(
server_models.ServerRole.ROLE.DRONE, self.PRIMARY_DRONE)
server_models.Server.objects.filter(
roles__role=server_models.ServerRole.ROLE.SCHEDULER,
status=server_models.Server.STATUS.PRIMARY).AndReturn(
[self.PRIMARY_SCHEDULER])
infra.execute_command(mox.IgnoreArg(), mox.IgnoreArg())
self.mox.ReplayAll()
server_manager._change_status(
server=self.PRIMARY_DRONE,
status=server_models.Server.STATUS.REPAIR_REQUIRED,
action=True)
def update_server(inputs):
"""Deploy for given server.
@param inputs: Inputs for the update action, including:
server: Name of the server to update.
status: Status of the server.
options: Options for the update.
@return: A tuple of (server, success, output), where:
server: Name of the server to be updated.
sucess: True if update succeeds, False otherwise.
output: A string of the deploy_production_local script output
including any errors.
"""
server = inputs['server']
status = inputs['status']
options = inputs['options']
print('Updating server %s...' % server)
if status == 'backup':
extra_args = ['--skip-service-status']
else:
extra_args = []
cmd = ('%s %s' %
(DEPLOY_PRODUCTION_LOCAL, ' '.join(options.args + extra_args)))
output = '%s: %s' % (server, cmd)
success = True
if not options.dryrun:
try:
output = infra.execute_command(server, cmd)
except subprocess.CalledProcessError as e:
success = False
output = e.output
return server, success, output
def bootstrap(user, password, source_host, dest_host):
"""Bootstrap the given user against dest_host.
Allow a user from source_host to access the db server running on
dest_host.
@param user: The user to bootstrap.
@param password: The password for the user.
@param source_host: The host from which the new user will access the db.
@param dest_host: The hostname of the remote db server.
@raises MySQLCommandError: If we can't ping the db server using the default
user/password specified in the shadow_config under default_db_*, or
we can't ping it with the new credentials after bootstrapping.
"""
# Confirm ssh/become access.
try:
infra.execute_command(dest_host, 'echo "hello"')
except subprocess.CalledProcessError as e:
logging.error("Cannot become/ssh into dest host. You need to bootstrap "
"it using fab -H <hostname> bootstrap from the "
"chromeos-admin repo.")
return
# Confirm the default user has at least database read privileges. Note if
# the default user has *only* read privileges everything else will still
# fail. This is a remote enough case given our current setup that we can
# avoid more complicated checking at this level.
MySQLCommandExecutor.ping(dest_host, use_ssh=True)
# Prepare and execute the grant statement for the new user.
creds = {
'new_user': user,
'new_pass': password,
'new_host': source_host,
}
# TODO(beeps): Restrict these permissions. For now we have a couple of
# databases which may/may-not exist on various roles that need refactoring.
grant_privileges = (
"GRANT ALL PRIVILEGES ON *.* to '%(new_user)s'@'%(new_host)s' "
"IDENTIFIED BY '%(new_pass)s'; FLUSH PRIVILEGES;")
MySQLCommandExecutor.execute(
dest_host, MySQLCommandExecutor.mysql_cmd(grant_privileges % creds))
# Confirm the new user can ping the remote database server from localhost.
MySQLCommandExecutor.ping(
dest_host, user=user, password=password, use_ssh=False)
def time_wait(server):
"""
Submits a stat for the number of TIME_WAIT sockets that are on the server.
@param server: The AFE server.
"""
out = infra.execute_command(server, 'ss -o state time-wait | wc -l')
stat = autotest_stats.Gauge(server, bare=True)
# ss prints out a header for the columns also, so we subtract one to report
# about only the data.
stat.send('time_wait_sockets', int(out.strip()) - 1)
def num_devserver_processes(server):
"""
Submits a stat for the number of devserver processes that are on the server.
@param server: The AFE server.
"""
out = infra.execute_command(server, 'ps -C devserver.py| wc -l')
stat = autotest_stats.Gauge(server, bare=True)
# ps prints out a header for the columns also, so we subtract one to report
# about only the data.
stat.send('num_devserver_processes', int(out.strip()) - 1)
def check_server(hostname, role):
"""Confirm server with given hostname is ready to be primary of given role.
If the server is a backup and failed to be verified for the role, remove
the role from its roles list. If it has no other role, set its status to
repair_required.
@param hostname: hostname of the server.
@param role: Role to be checked.
@return: True if server can be verified for the given role, otherwise
return False.
"""
# TODO(dshi): Add more logic to confirm server is ready for the role.
# For now, the function just checks if server is ssh-able.
try:
infra.execute_command(hostname, 'true')
return True
except subprocess.CalledProcessError as e:
print >> sys.stderr, ('Failed to check server %s, error: %s' %
(hostname, e))
return False
def get_gateway():
"""Return the address of the default gateway.
@raises: subprocess.CalledProcessError: If the address of the gateway
cannot be determined via netstat.
"""
cmd = 'netstat -rn | grep "^0.0.0.0 " | cut -d " " -f10 | head -1'
try:
return infra.execute_command('localhost', cmd).rstrip('\n')
except subprocess.CalledProcessError as e:
logging.error('Unable to get gateway: %s', e)
raise
def testChangeStatusSuccess_BackupToPrimary(self):
"""Test manager can change the status of a backup server to primary.
"""
server_models.validate(status=server_models.Server.STATUS.PRIMARY)
server_manager_utils.use_server_db().MultipleTimes().AndReturn(True)
self.mox.StubOutWithMock(self.BACKUP_DRONE, 'get_role_names')
self.BACKUP_DRONE.get_role_names().MultipleTimes().AndReturn(
[server_models.ServerRole.ROLE.DRONE])
self.mox.StubOutWithMock(self.BACKUP_DRONE.roles, 'filter')
self.BACKUP_DRONE.roles.filter(
role__in=server_models.ServerRole.ROLES_REQUIRE_UNIQUE_INSTANCE
).AndReturn(None)
server_models.Server.objects.filter(
roles__role=server_models.ServerRole.ROLE.SCHEDULER,
status=server_models.Server.STATUS.PRIMARY).AndReturn(
[self.PRIMARY_SCHEDULER])
infra.execute_command(mox.IgnoreArg(), mox.IgnoreArg())
self.mox.ReplayAll()
server_manager._change_status(
server=self.BACKUP_DRONE,
status=server_models.Server.STATUS.PRIMARY,
action=True)
def execute(dest_server, full_cmd):
"""Execute a mysql statement on a remote server by sshing into it.
@param dest_server: The hostname of the remote mysql server.
@param full_cmd: The full mysql command to execute.
@raises MySQLCommandError: If the full_cmd failed on dest_server.
"""
try:
return infra.execute_command(dest_server, full_cmd)
except subprocess.CalledProcessError as e:
raise MySQLCommandError('Failed to execute %s against %s' %
(full_cmd, dest_server))
def vagrant_cmd(cls, cmd, stream_output=False):
"""Execute a vagrant command in VAGRANT_DIR.
@param cmd: The command to execute.
@param stream_output: If True, stream the output of `cmd`.
Waits for `cmd` to finish and returns a string with the
output if false.
"""
with infra.chdir(VAGRANT_DIR):
try:
return infra.execute_command(
'localhost',
'vagrant %s' % cmd, stream_output=stream_output)
except subprocess.CalledProcessError as e:
raise VagrantCmdError(
'Command "vagrant %s" failed with %s' % (cmd, e))
def update_server(inputs):
"""Deploy for given server.
@param inputs: Inputs for the update action, including:
server: Name of the server to update.
status: Status of the server.
options: Options for the update.
@return: A tuple of (server, success, output), where:
server: Name of the server to be updated.
sucess: True if update succeeds, False otherwise.
output: A string of the deploy_server_local script output
including any errors.
"""
start = time.time()
server = inputs['server']
status = inputs['status']
# Shared list to record the finished server.
finished_servers = inputs['finished_servers']
options = inputs['options']
print('Updating server %s...' % server)
if status == 'backup':
extra_args = ['--skip-service-status']
else:
extra_args = []
cmd = ('%s %s' %
(DEPLOY_SERVER_LOCAL, ' '.join(options.args + extra_args)))
output = '%s: %s' % (server, cmd)
success = True
if not options.dryrun:
for i in range(5):
try:
print('[%s/5] Try to update server %s' % (i, server))
output = infra.execute_command(server, cmd)
finished_servers.append(server)
break
except subprocess.CalledProcessError as e:
print('%s: Command failed with error: %s' % (server, e))
success = False
output = e.output
print('Time used to update server %s: %s' % (server, time.time() - start))
return server, success, output
def _update_server(server, extra_args=[]):
"""Run deploy_server_local for given server.
@param server: hostname to update.
@param extra_args: args to be passed in to deploy_server_local.
@return: A tuple of (server, success, output), where:
server: Name of the server.
sucess: True if update succeeds, False otherwise.
output: A string of the deploy_server_local script output
including any errors.
"""
cmd = ('%s %s' %
(DEPLOY_SERVER_LOCAL, ' '.join(extra_args)))
success = False
try:
output = infra.execute_command(server, cmd)
success = True
except subprocess.CalledProcessError as e:
output = e.output
return server, success, output