def testAddRoleToPrimarySuccess(self): """Test manager can add a role to a primary server successfully. Confirm that actions needs to be taken, e.g., restart scheduler for new drone to be added. """ server_models.validate(role=server_models.ServerRole.ROLE.DRONE) server_manager_utils.check_server(mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(True) server_manager_utils.use_server_db().MultipleTimes().AndReturn(True) self.mox.StubOutWithMock(self.PRIMARY_SCHEDULER, 'get_role_names') self.PRIMARY_SCHEDULER.get_role_names().AndReturn( [server_models.ServerRole.ROLE.SCHEDULER]) server_models.ServerRole.objects.create( server=self.PRIMARY_SCHEDULER, role=server_models.ServerRole.ROLE.DRONE).AndReturn( self.DRONE_ROLE) server_models.Server.objects.filter( roles__role=server_models.ServerRole.ROLE.SCHEDULER, status=server_models.Server.STATUS.PRIMARY).AndReturn( [self.PRIMARY_SCHEDULER]) infra.execute_command(mox.IgnoreArg(), mox.IgnoreArg()) self.mox.ReplayAll() server_manager._add_role(self.PRIMARY_SCHEDULER, server_models.ServerRole.ROLE.DRONE, action=True)
def testDeleteRoleFromPrimarySuccess(self): """Test manager can delete a role from a primary server successfully. Confirm that database call is made, and actions are taken, e.g., restart scheduler to delete an existing drone. """ server_manager_utils.use_server_db().MultipleTimes().AndReturn(True) server_models.validate(role=server_models.ServerRole.ROLE.DRONE) self.mox.StubOutWithMock(self.PRIMARY_DRONE, 'get_role_names') self.PRIMARY_DRONE.get_role_names().MultipleTimes().AndReturn( [server_models.ServerRole.ROLE.DRONE]) self.mox.StubOutWithMock(self.PRIMARY_DRONE.roles, 'get') self.PRIMARY_DRONE.roles.get( role=server_models.ServerRole.ROLE.DRONE).AndReturn( self.DRONE_ROLE) server_models.Server.objects.filter( roles__role=server_models.ServerRole.ROLE.SCHEDULER, status=server_models.Server.STATUS.PRIMARY).AndReturn( [self.PRIMARY_SCHEDULER]) server_manager.server_manager_utils.warn_missing_role( server_models.ServerRole.ROLE.DRONE, self.PRIMARY_DRONE) infra.execute_command(mox.IgnoreArg(), mox.IgnoreArg()) self.mox.ReplayAll() server_manager._delete_role(self.PRIMARY_DRONE, server_models.ServerRole.ROLE.DRONE, action=True)
def apply(action): """Apply an given action. It usually involves ssh to the server with specific role and run the command, e.g., ssh to scheduler server and restart scheduler. @param action: A tuple of (the role of which the command should be executed, the command) @raise ServerActionError: If the action can't be applied due to database issue. @param subprocess.CalledProcessError: If command is failed to be executed. """ role = action[0] command = action[1] # Find the servers with role servers = server_manager_utils.get_servers( role=role, status=server_models.Server.STATUS.PRIMARY) if not servers: print >> sys.stderr, ('WARNING! Action %s failed to be applied. No ' 'server with given role %s was found.' % (action, role)) return for server in servers: print 'Run command `%s` on server %s' % (command, server.hostname) try: infra.execute_command(server.hostname, command) except subprocess.CalledProcessError as e: print >> sys.stderr, ('Failed to check server %s, error: %s' % (server.hostname, e))
def testChangeStatusSuccess_PrimaryToRepairFailed(self): """Test manager can change the status of a primary server to repair_required. """ server_models.validate( status=server_models.Server.STATUS.REPAIR_REQUIRED) self.mox.StubOutWithMock(self.PRIMARY_DRONE.roles, 'filter') self.mox.StubOutWithMock(self.PRIMARY_DRONE, 'get_role_names') self.PRIMARY_DRONE.get_role_names().MultipleTimes().AndReturn( [server_models.ServerRole.ROLE.DRONE]) self.PRIMARY_DRONE.roles.filter( role__in=server_models.ServerRole.ROLES_REQUIRE_UNIQUE_INSTANCE ).AndReturn(None) server_manager_utils.use_server_db().MultipleTimes().AndReturn(True) server_manager_utils.warn_missing_role( server_models.ServerRole.ROLE.DRONE, self.PRIMARY_DRONE) server_models.Server.objects.filter( roles__role=server_models.ServerRole.ROLE.SCHEDULER, status=server_models.Server.STATUS.PRIMARY).AndReturn( [self.PRIMARY_SCHEDULER]) infra.execute_command(mox.IgnoreArg(), mox.IgnoreArg()) self.mox.ReplayAll() server_manager._change_status( server=self.PRIMARY_DRONE, status=server_models.Server.STATUS.REPAIR_REQUIRED, action=True)
def update_server(inputs): """Deploy for given server. @param inputs: Inputs for the update action, including: server: Name of the server to update. status: Status of the server. options: Options for the update. @return: A tuple of (server, success, output), where: server: Name of the server to be updated. sucess: True if update succeeds, False otherwise. output: A string of the deploy_production_local script output including any errors. """ server = inputs['server'] status = inputs['status'] options = inputs['options'] print('Updating server %s...' % server) if status == 'backup': extra_args = ['--skip-service-status'] else: extra_args = [] cmd = ('%s %s' % (DEPLOY_PRODUCTION_LOCAL, ' '.join(options.args + extra_args))) output = '%s: %s' % (server, cmd) success = True if not options.dryrun: try: output = infra.execute_command(server, cmd) except subprocess.CalledProcessError as e: success = False output = e.output return server, success, output
def bootstrap(user, password, source_host, dest_host): """Bootstrap the given user against dest_host. Allow a user from source_host to access the db server running on dest_host. @param user: The user to bootstrap. @param password: The password for the user. @param source_host: The host from which the new user will access the db. @param dest_host: The hostname of the remote db server. @raises MySQLCommandError: If we can't ping the db server using the default user/password specified in the shadow_config under default_db_*, or we can't ping it with the new credentials after bootstrapping. """ # Confirm ssh/become access. try: infra.execute_command(dest_host, 'echo "hello"') except subprocess.CalledProcessError as e: logging.error("Cannot become/ssh into dest host. You need to bootstrap " "it using fab -H <hostname> bootstrap from the " "chromeos-admin repo.") return # Confirm the default user has at least database read privileges. Note if # the default user has *only* read privileges everything else will still # fail. This is a remote enough case given our current setup that we can # avoid more complicated checking at this level. MySQLCommandExecutor.ping(dest_host, use_ssh=True) # Prepare and execute the grant statement for the new user. creds = { 'new_user': user, 'new_pass': password, 'new_host': source_host, } # TODO(beeps): Restrict these permissions. For now we have a couple of # databases which may/may-not exist on various roles that need refactoring. grant_privileges = ( "GRANT ALL PRIVILEGES ON *.* to '%(new_user)s'@'%(new_host)s' " "IDENTIFIED BY '%(new_pass)s'; FLUSH PRIVILEGES;") MySQLCommandExecutor.execute( dest_host, MySQLCommandExecutor.mysql_cmd(grant_privileges % creds)) # Confirm the new user can ping the remote database server from localhost. MySQLCommandExecutor.ping( dest_host, user=user, password=password, use_ssh=False)
def time_wait(server): """ Submits a stat for the number of TIME_WAIT sockets that are on the server. @param server: The AFE server. """ out = infra.execute_command(server, 'ss -o state time-wait | wc -l') stat = autotest_stats.Gauge(server, bare=True) # ss prints out a header for the columns also, so we subtract one to report # about only the data. stat.send('time_wait_sockets', int(out.strip()) - 1)
def num_devserver_processes(server): """ Submits a stat for the number of devserver processes that are on the server. @param server: The AFE server. """ out = infra.execute_command(server, 'ps -C devserver.py| wc -l') stat = autotest_stats.Gauge(server, bare=True) # ps prints out a header for the columns also, so we subtract one to report # about only the data. stat.send('num_devserver_processes', int(out.strip()) - 1)
def check_server(hostname, role): """Confirm server with given hostname is ready to be primary of given role. If the server is a backup and failed to be verified for the role, remove the role from its roles list. If it has no other role, set its status to repair_required. @param hostname: hostname of the server. @param role: Role to be checked. @return: True if server can be verified for the given role, otherwise return False. """ # TODO(dshi): Add more logic to confirm server is ready for the role. # For now, the function just checks if server is ssh-able. try: infra.execute_command(hostname, 'true') return True except subprocess.CalledProcessError as e: print >> sys.stderr, ('Failed to check server %s, error: %s' % (hostname, e)) return False
def get_gateway(): """Return the address of the default gateway. @raises: subprocess.CalledProcessError: If the address of the gateway cannot be determined via netstat. """ cmd = 'netstat -rn | grep "^0.0.0.0 " | cut -d " " -f10 | head -1' try: return infra.execute_command('localhost', cmd).rstrip('\n') except subprocess.CalledProcessError as e: logging.error('Unable to get gateway: %s', e) raise
def testChangeStatusSuccess_BackupToPrimary(self): """Test manager can change the status of a backup server to primary. """ server_models.validate(status=server_models.Server.STATUS.PRIMARY) server_manager_utils.use_server_db().MultipleTimes().AndReturn(True) self.mox.StubOutWithMock(self.BACKUP_DRONE, 'get_role_names') self.BACKUP_DRONE.get_role_names().MultipleTimes().AndReturn( [server_models.ServerRole.ROLE.DRONE]) self.mox.StubOutWithMock(self.BACKUP_DRONE.roles, 'filter') self.BACKUP_DRONE.roles.filter( role__in=server_models.ServerRole.ROLES_REQUIRE_UNIQUE_INSTANCE ).AndReturn(None) server_models.Server.objects.filter( roles__role=server_models.ServerRole.ROLE.SCHEDULER, status=server_models.Server.STATUS.PRIMARY).AndReturn( [self.PRIMARY_SCHEDULER]) infra.execute_command(mox.IgnoreArg(), mox.IgnoreArg()) self.mox.ReplayAll() server_manager._change_status( server=self.BACKUP_DRONE, status=server_models.Server.STATUS.PRIMARY, action=True)
def execute(dest_server, full_cmd): """Execute a mysql statement on a remote server by sshing into it. @param dest_server: The hostname of the remote mysql server. @param full_cmd: The full mysql command to execute. @raises MySQLCommandError: If the full_cmd failed on dest_server. """ try: return infra.execute_command(dest_server, full_cmd) except subprocess.CalledProcessError as e: raise MySQLCommandError('Failed to execute %s against %s' % (full_cmd, dest_server))
def vagrant_cmd(cls, cmd, stream_output=False): """Execute a vagrant command in VAGRANT_DIR. @param cmd: The command to execute. @param stream_output: If True, stream the output of `cmd`. Waits for `cmd` to finish and returns a string with the output if false. """ with infra.chdir(VAGRANT_DIR): try: return infra.execute_command( 'localhost', 'vagrant %s' % cmd, stream_output=stream_output) except subprocess.CalledProcessError as e: raise VagrantCmdError( 'Command "vagrant %s" failed with %s' % (cmd, e))
def update_server(inputs): """Deploy for given server. @param inputs: Inputs for the update action, including: server: Name of the server to update. status: Status of the server. options: Options for the update. @return: A tuple of (server, success, output), where: server: Name of the server to be updated. sucess: True if update succeeds, False otherwise. output: A string of the deploy_server_local script output including any errors. """ start = time.time() server = inputs['server'] status = inputs['status'] # Shared list to record the finished server. finished_servers = inputs['finished_servers'] options = inputs['options'] print('Updating server %s...' % server) if status == 'backup': extra_args = ['--skip-service-status'] else: extra_args = [] cmd = ('%s %s' % (DEPLOY_SERVER_LOCAL, ' '.join(options.args + extra_args))) output = '%s: %s' % (server, cmd) success = True if not options.dryrun: for i in range(5): try: print('[%s/5] Try to update server %s' % (i, server)) output = infra.execute_command(server, cmd) finished_servers.append(server) break except subprocess.CalledProcessError as e: print('%s: Command failed with error: %s' % (server, e)) success = False output = e.output print('Time used to update server %s: %s' % (server, time.time() - start)) return server, success, output
def _update_server(server, extra_args=[]): """Run deploy_server_local for given server. @param server: hostname to update. @param extra_args: args to be passed in to deploy_server_local. @return: A tuple of (server, success, output), where: server: Name of the server. sucess: True if update succeeds, False otherwise. output: A string of the deploy_server_local script output including any errors. """ cmd = ('%s %s' % (DEPLOY_SERVER_LOCAL, ' '.join(extra_args))) success = False try: output = infra.execute_command(server, cmd) success = True except subprocess.CalledProcessError as e: output = e.output return server, success, output