def add(self, machine, apc_outlet): """ This adds a machine to the configuration """ # We can add a machine only when we are in active state if install_utilities.install_state( self.cfg.getGlobalParam('VERSION')) != "ACTIVE": logging.error("Can add a machine only when we are in active state") return 1 # First test for accessibility of the machine. if E.execute([machine], 'echo 1', None, 1) != E.ERR_OK: logging.error("Could not ssh into the machine %s" % machine) return 1 # start the svs on the remote machine restart_svs_cmd = "%s/local/google3/enterprise/legacy/util/svs_utilities.py %s %s" % ( self.cfg.getGlobalParam('ENTERPRISE_HOME'), self.cfg.getGlobalParam('ENTERPRISE_HOME'), machine) if E.execute([E.getCrtHostName()], SECURE_WRAPPER_COMMAND % ( \ self.cfg.getGlobalParam('ENTERPRISE_HOME'), "-p2", restart_svs_cmd), None, 0) != E.ERR_OK: logging.error("Could not start svs on machine %s" % machine) return 1 # wait for some time for svs to come up time.sleep(5) # check to see if the svs is up and is the right version if not svs_utilities.PingAndCheckSvsVersion( self.cfg.getGlobalParam('ENTERPRISE_BASHRC'), self.cfg.getGlobalParam('ENTERPRISE_HOME'), machine): logging.error("Svs not running correctly on machine %s" % machine) return 1 ver = self.cfg.getGlobalParam('VERSION') home = self.cfg.getGlobalParam('ENTERPRISE_HOME') testver = install_utilities.is_test(ver) # update MACHINES machines = self.cfg.getGlobalParam('MACHINES') if machine not in machines: machines.append(machine) self.cfg.setGlobalParam('MACHINES', machines) ret = core_utils.RemDeadNode(ver, testver, machine) if ret: logging.error('Cannot remove dead node from lockserver.') # we ignore this error for now # We just added a new machine into the config # this will lead to a change in concentrator config # so we need to re-run serve service which will # write the new config and restart the concentrator serve_cmd = ". %s && cd %s/local/google3/enterprise/legacy/scripts && " \ "./serve_service.py %s" % ( self.cfg.getGlobalParam('ENTERPRISE_BASHRC'), self.cfg.getGlobalParam('ENTERPRISE_HOME'), self.cfg.getGlobalParam('ENTERPRISE_HOME')) E.exe("%s %s" % (serve_cmd, "babysit")) num_tries = 5 cur_try = 0 while cur_try < num_tries: cur_try = cur_try + 1 all_disks = self.cfg.mach_param_cache.GetFact( "mounted-drives", machine) bad_disks = self.cfg.mach_param_cache.GetFact( "var_log_badhds", machine) if bad_disks and all_disks: break time.sleep(60) if all_disks == None or bad_disks == None: logging.error("Could not get machine information about %s" % machine) return 1 bad_disks = string.split(bad_disks, ' ') all_disks = string.split(all_disks, ' ') good_disks = filter(lambda x, y=bad_disks: x not in y, all_disks) good_disks = map(lambda x: "%s3" % x, good_disks) # change sda3 to hda3 etc. good_disks = map(lambda x: re.sub(r'^s', 'h', x), good_disks) # Preprocess disks before adding to remove duplicates. unique_good_disks = [] [ unique_good_disks.append(disk) for disk in good_disks if disk not in unique_good_disks ] # Add disks self.updatedisk(machine, unique_good_disks, true) # apc map update apc_map = self.cfg.globalParams.var_copy('APC_MAP') apc_map[machine] = apc_util.PortMap(apc_outlet) if not self.cfg.setGlobalParam('APC_MAP', apc_map): logging.error("ERROR setting apc map to %s" % repr(apc_map)) return 1 # create appropriate datadirs on that machine if not self.cfg.createDataDirs([machine], node_replacement=1): logging.error("ERROR could not create datadirs on machine %s" % machine) return 1 # Replicate the config self.cfg.replicateConfigOnMachine(machine) # Reconfigure net on the target machine if not reconfigurenet_util.doReconfigureNet( self.cfg.globalParams, [machine], i_am_master=0): logging.error('reconfigurenet failed for %s' % machine) return 1 # Start core services on the new node if not install_utilities.start_core(ver, home, [machine], ignore=0): logging.error("ERROR could not start core services on %s" % machine) return 1 # Add the chunkserver back gfs_utils.AddGFSChunkservers(ver, testver, [machine]) # first we need to do Machine allocation. # this will assign things that will satisfy the constraints if not self.cfg.DoMachineAllocation(serversets=['workqueue-slave']): logging.error("ERROR doing machine allocation") return 1 # now try to relllocate some servers from existing machines to the new machine replaced = self.cfg.AllocateServersToNewMachine(machine) if not replaced: logging.error("ERROR allocating services to the new machine") return 1 # first we need to restart the babysitter E.exe("%s %s" % (serve_cmd, "babysit")) time.sleep(60) # Now we need to stop all the replaced services for server_string in replaced: server = serverlib.Server() server.InitFromName(server_string) replaced_type = server.servertype() kill_cmd = servertype.GetKillCmd(replaced_type, server.port()) if E.execute([server.host()], kill_cmd, None, 1) != E.ERR_OK: logging.error("ERROR killing %s running on port %d on %s" % \ (replaced_type, server.port(), server.host())) # we should make it active if not install_utilities.set_install_state( machine, self.cfg.getGlobalParam('ENTERPRISE_HOME'), "ACTIVE"): logging.error("ERROR changing state on machine %s. " "Please make it active and activate and " "start crawl service on it" % machine) return 1 crawl_cmd = ". %s && cd %s/local/google3/enterprise/legacy/scripts && " \ "./crawl_service.py %s" % ( self.cfg.getGlobalParam('ENTERPRISE_BASHRC'), self.cfg.getGlobalParam('ENTERPRISE_HOME'), self.cfg.getGlobalParam('ENTERPRISE_HOME')) if E.execute([machine], "%s %s" % (crawl_cmd, "start"), None, 1) != E.ERR_OK: logging.error("Could not start crawl service on %s" % machine) return 1 # save all the params self.cfg.saveParams() # for faster crawl recovery, lets restart all crawl processes self.restart_crawl_processes(serve_cmd) # activate the crawl and logcontrol service on the remote machine crawl_activate_cmd = "/etc/rc.d/init.d/crawl_%s activate >&/dev/null" \ "</dev/null" % self.cfg.getGlobalParam('VERSION') if E.execute([machine], SECURE_WRAPPER_COMMAND % ( \ self.cfg.getGlobalParam('ENTERPRISE_HOME'), "-e", crawl_activate_cmd), None, 0) != E.ERR_OK: logging.error("Could not activate crawl service on machine %s" % machine) logging.error("Please activate by hand") return 1 log_activate_cmd = "/etc/rc.d/init.d/logcontrol_%s activate >&/dev/null" \ "</dev/null" % self.cfg.getGlobalParam('VERSION') if E.execute([machine], SECURE_WRAPPER_COMMAND % ( \ self.cfg.getGlobalParam('ENTERPRISE_HOME'), "-e", log_activate_cmd), None, 0) != E.ERR_OK: logging.error( "Could not activate logcontrol service on machine %s" % machine) logging.error("Please activate by hand") return 1 serve_activate_cmd = "/etc/rc.d/init.d/serve_%s activate >&/dev/null" \ "</dev/null" % self.cfg.getGlobalParam('VERSION') if E.execute([machine], SECURE_WRAPPER_COMMAND % ( \ self.cfg.getGlobalParam('ENTERPRISE_HOME'), "-e", serve_activate_cmd), None, 0) != E.ERR_OK: logging.error("Could not activate serve service on machine %s" % machine) logging.error("Please activate by hand") return 1 logging.info("Machine %s successfully added into the system" % machine) if not mail_already_sent(M.MSG_MACHINEADDED % machine): SendMail.send(self.cfg, None, false, M.MSG_MACHINEADDED % machine, "", true) return 0
def change_install_state(self): """ Tries to change the state of the present version to target_state. Returns true in case of success. Here is sumary of what it does: 1. Get list of active nodes 2. Get list of services to start and stop 3. In case there is something to start a. reconfigure's net on all nodes after verifying quorum b. starts core servics 4. Verifies there is a master elected. 5. Starts thread for each node to start and stop the needed services 6. Waits for output from each thread 7. Calculates success of failure based on thread results 8. Asks each thread to print its status regarding what services it actually started or stopped and what was the return code and error message if any. """ if not install_utilities.safe_transition(self.version_, self.target_state_): return 0 current_state = install_utilities.install_state(self.version_) start = time.time() # make sure svs is running svs_utilities.CheckSvsAlive(self.machines_) # get list of active nodes active_nodes = core_utils.GetLiveNodes(logging, self.retry_) ignore = core_utils.GetNodeFailures(core_utils.GetTotalNodes()) # account for already inactive nodes ignore = ignore - (core_utils.GetTotalNodes() - len(active_nodes)) ver = self.version_ home = self.enthome_ # See what we have to start / stop services_to_start = install_utilities.state_services_to_start( self.target_state_, self.machines_) services_to_stop = install_utilities.state_services_to_stop( install_utilities.install_state(self.version_), self.machines_) # Make some decisions total_nodes = len(self.cp_.var('ENT_ALL_MACHINES')) onebox = (total_nodes == 1) startcore = services_to_start and not onebox and not self.nonecore_only_ checkquorum = startcore stopcore = (services_to_stop and not onebox and not self.nonecore_only_ and self.target_state_ == 'INACTIVE') doservices = (not self.core_only_ and (services_to_start or services_to_stop)) if self.target_state_ in ['INACTIVE']: # ent_core does not really know the state. install_manager # has to tell ent_core when "makeinactive" testver = install_utilities.install_state(self.version_) else: testver = self.target_state_ in ['TEST', 'INSTALL'] # If it is onebox and target state is INSTALL, do not run reconfigure_net # This is to support pre 4.4 version migration code. reconfigurenet_enabled = not (onebox and (self.target_state_ == 'INSTALL')) # if stop coreonly services, check if none-core components are running if (install_utilities.install_state(self.version_) == 'ACTIVE' and self.target_state_ == 'INACTIVE' and self.core_only_): logging.fatal("cannot stop core services while none core services "\ "are running") # Execute the decisions if checkquorum: # We check quorum only when services are to be started. # We mainly need quorum for core services. For non core services like # crawl, logcontrol etc. we use users specified machines. core_utils.VerifyQuorum(active_nodes) # check if syslogd.conf and klogd.conf exist install_utilities.check_klogd_syslogd_conf(active_nodes, home) # Kill any spurious adminrunner/adminconsole processes if we are entering # TEST or ACTIVE mode. if self.target_state_ in ['TEST', 'ACTIVE']: install_utilities.kill_service(['adminrunner', 'adminconsole'], core_utils.GetNodes(1)) # reconfigure without restarting gems success = 1 if reconfigurenet_enabled and services_to_start: # check if we need to force NTP reconfig if this is to upgrade from 4.4 force_ntp_reconfig = 0 if self.target_state_ in ['TEST', 'ACTIVE']: last_version = install_utilities.get_latest_version(except_for=1) if (last_version is None or version_utilities.CmpVersions(last_version, NEW_NTP_OPTION_GSA_VERSION) > 0): force_ntp_reconfig = 1 success = reconfigurenet_util.doReconfigureNet(self.cp_, active_nodes, force_ntp_reconfig=force_ntp_reconfig) if not success: logging.error('reconfigurenet failed.') # if start nonecore services, check if core services are running if (not onebox and self.nonecore_only_ and self.target_state_ in ['TEST', 'ACTIVE']): core_running = install_utilities.is_core_running(ver, home, active_nodes, ignore=ignore, testver=testver) if not core_running: logging.fatal("cannot start none core services "\ "when core services are not running") # start core services if needed if startcore and success: # Retry 3 times for master verification failures num_retry = 3 # it is always OK to reinit core services if the version is in # INSTALLED state self.reinitok_ = install_utilities.reinit_core_ok(ver, home, active_nodes, ignore=ignore, testver=testver) i = 1 while i <= num_retry: # stop core services when retrying if i > 1: time.sleep(15) install_utilities.stop_core(ver, home, active_nodes, testver=testver) time.sleep(15) i = i + 1 # Run ent_core --ver=<ver> --activate --gfs=0 through install_utilities.py success = install_utilities.start_core(ver, home, active_nodes, ignore=ignore, testver=testver, gfs=0) if not success: if i <= num_retry: logging.error('Error activating core services. Retrying...') elif self.reinitok_: # it is OK to ignore errors when trying to re-init core services install_utilities.reinit_core(ver, home, active_nodes, ignore=1, testver=testver) i = 1 self.reinitok_ = None else: logging.error('Error activating core services.') else: # Make sure a master has been elected. If we go ahead without # verifying the master then it will take very long time for # services to be started. Making sure master is elected by now # results in very quick adminrunner startup. success = verify_master(ver, testver) if success: if not core_utils.InitDeadNodes(ver, testver, logging) == 0: logging.fatal('Error updating dead nodes to the lockserver.') break if i <= num_retry: logging.error('Error verifying the master. Retrying...') elif self.reinitok_: # it is OK to ignore errors when trying to re-init core services install_utilities.reinit_core(ver, home, active_nodes, ignore=1, testver=testver) i = 1 self.reinitok_ = None else: raise core_utils.EntMasterError, ('Error getting current GSA master' ' from chubby.') # force gsa master on the desired node desired_gsa_master_node = core_utils.DesiredMasterNode() if desired_gsa_master_node is None: logging.fatal('No suitable node to run GSA master') logging.info('Forcing %s to become GSA master' % desired_gsa_master_node) find_master.ForceMaster(desired_gsa_master_node, testver) # make sure the transaction logs are in sync and start gfs success = install_utilities.start_gfs(ver, home, active_nodes, ignore=ignore, testver=testver) # make sure gfs master is not the GSA master node logging.info('Ensuring %s not to become GFS master' % desired_gsa_master_node) gfs_utils.AvoidGFSMasterOnNode(ver, testver, desired_gsa_master_node) if doservices and success: node_threads = {} for n in self.machines_: node_threads[n] = NodeInstallManager(n, self.target_state_, self.version_, services_to_start, services_to_stop) # start node threads for (n, t) in node_threads.items(): logging.info('STATUS: Starting thread for %s' % n) t.start() # wait for threads for (n,t) in node_threads.items(): t.join() success = success and (t.err_ == 0) for (n,t) in node_threads.items(): t.print_status() if stopcore and success: func = lambda: install_utilities.stop_core(ver, home, active_nodes, testver=testver) success = try_repeatedly(func, success=1) if not success: logging.error('Error inactivating core services.') # Start/Stop Borgmon and Reactor if self.cp_.var('ENT_ENABLE_EXTERNAL_BORGMON'): enable_external_borgmon = '--enable_external' else: enable_external_borgmon = '--noenable_external' borgmon_cmd = ( "/export/hda3/%s/local/google3/enterprise/util/borgmon_util.py " "--ver %s --logtostderr %s" % (self.version_, self.version_, enable_external_borgmon)) if success and current_state != self.target_state_: # 1) Stop Borgmon and Reactor if required if current_state in ['SERVE', 'TEST', 'ACTIVE']: E.execute(self.machines_, "%s --mode %s --stop" % (borgmon_cmd, current_state), None, 0) # 2) Start Borgmon and Reactor if required logging.info("target_state: %s" % self.target_state_) if self.target_state_ in ['SERVE', 'TEST', 'ACTIVE']: E.execute(self.machines_, "%s --mode %s --start" % (borgmon_cmd, self.target_state_), None, 0) # Start/Stop Session Manager only for oneways if core_utils.GetTotalNodes() == 1: if self.target_state_ in ['SERVE', 'TEST', 'ACTIVE']: sessionmanager_util.ActivateSessionManager(ver, testver) if self.target_state_ == 'INACTIVE' and success: sessionmanager_util.DeactivateSessionManager(ver, testver) # Kill any spurious adminrunner/adminconsole processes if we are entering # INACTIVE or SERVE mode. if self.target_state_ in ['SERVE', 'INACTIVE']: install_utilities.kill_service(['adminrunner', 'adminconsole'], core_utils.GetNodes(1)) if self.target_state_ == 'INACTIVE' and success and not self.nonecore_only_: install_utilities.InactivateCleanup(ver, home, active_nodes) end = time.time() diff = (end - start)/60 logging.info("STAT: change_install_state took %.2f minutes." % diff) return success
def add(self, machine, apc_outlet): """ This adds a machine to the configuration """ # We can add a machine only when we are in active state if install_utilities.install_state(self.cfg.getGlobalParam('VERSION')) != "ACTIVE": logging.error("Can add a machine only when we are in active state") return 1 # First test for accessibility of the machine. if E.execute([machine], 'echo 1', None, 1) != E.ERR_OK: logging.error("Could not ssh into the machine %s" % machine) return 1 # start the svs on the remote machine restart_svs_cmd = "%s/local/google3/enterprise/legacy/util/svs_utilities.py %s %s" % ( self.cfg.getGlobalParam('ENTERPRISE_HOME'), self.cfg.getGlobalParam('ENTERPRISE_HOME'), machine) if E.execute([E.getCrtHostName()], SECURE_WRAPPER_COMMAND % ( \ self.cfg.getGlobalParam('ENTERPRISE_HOME'), "-p2", restart_svs_cmd), None, 0) != E.ERR_OK: logging.error("Could not start svs on machine %s" % machine) return 1 # wait for some time for svs to come up time.sleep(5) # check to see if the svs is up and is the right version if not svs_utilities.PingAndCheckSvsVersion( self.cfg.getGlobalParam('ENTERPRISE_BASHRC'), self.cfg.getGlobalParam('ENTERPRISE_HOME'), machine): logging.error("Svs not running correctly on machine %s" % machine) return 1 ver = self.cfg.getGlobalParam('VERSION') home = self.cfg.getGlobalParam('ENTERPRISE_HOME') testver = install_utilities.is_test(ver) # update MACHINES machines = self.cfg.getGlobalParam('MACHINES') if machine not in machines: machines.append(machine) self.cfg.setGlobalParam('MACHINES', machines) ret = core_utils.RemDeadNode(ver, testver, machine) if ret: logging.error('Cannot remove dead node from lockserver.') # we ignore this error for now # We just added a new machine into the config # this will lead to a change in concentrator config # so we need to re-run serve service which will # write the new config and restart the concentrator serve_cmd = ". %s && cd %s/local/google3/enterprise/legacy/scripts && " \ "./serve_service.py %s" % ( self.cfg.getGlobalParam('ENTERPRISE_BASHRC'), self.cfg.getGlobalParam('ENTERPRISE_HOME'), self.cfg.getGlobalParam('ENTERPRISE_HOME')) E.exe("%s %s" % (serve_cmd, "babysit")) num_tries = 5 cur_try = 0 while cur_try < num_tries: cur_try = cur_try + 1 all_disks = self.cfg.mach_param_cache.GetFact("mounted-drives", machine) bad_disks = self.cfg.mach_param_cache.GetFact("var_log_badhds", machine) if bad_disks and all_disks: break time.sleep(60) if all_disks == None or bad_disks == None: logging.error("Could not get machine information about %s" % machine) return 1 bad_disks = string.split(bad_disks, ' ') all_disks = string.split(all_disks, ' ') good_disks = filter(lambda x, y=bad_disks: x not in y, all_disks) good_disks = map(lambda x: "%s3" % x, good_disks) # change sda3 to hda3 etc. good_disks = map(lambda x: re.sub(r'^s', 'h', x), good_disks) # Preprocess disks before adding to remove duplicates. unique_good_disks = [] [unique_good_disks.append(disk) for disk in good_disks if disk not in unique_good_disks] # Add disks self.updatedisk(machine, unique_good_disks, true) # apc map update apc_map = self.cfg.globalParams.var_copy('APC_MAP') apc_map[machine] = apc_util.PortMap(apc_outlet) if not self.cfg.setGlobalParam('APC_MAP', apc_map): logging.error("ERROR setting apc map to %s" % repr(apc_map)) return 1 # create appropriate datadirs on that machine if not self.cfg.createDataDirs([machine], node_replacement = 1): logging.error("ERROR could not create datadirs on machine %s" % machine) return 1 # Replicate the config self.cfg.replicateConfigOnMachine(machine) # Reconfigure net on the target machine if not reconfigurenet_util.doReconfigureNet(self.cfg.globalParams, [machine], i_am_master=0): logging.error('reconfigurenet failed for %s' % machine) return 1 # Start core services on the new node if not install_utilities.start_core(ver, home, [machine], ignore=0): logging.error("ERROR could not start core services on %s" % machine) return 1 # Add the chunkserver back gfs_utils.AddGFSChunkservers(ver, testver, [machine]) # first we need to do Machine allocation. # this will assign things that will satisfy the constraints if not self.cfg.DoMachineAllocation(serversets=['workqueue-slave']): logging.error("ERROR doing machine allocation") return 1 # now try to relllocate some servers from existing machines to the new machine replaced = self.cfg.AllocateServersToNewMachine(machine) if not replaced: logging.error("ERROR allocating services to the new machine") return 1 # first we need to restart the babysitter E.exe("%s %s" % (serve_cmd, "babysit")) time.sleep(60) # Now we need to stop all the replaced services for server_string in replaced: server = serverlib.Server() server.InitFromName(server_string) replaced_type = server.servertype() kill_cmd = servertype.GetKillCmd(replaced_type, server.port()) if E.execute([server.host()], kill_cmd, None, 1) != E.ERR_OK: logging.error("ERROR killing %s running on port %d on %s" % \ (replaced_type, server.port(), server.host())) # we should make it active if not install_utilities.set_install_state(machine, self.cfg.getGlobalParam('ENTERPRISE_HOME'), "ACTIVE"): logging.error("ERROR changing state on machine %s. " "Please make it active and activate and " "start crawl service on it" % machine) return 1 crawl_cmd = ". %s && cd %s/local/google3/enterprise/legacy/scripts && " \ "./crawl_service.py %s" % ( self.cfg.getGlobalParam('ENTERPRISE_BASHRC'), self.cfg.getGlobalParam('ENTERPRISE_HOME'), self.cfg.getGlobalParam('ENTERPRISE_HOME')) if E.execute([machine], "%s %s" % (crawl_cmd, "start"), None, 1) != E.ERR_OK: logging.error("Could not start crawl service on %s" % machine) return 1 # save all the params self.cfg.saveParams() # for faster crawl recovery, lets restart all crawl processes self.restart_crawl_processes(serve_cmd) # activate the crawl and logcontrol service on the remote machine crawl_activate_cmd = "/etc/rc.d/init.d/crawl_%s activate >&/dev/null" \ "</dev/null" % self.cfg.getGlobalParam('VERSION') if E.execute([machine], SECURE_WRAPPER_COMMAND % ( \ self.cfg.getGlobalParam('ENTERPRISE_HOME'), "-e", crawl_activate_cmd), None, 0) != E.ERR_OK: logging.error("Could not activate crawl service on machine %s" % machine) logging.error("Please activate by hand") return 1 log_activate_cmd = "/etc/rc.d/init.d/logcontrol_%s activate >&/dev/null" \ "</dev/null" % self.cfg.getGlobalParam('VERSION') if E.execute([machine], SECURE_WRAPPER_COMMAND % ( \ self.cfg.getGlobalParam('ENTERPRISE_HOME'), "-e", log_activate_cmd), None, 0) != E.ERR_OK: logging.error("Could not activate logcontrol service on machine %s" % machine) logging.error("Please activate by hand") return 1 serve_activate_cmd = "/etc/rc.d/init.d/serve_%s activate >&/dev/null" \ "</dev/null" % self.cfg.getGlobalParam('VERSION') if E.execute([machine], SECURE_WRAPPER_COMMAND % ( \ self.cfg.getGlobalParam('ENTERPRISE_HOME'), "-e", serve_activate_cmd), None, 0) != E.ERR_OK: logging.error("Could not activate serve service on machine %s" % machine) logging.error("Please activate by hand") return 1 logging.info("Machine %s successfully added into the system" % machine) if not mail_already_sent(M.MSG_MACHINEADDED % machine): SendMail.send(self.cfg, None, false, M.MSG_MACHINEADDED % machine, "", true) return 0
def reconfigurenet(self): """ Call reconfigurenet script on all machines """ return not reconfigurenet_util.doReconfigureNet(self.cfg.globalParams)
def change_install_state(self): """ Tries to change the state of the present version to target_state. Returns true in case of success. Here is sumary of what it does: 1. Get list of active nodes 2. Get list of services to start and stop 3. In case there is something to start a. reconfigure's net on all nodes after verifying quorum b. starts core servics 4. Verifies there is a master elected. 5. Starts thread for each node to start and stop the needed services 6. Waits for output from each thread 7. Calculates success of failure based on thread results 8. Asks each thread to print its status regarding what services it actually started or stopped and what was the return code and error message if any. """ if not install_utilities.safe_transition(self.version_, self.target_state_): return 0 current_state = install_utilities.install_state(self.version_) start = time.time() # make sure svs is running svs_utilities.CheckSvsAlive(self.machines_) # get list of active nodes active_nodes = core_utils.GetLiveNodes(logging, self.retry_) ignore = core_utils.GetNodeFailures(core_utils.GetTotalNodes()) # account for already inactive nodes ignore = ignore - (core_utils.GetTotalNodes() - len(active_nodes)) ver = self.version_ home = self.enthome_ # See what we have to start / stop services_to_start = install_utilities.state_services_to_start( self.target_state_, self.machines_) services_to_stop = install_utilities.state_services_to_stop( install_utilities.install_state(self.version_), self.machines_) # Make some decisions total_nodes = len(self.cp_.var('ENT_ALL_MACHINES')) onebox = (total_nodes == 1) startcore = services_to_start and not onebox and not self.nonecore_only_ checkquorum = startcore stopcore = (services_to_stop and not onebox and not self.nonecore_only_ and self.target_state_ == 'INACTIVE') doservices = (not self.core_only_ and (services_to_start or services_to_stop)) if self.target_state_ in ['INACTIVE']: # ent_core does not really know the state. install_manager # has to tell ent_core when "makeinactive" testver = install_utilities.install_state(self.version_) else: testver = self.target_state_ in ['TEST', 'INSTALL'] # If it is onebox and target state is INSTALL, do not run reconfigure_net # This is to support pre 4.4 version migration code. reconfigurenet_enabled = not (onebox and (self.target_state_ == 'INSTALL')) # if stop coreonly services, check if none-core components are running if (install_utilities.install_state(self.version_) == 'ACTIVE' and self.target_state_ == 'INACTIVE' and self.core_only_): logging.fatal("cannot stop core services while none core services "\ "are running") # Execute the decisions if checkquorum: # We check quorum only when services are to be started. # We mainly need quorum for core services. For non core services like # crawl, logcontrol etc. we use users specified machines. core_utils.VerifyQuorum(active_nodes) # check if syslogd.conf and klogd.conf exist install_utilities.check_klogd_syslogd_conf(active_nodes, home) # Kill any spurious adminrunner/adminconsole processes if we are entering # TEST or ACTIVE mode. if self.target_state_ in ['TEST', 'ACTIVE']: install_utilities.kill_service(['adminrunner', 'adminconsole'], core_utils.GetNodes(1)) # reconfigure without restarting gems success = 1 if reconfigurenet_enabled and services_to_start: # check if we need to force NTP reconfig if this is to upgrade from 4.4 force_ntp_reconfig = 0 if self.target_state_ in ['TEST', 'ACTIVE']: last_version = install_utilities.get_latest_version( except_for=1) if (last_version is None or version_utilities.CmpVersions( last_version, NEW_NTP_OPTION_GSA_VERSION) > 0): force_ntp_reconfig = 1 success = reconfigurenet_util.doReconfigureNet( self.cp_, active_nodes, force_ntp_reconfig=force_ntp_reconfig) if not success: logging.error('reconfigurenet failed.') # if start nonecore services, check if core services are running if (not onebox and self.nonecore_only_ and self.target_state_ in ['TEST', 'ACTIVE']): core_running = install_utilities.is_core_running(ver, home, active_nodes, ignore=ignore, testver=testver) if not core_running: logging.fatal("cannot start none core services "\ "when core services are not running") # start core services if needed if startcore and success: # Retry 3 times for master verification failures num_retry = 3 # it is always OK to reinit core services if the version is in # INSTALLED state self.reinitok_ = install_utilities.reinit_core_ok(ver, home, active_nodes, ignore=ignore, testver=testver) i = 1 while i <= num_retry: # stop core services when retrying if i > 1: time.sleep(15) install_utilities.stop_core(ver, home, active_nodes, testver=testver) time.sleep(15) i = i + 1 # Run ent_core --ver=<ver> --activate --gfs=0 through install_utilities.py success = install_utilities.start_core(ver, home, active_nodes, ignore=ignore, testver=testver, gfs=0) if not success: if i <= num_retry: logging.error( 'Error activating core services. Retrying...') elif self.reinitok_: # it is OK to ignore errors when trying to re-init core services install_utilities.reinit_core(ver, home, active_nodes, ignore=1, testver=testver) i = 1 self.reinitok_ = None else: logging.error('Error activating core services.') else: # Make sure a master has been elected. If we go ahead without # verifying the master then it will take very long time for # services to be started. Making sure master is elected by now # results in very quick adminrunner startup. success = verify_master(ver, testver) if success: if not core_utils.InitDeadNodes(ver, testver, logging) == 0: logging.fatal( 'Error updating dead nodes to the lockserver.') break if i <= num_retry: logging.error( 'Error verifying the master. Retrying...') elif self.reinitok_: # it is OK to ignore errors when trying to re-init core services install_utilities.reinit_core(ver, home, active_nodes, ignore=1, testver=testver) i = 1 self.reinitok_ = None else: raise core_utils.EntMasterError, ( 'Error getting current GSA master' ' from chubby.') # force gsa master on the desired node desired_gsa_master_node = core_utils.DesiredMasterNode() if desired_gsa_master_node is None: logging.fatal('No suitable node to run GSA master') logging.info('Forcing %s to become GSA master' % desired_gsa_master_node) find_master.ForceMaster(desired_gsa_master_node, testver) # make sure the transaction logs are in sync and start gfs success = install_utilities.start_gfs(ver, home, active_nodes, ignore=ignore, testver=testver) # make sure gfs master is not the GSA master node logging.info('Ensuring %s not to become GFS master' % desired_gsa_master_node) gfs_utils.AvoidGFSMasterOnNode(ver, testver, desired_gsa_master_node) if doservices and success: node_threads = {} for n in self.machines_: node_threads[n] = NodeInstallManager(n, self.target_state_, self.version_, services_to_start, services_to_stop) # start node threads for (n, t) in node_threads.items(): logging.info('STATUS: Starting thread for %s' % n) t.start() # wait for threads for (n, t) in node_threads.items(): t.join() success = success and (t.err_ == 0) for (n, t) in node_threads.items(): t.print_status() if stopcore and success: func = lambda: install_utilities.stop_core( ver, home, active_nodes, testver=testver) success = try_repeatedly(func, success=1) if not success: logging.error('Error inactivating core services.') # Start/Stop Borgmon and Reactor if self.cp_.var('ENT_ENABLE_EXTERNAL_BORGMON'): enable_external_borgmon = '--enable_external' else: enable_external_borgmon = '--noenable_external' borgmon_cmd = ( "/export/hda3/%s/local/google3/enterprise/util/borgmon_util.py " "--ver %s --logtostderr %s" % (self.version_, self.version_, enable_external_borgmon)) if success and current_state != self.target_state_: # 1) Stop Borgmon and Reactor if required if current_state in ['SERVE', 'TEST', 'ACTIVE']: E.execute(self.machines_, "%s --mode %s --stop" % (borgmon_cmd, current_state), None, 0) # 2) Start Borgmon and Reactor if required logging.info("target_state: %s" % self.target_state_) if self.target_state_ in ['SERVE', 'TEST', 'ACTIVE']: E.execute( self.machines_, "%s --mode %s --start" % (borgmon_cmd, self.target_state_), None, 0) # Start/Stop Session Manager only for oneways if core_utils.GetTotalNodes() == 1: if self.target_state_ in ['SERVE', 'TEST', 'ACTIVE']: sessionmanager_util.ActivateSessionManager(ver, testver) if self.target_state_ == 'INACTIVE' and success: sessionmanager_util.DeactivateSessionManager(ver, testver) # Kill any spurious adminrunner/adminconsole processes if we are entering # INACTIVE or SERVE mode. if self.target_state_ in ['SERVE', 'INACTIVE']: install_utilities.kill_service(['adminrunner', 'adminconsole'], core_utils.GetNodes(1)) if self.target_state_ == 'INACTIVE' and success and not self.nonecore_only_: install_utilities.InactivateCleanup(ver, home, active_nodes) end = time.time() diff = (end - start) / 60 logging.info("STAT: change_install_state took %.2f minutes." % diff) return success