def update_slave(task_item, config): is_slave_updated = None try: if Const.isOsaWinPlatform(task_item.host.platform.os): if task_item.paagent_dist_url: schedule_update_windows_slave(task_item) uAction.ntimes_retried(check_update_windows_slave, 8, 30)(task_item, config) is_slave_updated = True else: linux_update_result = update_linux_slave(task_item, config) check_update_linux_slave(task_item.host, linux_update_result) # This is a workaround: see async restart code in pa-agent RPM async_upgrade_finished_sec = 11 uLogging.debug("Waiting {async_upgrade_finished_sec} sec for agent upgrade process is finished " "on host {host}".format(host=task_item.host, async_upgrade_finished_sec=async_upgrade_finished_sec)) sleep(async_upgrade_finished_sec) uAction.ntimes_retried(wait_for_linux_slave, 8, 30)(task_item) is_slave_updated = True except Exception as e: uLogging.warn("Error happen during upgrade slave {host}, check details in log or wait for moment when " "the result will be processed".format(host=task_item.host)) uLogging.save_traceback() raise e return is_slave_updated
def is_slave_upgradable(host, binfo, config): uLogging.debug("Checking if slave {host} is upgradable".format(host=host)) is_windows_host = Const.isOsaWinPlatform(host.platform.os) if host.host_id in binfo.progress.updated_hosts: uLogging.debug("Skipping already updated slave: {host}".format(host=host)) return False if config.skip_win and is_windows_host: uLogging.debug("Skipping windows host {host} because was passed option '--skip:win'".format(host=host)) return False if config.skip_rpms and not is_windows_host: uLogging.debug("Skipping Linux (RPM) host {host} because was passed option '--skip:rpms'".format(host=host)) return False if not uPEM.canProceedWithHost(host): uLogging.warn("Skipping unavailable slave: {host}".format(host=host)) return False # platform-specific URL for agent agent_url = prepare_paagent_url(host.platform, binfo, config) if not agent_url and is_windows_host: uLogging.debug("Skipping windows slave {host} as we have no pa-agent distrib " "for its platform ({platform})".format(host=host, platform=host.platform)) return False if not agent_url and not config.need_update_paagent and not config.need_update_yum_repourl: uLogging.debug("Skipping slave {host} as we have no pa-agent distrib for its platform ({platform}) " "and updating YUM repourl is not needed also".format(host=host, platform=host.platform)) return False uLogging.debug("Slave is upgradable: {host}".format(host=host)) return True
def runHCLCmd(hostId, commandText): uLogging.debug(commandText) con = uSysDB.connect() host = uPEM.getHost(hostId) commandText = commandText.replace( "${", '$${' ) #for more details see following issue https://jira.int.zone/browse/POA-109131 rq = Request(user='******', group='root') rq.command(commandText, stdout='stdout', stderr='stderr', valid_exit_codes=[0]) if Const.isOsaWinPlatform(host.platform.os): rq.export("default_shell_name", "cmd.exe", True) rq.export("shell_cmd_switch", "/C", True) else: rq.export("default_shell_name", "/bin/sh", True) rq.export("shell_cmd_switch", "-c", True) rqRv = None try: rqRv = rq.send(host) except uUtil.ExecFailed: rqRv = rq.performRaw(host) o = rqRv["stdout"] if o: uLogging.debug(o) return o
def performRaw(self, host): if Const.isOsaWinPlatform(host.platform.os): shell = "cmd.exe" cmd_switch = "/C" else: shell = "/bin/sh" cmd_switch = "-c" self.export("default_shell_name", shell, True) self.export("shell_cmd_switch", cmd_switch, True) try: return self.__performHCLRequest(host, self.__document) except uUtil.ExecFailed, e: # Notation "ex_type_id:'103'" in stderr it is a sign that exception OBJECT_NOT_EXIST is raised # (from modules/platform/u/EAR/poakernel-public/Common.edl). # We need to retry HCL request because the reason may be an outdated CORBA cache. # Cache will be invalidated in this case, so repeated request will pass # (modules/platform/cells/pem_client/cpp/Naming/Naming.cpp). deprecated_cache_error_pattern = re.compile( "OBJ_ADAPTER|OBJECT_NOT_EXIST|ex_type_id:'103'") if re.search(deprecated_cache_error_pattern, e.err): uLogging.debug("HCL request exec failed. Retrying...") return self.__performHCLRequest(host, self.__document) else: raise
def prepare_paagent_url(platform, binfo, config): if platform not in paagent_urls: if Const.isOsaWinPlatform(platform.os): paagent_urls[platform] = prepare_paagent_exe_url(platform, binfo, config) else: paagent_urls[platform] = prepare_paagent_rpm_url(platform, binfo, config) uLogging.debug("Defined pa-agent URL for platform {platform} is {url}".format( platform=platform, url=paagent_urls[platform] )) return paagent_urls[platform]
def get_path_module(host): if Const.isOsaWinPlatform(host.platform.os): return ntpath else: return posixpath
def get_hosts_with_agent_exe(): return [host for host in getAllHosts() if host.pleskd_id and Const.isOsaWinPlatform(host.platform.os)]
def get_hosts_with_agent_rpm(): return [host for host in getAllHosts() if host.type in ('n', 'c', 'f', 'g') and not Const.isOsaWinPlatform(host.platform.os)]
def check_internal_net_bandwidth(config): """ Here we are checking network bandwidth to be sure that pa-agents will be delivered in time on all slaves during update. """ if config.skip_check_internal_net_bandwidth: uLogging.warn('Checking internal net bandwidth was skipped.') return from poaupdater import uSlaveUpdater # MB (megabytes) per second, we are assuming to utilize ethernet (100BaseT) # with host count calculated by uSlaveUpdater.max_slave_upgrade_threads() min_mbps = 1 # Initializing file for test firstly to get its instance further and undeploy it finally file_for_speedtest = FileForSpeedtest(config.communication_ip) slaves_with_agent = [host for host in get_hosts_with_agent() if int(host.host_id) != 1] uLogging.debug("Found slaves with agent: \n{slaves}".format(slaves=pprint.pformat(slaves_with_agent))) if config.skip_rpms: slaves_with_agent = filter(lambda host: Const.isOsaWinPlatform(host.platform.os), slaves_with_agent) uLogging.debug("Filtered out Linux slaves because was passed option '--skip:rpms', " "now slave list is \n{slaves}".format(slaves=pprint.pformat(slaves_with_agent))) if config.skip_win: slaves_with_agent = filter(lambda host: not Const.isOsaWinPlatform(host.platform.os), slaves_with_agent) uLogging.debug("Filtered out Windows slaves because was passed option '--skip:win', " "now slave list is \n{slaves}".format(slaves=pprint.pformat(slaves_with_agent))) if not slaves_with_agent: uLogging.debug("No slaves found, no need to check internal network bandwidth") return thread_count = uSlaveUpdater.max_slave_upgrade_threads( hosts_to_upgrade_count=len(slaves_with_agent), slave_upgrade_threads=config.slave_upgrade_threads ) uLogging.info("Checking network bandwidth between MN and slaves " "(parallel: {thread_count})".format(thread_count=thread_count)) pool = uThreadPool.ThreadPool(check_net_bandwidth_for_host) map(pool.put, slaves_with_agent) try: pool.start(thread_count) check_results = pool.get_all_non_empty_results() finally: pool.terminate() file_for_speedtest.undeploy_file_for_test() if "__iter__" not in dir(check_results) or len(check_results) < 1: raise Exception("Check for speed test returned nothing, this is unexpected") completely_failed_checks = filter(lambda result: type(result.result) is not HostBandwidthReport, check_results) if completely_failed_checks: raise Exception("Some of check has completely failed, summary report is inconsistent: \n{failed_items}".format( failed_items=pprint.pformat(completely_failed_checks))) # uThreadPool returns namedtuple ["task_item", "result", "duration"], we need only result check_results = [x.result for x in check_results] uLogging.debug("Checking reports for speed sufficiency") map(lambda report: report.calc_is_sufficient(min_mbps), check_results) uLogging.debug("Results: \n" + pprint.pformat(check_results)) failed_checks_report = [str(check_result) for check_result in check_results if not check_result.sufficient] if failed_checks_report: raise uPrecheck.PrecheckFailed( reason="detected network issue, bandwidth between MN and slave is not sufficient", what_to_do="please fix network issue for hosts reported below: \n{bandwidth_reports}".format( bandwidth_reports="\n".join(failed_checks_report) ) ) uLogging.debug("Checking network bandwidth finished successfully - all slaves are ok.")
def ping(host_id): ip = getHostCommunicationIP(host_id) port = 8352 try: if ip: uURLChecker.try_connect((ip, port), 2) except socket.error, e: raise Exception('Connection to host %s failed (%s). Please ensure host is online and pa-agent service is running on it.' % (ip, e)) # node is reachable by http. remove from unreachable_hosts just in case (POA-113200) cleanUnreachableState(host_id) hcl = uHCL.Request(host_id=host_id) host = getHost(host_id) if Const.isOsaWinPlatform(host.platform.os): hcl.command("echo 0") else: hcl.set_creds(user='******') hcl.command("/bin/echo 0") try: hcl.performCompat() return except uUtil.ExecFailed: pass execCtl('pleskd_ctl', ['ping', str(host_id)]) def _execCtl(ctlname, fun, *args): platform, root = getMNInfo() if len(args) == 1 and type(args[0]) in (tuple, list):