def update_slave(task_item, config):
is_slave_updated = None
try:
if Const.isOsaWinPlatform(task_item.host.platform.os):
if task_item.paagent_dist_url:
schedule_update_windows_slave(task_item)
uAction.ntimes_retried(check_update_windows_slave, 8, 30)(task_item, config)
is_slave_updated = True
else:
linux_update_result = update_linux_slave(task_item, config)
check_update_linux_slave(task_item.host, linux_update_result)
# This is a workaround: see async restart code in pa-agent RPM
async_upgrade_finished_sec = 11
uLogging.debug("Waiting {async_upgrade_finished_sec} sec for agent upgrade process is finished "
"on host {host}".format(host=task_item.host,
async_upgrade_finished_sec=async_upgrade_finished_sec))
sleep(async_upgrade_finished_sec)
uAction.ntimes_retried(wait_for_linux_slave, 8, 30)(task_item)
is_slave_updated = True
except Exception as e:
uLogging.warn("Error happen during upgrade slave {host}, check details in log or wait for moment when "
"the result will be processed".format(host=task_item.host))
uLogging.save_traceback()
raise e
return is_slave_updated
def is_slave_upgradable(host, binfo, config):
uLogging.debug("Checking if slave {host} is upgradable".format(host=host))
is_windows_host = Const.isOsaWinPlatform(host.platform.os)
if host.host_id in binfo.progress.updated_hosts:
uLogging.debug("Skipping already updated slave: {host}".format(host=host))
return False
if config.skip_win and is_windows_host:
uLogging.debug("Skipping windows host {host} because was passed option '--skip:win'".format(host=host))
return False
if config.skip_rpms and not is_windows_host:
uLogging.debug("Skipping Linux (RPM) host {host} because was passed option '--skip:rpms'".format(host=host))
return False
if not uPEM.canProceedWithHost(host):
uLogging.warn("Skipping unavailable slave: {host}".format(host=host))
return False
# platform-specific URL for agent
agent_url = prepare_paagent_url(host.platform, binfo, config)
if not agent_url and is_windows_host:
uLogging.debug("Skipping windows slave {host} as we have no pa-agent distrib "
"for its platform ({platform})".format(host=host, platform=host.platform))
return False
if not agent_url and not config.need_update_paagent and not config.need_update_yum_repourl:
uLogging.debug("Skipping slave {host} as we have no pa-agent distrib for its platform ({platform}) "
"and updating YUM repourl is not needed also".format(host=host, platform=host.platform))
return False
uLogging.debug("Slave is upgradable: {host}".format(host=host))
return True
def runHCLCmd(hostId, commandText):
uLogging.debug(commandText)
con = uSysDB.connect()
host = uPEM.getHost(hostId)
commandText = commandText.replace(
"${", '$${'
) #for more details see following issue https://jira.int.zone/browse/POA-109131
rq = Request(user='******', group='root')
rq.command(commandText,
stdout='stdout',
stderr='stderr',
valid_exit_codes=[0])
if Const.isOsaWinPlatform(host.platform.os):
rq.export("default_shell_name", "cmd.exe", True)
rq.export("shell_cmd_switch", "/C", True)
else:
rq.export("default_shell_name", "/bin/sh", True)
rq.export("shell_cmd_switch", "-c", True)
rqRv = None
try:
rqRv = rq.send(host)
except uUtil.ExecFailed:
rqRv = rq.performRaw(host)
o = rqRv["stdout"]
if o: uLogging.debug(o)
return o
def performRaw(self, host):
if Const.isOsaWinPlatform(host.platform.os):
shell = "cmd.exe"
cmd_switch = "/C"
else:
shell = "/bin/sh"
cmd_switch = "-c"
self.export("default_shell_name", shell, True)
self.export("shell_cmd_switch", cmd_switch, True)
try:
return self.__performHCLRequest(host, self.__document)
except uUtil.ExecFailed, e:
# Notation "ex_type_id:'103'" in stderr it is a sign that exception OBJECT_NOT_EXIST is raised
# (from modules/platform/u/EAR/poakernel-public/Common.edl).
# We need to retry HCL request because the reason may be an outdated CORBA cache.
# Cache will be invalidated in this case, so repeated request will pass
# (modules/platform/cells/pem_client/cpp/Naming/Naming.cpp).
deprecated_cache_error_pattern = re.compile(
"OBJ_ADAPTER|OBJECT_NOT_EXIST|ex_type_id:'103'")
if re.search(deprecated_cache_error_pattern, e.err):
uLogging.debug("HCL request exec failed. Retrying...")
return self.__performHCLRequest(host, self.__document)
else:
raise
def prepare_paagent_url(platform, binfo, config):
if platform not in paagent_urls:
if Const.isOsaWinPlatform(platform.os):
paagent_urls[platform] = prepare_paagent_exe_url(platform, binfo, config)
else:
paagent_urls[platform] = prepare_paagent_rpm_url(platform, binfo, config)
uLogging.debug("Defined pa-agent URL for platform {platform} is {url}".format(
platform=platform, url=paagent_urls[platform]
))
return paagent_urls[platform]
def get_path_module(host):
if Const.isOsaWinPlatform(host.platform.os):
return ntpath
else:
return posixpath
def get_hosts_with_agent_exe():
return [host for host in getAllHosts() if host.pleskd_id and Const.isOsaWinPlatform(host.platform.os)]
def get_hosts_with_agent_rpm():
return [host for host in getAllHosts()
if host.type in ('n', 'c', 'f', 'g') and not Const.isOsaWinPlatform(host.platform.os)]
def check_internal_net_bandwidth(config):
""" Here we are checking network bandwidth to be sure that pa-agents will be delivered in time on all slaves
during update.
"""
if config.skip_check_internal_net_bandwidth:
uLogging.warn('Checking internal net bandwidth was skipped.')
return
from poaupdater import uSlaveUpdater
# MB (megabytes) per second, we are assuming to utilize ethernet (100BaseT)
# with host count calculated by uSlaveUpdater.max_slave_upgrade_threads()
min_mbps = 1
# Initializing file for test firstly to get its instance further and undeploy it finally
file_for_speedtest = FileForSpeedtest(config.communication_ip)
slaves_with_agent = [host for host in get_hosts_with_agent() if int(host.host_id) != 1]
uLogging.debug("Found slaves with agent: \n{slaves}".format(slaves=pprint.pformat(slaves_with_agent)))
if config.skip_rpms:
slaves_with_agent = filter(lambda host: Const.isOsaWinPlatform(host.platform.os), slaves_with_agent)
uLogging.debug("Filtered out Linux slaves because was passed option '--skip:rpms', "
"now slave list is \n{slaves}".format(slaves=pprint.pformat(slaves_with_agent)))
if config.skip_win:
slaves_with_agent = filter(lambda host: not Const.isOsaWinPlatform(host.platform.os), slaves_with_agent)
uLogging.debug("Filtered out Windows slaves because was passed option '--skip:win', "
"now slave list is \n{slaves}".format(slaves=pprint.pformat(slaves_with_agent)))
if not slaves_with_agent:
uLogging.debug("No slaves found, no need to check internal network bandwidth")
return
thread_count = uSlaveUpdater.max_slave_upgrade_threads(
hosts_to_upgrade_count=len(slaves_with_agent), slave_upgrade_threads=config.slave_upgrade_threads
)
uLogging.info("Checking network bandwidth between MN and slaves "
"(parallel: {thread_count})".format(thread_count=thread_count))
pool = uThreadPool.ThreadPool(check_net_bandwidth_for_host)
map(pool.put, slaves_with_agent)
try:
pool.start(thread_count)
check_results = pool.get_all_non_empty_results()
finally:
pool.terminate()
file_for_speedtest.undeploy_file_for_test()
if "__iter__" not in dir(check_results) or len(check_results) < 1:
raise Exception("Check for speed test returned nothing, this is unexpected")
completely_failed_checks = filter(lambda result: type(result.result) is not HostBandwidthReport, check_results)
if completely_failed_checks:
raise Exception("Some of check has completely failed, summary report is inconsistent: \n{failed_items}".format(
failed_items=pprint.pformat(completely_failed_checks)))
# uThreadPool returns namedtuple ["task_item", "result", "duration"], we need only result
check_results = [x.result for x in check_results]
uLogging.debug("Checking reports for speed sufficiency")
map(lambda report: report.calc_is_sufficient(min_mbps), check_results)
uLogging.debug("Results: \n" + pprint.pformat(check_results))
failed_checks_report = [str(check_result) for check_result in check_results if not check_result.sufficient]
if failed_checks_report:
raise uPrecheck.PrecheckFailed(
reason="detected network issue, bandwidth between MN and slave is not sufficient",
what_to_do="please fix network issue for hosts reported below: \n{bandwidth_reports}".format(
bandwidth_reports="\n".join(failed_checks_report)
)
)
uLogging.debug("Checking network bandwidth finished successfully - all slaves are ok.")
def ping(host_id):
ip = getHostCommunicationIP(host_id)
port = 8352
try:
if ip:
uURLChecker.try_connect((ip, port), 2)
except socket.error, e:
raise Exception('Connection to host %s failed (%s). Please ensure host is online and pa-agent service is running on it.' % (ip, e))
# node is reachable by http. remove from unreachable_hosts just in case (POA-113200)
cleanUnreachableState(host_id)
hcl = uHCL.Request(host_id=host_id)
host = getHost(host_id)
if Const.isOsaWinPlatform(host.platform.os):
hcl.command("echo 0")
else:
hcl.set_creds(user='******')
hcl.command("/bin/echo 0")
try:
hcl.performCompat()
return
except uUtil.ExecFailed:
pass
execCtl('pleskd_ctl', ['ping', str(host_id)])
def _execCtl(ctlname, fun, *args):
platform, root = getMNInfo()
if len(args) == 1 and type(args[0]) in (tuple, list):
