def runHCLCmd(hostId, commandText): uLogging.debug(commandText) con = uSysDB.connect() host = uPEM.getHost(hostId) commandText = commandText.replace( "${", '$${' ) #for more details see following issue https://jira.int.zone/browse/POA-109131 rq = Request(user='******', group='root') rq.command(commandText, stdout='stdout', stderr='stderr', valid_exit_codes=[0]) if Const.isOsaWinPlatform(host.platform.os): rq.export("default_shell_name", "cmd.exe", True) rq.export("shell_cmd_switch", "/C", True) else: rq.export("default_shell_name", "/bin/sh", True) rq.export("shell_cmd_switch", "-c", True) rqRv = None try: rqRv = rq.send(host) except uUtil.ExecFailed: rqRv = rq.performRaw(host) o = rqRv["stdout"] if o: uLogging.debug(o) return o
def perform(self, host_id=None): host_id = host_id or self.__host_id if host_id is None: raise Exception("host_id not defined") con = uSysDB.connect() host = uPEM.getHost(host_id) return self.performRaw(host)
def performCompat(self, host_id=None): host_id = host_id or self.__host_id if host_id is None: raise Exception("host_id not defined") con = uSysDB.connect() host = uPEM.getHost(host_id) try: return self.performRaw(host) except uUtil.ExecFailed: return self.send(host)
def send(self, host): from poaupdater import uConfig status_code = -1 resp_out = "" if self.__issuer is None: self.__issuer = getHostCertificateDigest(uPEM.getHost(1)) config = uConfig.Config() authToken = getHostAuthToken(host, config, self.__issuer) host_ip = uPEM.getHostCommunicationIP(host.host_id) uLogging.debug("Send REST HCL request to host %s (%s) ..." % (str(host.host_id), host_ip)) # uLogging.debug("%s" % self.toxml()) headers = { "Authorization": authToken, "X-Auth-Host%s" % str(host.host_id): authToken } for d_host_id in self.__delegated_hosts: if d_host_id != host.host_id: headers["X-Auth-Host%s" % str(d_host_id)] = getHostAuthToken( uPEM.getHost(d_host_id), config, self.__issuer) http_req = urllib2.Request("https://%s:8352/process" % host_ip, headers=headers, data=None) http_req.get_method = lambda: 'POST' try: if uLinux.SSL_VERIFICATION_ENFORCED: resp = urllib2.urlopen( http_req, context=ssl._create_unverified_context(), data=self.toxml()) else: resp = urllib2.urlopen(http_req, data=self.toxml()) resp_out = resp.read() try: status_code = resp.getcode() except AttributeError: status_code = resp.code except urllib2.HTTPError, error: status_code = error.code resp_out = error.read()
def removePgSlave(slaveHostID, masterRootPwd): if not uLogging.logfile: uLogging.init2("/var/log/pa/deregister_slave.log", True, False) slave = uPEM.getHost(slaveHostID) slaveCommunicationIP = uPEM.getHostCommunicationIP(slaveHostID) runOnSlave = lambda cmd: uHCL.runHCLCmd(slaveHostID, cmd) uLogging.info("Slave database server at %s (%s) is going to be removed.", slaveCommunicationIP, slave.name) pghaSettings = getPghaSettings() pgsqlOnSlave = None try: pgsqlOnSlave = uPgSQL.PostgreSQLConfig(commander = runOnSlave) except Exception, e: uLogging.info("Could not find slave database server at %s (%s): %s", slaveCommunicationIP, slave.name, str(e)) return
def upgrade_paagent_and_repourl(binfo, config, slave_hosts): config.need_update_paagent = is_need_to_update_paagent(binfo) if not config.need_update_paagent and not config.need_update_yum_repourl: uLogging.info("No new agent RPMs in build, updating YUM repo URL is not needed also, skipping agent update.") return uLogging.debug( "Need update YUM repo URL: {need_update_yum_repourl}; " "there are new agents in build: {need_update_paagent}".format( need_update_yum_repourl=config.need_update_yum_repourl, need_update_paagent=config.need_update_paagent)) uAction.progress.do("Updating agents on slave nodes") config.mn_as_cert_issuer = uHCL.getHostCertificateDigest(uPEM.getHost(1)) # Filtering out non-upgradable slaves slaves = [host for host in slave_hosts if host.host_id != 1] uLogging.debug("Found slaves with agent: \n{slaves}".format(slaves=pformat(slaves))) slaves = filter(lambda slave: is_slave_upgradable(slave, binfo, config), slaves) uLogging.debug("All non-upgradable slaves are filtered out, actual slave-to-upgrade list now: " "\n{slaves}".format(slaves=pformat(slaves))) if not slaves: uLogging.info("There is no slaves marked for update.") uAction.progress.done() return # Preparing task pool for updating. pool = preparePool(config) slave_upgrade_monitoring = SlaveUpgradeMonitoring(pool, binfo, slaves) # Filling task pool for slaves for host in slaves: try: # Defining paagent url. If this will be eq None, package will not be updated paagent_url = prepare_paagent_url(host.platform, binfo, config) uLogging.debug("Slave '{host}' upgrade scheduled. Pa-agent URL is: '{agent_url}'" .format(host=host, agent_url=paagent_url)) pool.put(TaskItem(host, paagent_url)) except uAction.ActionIgnored: continue thread_count = max_slave_upgrade_threads(hosts_to_upgrade_count=len(slaves), slave_upgrade_threads=config.slave_upgrade_threads) try: pool.start(thread_count) slave_upgrade_monitoring.start() uLogging.info("Upgrade for agents on slaves started (parallel: {thread_count})".format( thread_count=thread_count)) uAction.retriable(process_pool_results)(pool, binfo, config) uLogging.info("All results of upgrading agents on slaves processed. ") report_skipped_hosts(binfo) except (Exception, KeyboardInterrupt) as e: # to remember and raise what is happened under try statement if finally statement has failures too uLogging.save_traceback() pool.terminateLocked() raise e finally: pool.terminate() slave_upgrade_monitoring.terminate() uAction.progress.done()
def deployPgSlave(slaveHostID, isBillingMaster, masterRootPwd, readOnlyUserType, additionalIPs, slaveScript, slaveScriptArgs): if not uLogging.logfile: uLogging.init2("/var/log/pa/register_slave.log", True, False) uLogging.info("Deploying PostgreSQL slave server on PA service node #%d...", slaveHostID) masterHostID = 0 pghaSettings = getPghaSettings() if not isBillingMaster: if slaveHostID == 1: raise Exception("The target slave host is MN: no possibility to use MN node as a database replica.") row = _getPgDbInfo() databaseName = row[2] if pghaSettings.isHa: masterAddr = getHaMasterAddr(pghaSettings) masterPort = pghaSettings.haBackendPort targetReplicationSourceMasterAddr = pghaSettings.vip_2 else: masterAddr = row[0] masterPort = int(row[1]) targetReplicationSourceMasterAddr = masterAddr uLogging.info("Master DB location: '%s at %d'" % (masterAddr, masterPort)) runOnMaster = _getCommandRunner(masterAddr, masterRootPwd) if not runOnMaster.isLocal: uLogging.info("Master is automation database server running remotely at %s:%d.", masterAddr, masterPort) else: uLogging.info("Master is automation database server running locally at %s:%d.", masterAddr, masterPort) masterHostID = 1 else: if slaveHostID in (b.get_host_id() for b in uBilling.get_billing_hosts()): raise Exception("The target slave host is billing node: no possibility to use billing node as a database slave.") dbParams = uBilling.PBAConf.getBillingDBPrams() masterAddr = uBilling.PBAConf.getBillingDBHost() masterPort = int(uBilling.PBAConf.getBillingDBPort()) databaseName = uBilling.PBAConf.getBillingDBName() runOnMaster = _getCommandRunner(masterAddr, masterRootPwd) targetReplicationSourceMasterAddr = masterAddr uLogging.info("Master is billing database server running at %s:%d.", masterAddr, masterPort) masterHostID = None isPermitted = False slave = uPEM.getHost(slaveHostID) if not runOnMaster.isLocal: try: runCheck = lambda cmd: uUtil.runLocalCmd(cmd) checkHostPermittedToBeReplicaOfDB(runCheck, slave.name) isPermitted = True except: pass if not isPermitted: checkHostPermittedToBeReplicaOfDB(runOnMaster, slave.name) slaveCommunicationIP = uPEM.getHostCommunicationIP(slaveHostID) ipAddrJoined = ipAddrToUserUniqPostfix(slaveCommunicationIP) replUserName = "******"+ipAddrJoined replUserPwd = uUtil.generate_random_password(16) runOnSlave = lambda cmd: uHCL.runHCLCmd(slaveHostID, cmd) uLogging.info("Slave database server is going to be deployed at %s (%s)", slaveCommunicationIP, slave.name) pgsqlOnMaster = uPgSQL.PostgreSQLConfig(commander = runOnMaster) pgsqlVer = str(pgsqlOnMaster.get_version_as_int()) uLogging.info("Current running PostgreSQL version is '%s'" % pgsqlVer) verifyPostgresCertificate(pgsqlOnMaster) uLogging.info("Instaling PostgreSQL Server on the slave...") runOnSlave("yum install -y odin-perftools postgresql%s postgresql%s-server postgresql%s-contrib" % (pgsqlVer, pgsqlVer, pgsqlVer)) runOnSlave("yum reinstall -y odin-perftools postgresql%s postgresql%s-server postgresql%s-contrib" % (pgsqlVer, pgsqlVer, pgsqlVer)) uLogging.info("Installation has finished!") uLogging.info("Initializing database on slave...") pgsqlOnSlave = uPgSQL.PostgreSQLConfig(commander = runOnSlave) pgsqlOnSlave.cleanup() pgsqlOnSlave.init_db() uLinux.configureDatabaseImpl(pgsqlOnSlave, None, []) uLogging.info("Saving some slave personal configuration files...") slavePersonalFilesBu = [] slavePersonalFiles = ( pgsqlOnSlave.get_data_dir()+"/server.key", pgsqlOnSlave.get_data_dir()+"/server.crt", # pgsqlOnSlave.get_postgresql_conf(), pgsqlOnSlave.get_pghba_conf() ) slavePersonalDir = os.path.dirname(pgsqlOnSlave.get_data_dir().rstrip("/")) for pf in slavePersonalFiles: runOnSlave(""" su - postgres -c 'cp -f "%s" "%s/"' """ % (pf, slavePersonalDir)) slavePersonalFilesBu.append(os.path.join(slavePersonalDir, os.path.basename(pf))) pgsqlOnSlave.stop() uLogging.info("Database has been initialized!") uLogging.info("Enabling replication connection from slave to master...") runOnMaster(""" su - postgres -c "psql --port=%d -c \\"DROP ROLE IF EXISTS %s\\"" """ % (masterPort, replUserName,)) runOnMaster(""" su - postgres -c "psql --port=%d -c \\"CREATE ROLE %s WITH REPLICATION ENCRYPTED PASSWORD '%s' LOGIN CONNECTION LIMIT 8\\"" """ % (masterPort, replUserName, replUserPwd)) uLogging.info("Creating read-only user and users to be replicated from master to slave for farther readonly use on the slave node.") roUserName = "******"+ipAddrJoined roUserPwd = uUtil.generate_random_password(32) # Provide the reentrancy, make sure the database doesn't contain objects created by possible previous launches runOnMaster(""" su - postgres -c "psql --port=%d --dbname=%s -c \\"REVOKE SELECT ON ALL TABLES IN SCHEMA public FROM %s\\"" 2> /dev/null || echo -n """ % (masterPort, databaseName, roUserName)) runOnMaster(""" su - postgres -c "psql --port=%d -c \\"REVOKE EXECUTE ON FUNCTION func_stat_wal_receiver() from %s\\"" 2> /dev/null || echo -n """ % (masterPort, roUserName,)) runOnMaster(""" su - postgres -c "psql --port=%d -c \\"DROP ROLE IF EXISTS %s\\"" """ % (masterPort, roUserName,)) runOnMaster(""" su - postgres -c "psql --port=%d -c \\"CREATE ROLE %s WITH ENCRYPTED PASSWORD '%s' LOGIN\\"" """ % (masterPort, roUserName, roUserPwd)) #add ability to monitor replication status for RO user def psql_as_postgres(input): return r'su - postgres -c "psql --port=%d -c \"%s\""' % (masterPort, input) runOnMaster(psql_as_postgres("DROP FUNCTION IF EXISTS func_stat_wal_receiver();")) psql11_fields = [] if LooseVersion(pgsqlVer) >= LooseVersion("11"): psql11_fields = ["cast('' as text) as sender_host", "-1 as sender_port"] pg_stat_wal_receiver_sql = 'CREATE FUNCTION func_stat_wal_receiver() RETURNS SETOF pg_stat_wal_receiver as ' columns_to_select = ", ".join(["pid", "status", "receive_start_lsn", "receive_start_tli", "received_lsn", "received_tli", "last_msg_send_time", "last_msg_receipt_time", "latest_end_lsn", "latest_end_time", "cast('' as text) as slot_name"] + psql11_fields + ["cast('' as text) as conninfo"]) pg_stat_wal_receiver_sql += r'\\$\\$ select %s from pg_stat_wal_receiver; \\$\\$ LANGUAGE sql SECURITY DEFINER;' % columns_to_select runOnMaster(psql_as_postgres(pg_stat_wal_receiver_sql)) runOnMaster(psql_as_postgres("REVOKE EXECUTE ON FUNCTION func_stat_wal_receiver() FROM public;")) runOnMaster(psql_as_postgres("GRANT EXECUTE ON FUNCTION func_stat_wal_receiver() to %s;" % roUserName)) if readOnlyUserType == "uinode": uiBoosterTables = ("aps_resource", "aps_property_value", "aps_resource_link", "aps_application", "aps_property_info", "aps_package", "aps_relation_info", "aps_relation_types", "aps_type_info", "aps_type_inheritance", "aps_package_series", "aps_package_service", "aps_property_enum_info", "aps_type_info_to_package", "aps_operation_param", "aps_operation_info") runOnMaster(""" su - postgres -c "psql --port=%d --dbname=%s -c \\"GRANT SELECT ON TABLE %s TO %s\\"" """ % (masterPort, databaseName, ",".join(uiBoosterTables), roUserName)) else: runOnMaster(""" su - postgres -c "psql --port=%d --dbname=%s -c \\"GRANT SELECT ON ALL TABLES IN SCHEMA public TO %s\\"" """ % (masterPort, databaseName, roUserName)) uLogging.info("Read-only user has been created.") _tunePgHba(runOnMaster, "hostssl", "replication", replUserName, slaveCommunicationIP, pgsqlOnMaster.get_pghba_conf()) if int(getWalKeepSegments(runOnMaster, masterPort)) != 16384: runOnMaster(""" sed -i '/^[ \t]*wal_keep_segments[ \t]*=.*/d' "%s" """ % (pgsqlOnMaster.get_postgresql_conf(),)) runOnMaster(""" sed -i -e '$,+0a\wal_keep_segments = 16384' "%s" """ % (pgsqlOnMaster.get_postgresql_conf(),)) #For more details see the following KB: https://kb.cloudblue.com/en/115916 #Chain called Postgres could be absent if KB is not applied, so that we have to add that rules only in case if KB applied if runOnMaster(""" iptables -nL Postgres 2> /dev/null || echo -n """): uLogging.info("Configuring iptables for replication access") iptablesConfigAllowDb(run = runOnMaster, slaveCommunicationIP= slaveCommunicationIP, masterPort=masterPort) uLogging.info("Configuring iptables on master done!") if pghaSettings.isHa: pghaSlaveAddr = pghaSettings.bDbNode if masterAddr == pghaSettings.aDbNode else pghaSettings.aDbNode uLogging.info("Configuring iptables for replication access on PGHA slave '%s'" % pghaSlaveAddr) runOnPghaSlave = uUtil.getSSHRemoteRunner(pghaSlaveAddr, masterRootPwd) # providing of password is an extra measure since SSH certificates are distributed iptablesConfigAllowDb(run = runOnPghaSlave, slaveCommunicationIP= slaveCommunicationIP, masterPort=masterPort) uLogging.info("Configuring iptables o PGHA slave done!") pgsqlOnMaster.reload() uLogging.info("Replication connection has been enabled!") if pghaSettings.isHa: forceHaMasterSyncConf(runOnMaster) uLogging.info("Setting up initial database replication...") cleanPgCertificate(pgsqlOnSlave.get_data_dir(), runOnSlave) # clean certificate if exists baseBackupCmd = """ su - postgres -c 'PGPASSWORD=%s "%s/pg_basebackup" -X stream --host=%s --port=%s "--pgdata=%s" "--username=%s" --write-recovery-conf --checkpoint=fast' """ % (replUserPwd, pgsqlOnSlave.get_bin_dir(), targetReplicationSourceMasterAddr, str(masterPort), pgsqlOnSlave.get_data_dir(), replUserName) pgsqlOnSlave.cleanup() #targeting errors like f.e. this-> ERROR: could not open file "./pg_hba.conf.bak": Permission denied runOnMaster(""" chown -R postgres:postgres "%s" """ % (pgsqlOnMaster.get_data_dir(),)) runOnSlave(baseBackupCmd) uLogging.info("Initial database replication has been done!") uLogging.info("Doing post-configuration...") dotPostgresDir = os.path.dirname(os.path.dirname(pgsqlOnSlave.get_data_dir().rstrip("/"))) + "/.postgresql" runOnSlave(""" su - postgres -c 'mkdir -p "%s"' """ % (dotPostgresDir,)) runOnSlave(""" su - postgres -c 'cp -f "%s/%s" "%s/%s"' """ % (pgsqlOnSlave.get_data_dir(), "server.crt", dotPostgresDir, "root.crt")) for i, pf in enumerate(slavePersonalFilesBu): runOnSlave(""" su - postgres -c 'mv -f "%s" "%s/"' """ % (pf, os.path.dirname(slavePersonalFiles[i]))) uLinux.tunePostgresParamsImpl(pgsqlOnSlave) runOnSlave("sed -i -E 's|(.*[ \\t]+sslmode[ \\t]*=[ \\t]*)prefer([ \\t]+.*)|\\1verify-ca\\2|g' \"%s/recovery.conf\" " % (pgsqlOnSlave.get_data_dir().rstrip("/"),)) #marking server as a hot standby runOnSlave(""" sed -i '/^[ \t]*hot_standby[ \t]*=.*/d' "%s" """ % (pgsqlOnSlave.get_postgresql_conf(),)) runOnSlave(""" sed -i -e '$,+0a\\hot_standby = on' "%s" """ % (pgsqlOnSlave.get_postgresql_conf(),)) if additionalIPs is not None: for ip in additionalIPs: ipEsc = ip.replace(".", "\\.") runOnSlave(""" sed -i -e '$,+0a\hostssl all all %s\/32 md5' "%s" """ % (ipEsc, pgsqlOnSlave.get_pghba_conf())) runOnSlave(""" sed -i '/^listen_addresses/s/\*/127.0.0.1/g' "%s" """ % (pgsqlOnSlave.get_postgresql_conf(),)) uLogging.info("Post-configuration has been done!") uLogging.info("Starting new slave database server!") pgsqlOnSlave.restart() pgsqlOnSlave.set_autostart() waitSlaveRecoveryComplete(runOnSlave) # make sure recovery stage is complete uLinux.tunePostgresLogs(runOnSlave) pgsqlOnSlave.reload() uLogging.info("New slave database server has started!") if slaveScript: uLogging.info("Running post configuration script on slave: %s", slaveScript) cbCmd = """python "%s" connect_slave "%s" "%s" "%s" "%s" """ % (slaveScript, slaveCommunicationIP, databaseName, roUserName, roUserPwd) for a in slaveScriptArgs: cbCmd = cbCmd + ' "%s" ' % a runOnSlave(cbCmd) uLogging.info("Post configuration has been done!") rv = DeployPgSlaveResult() rv.replUserName = replUserName rv.roUserName = roUserName rv.masterHostID = masterHostID rv.masterAddr = masterAddr return rv