def runHCLCmd(hostId, commandText):
uLogging.debug(commandText)
con = uSysDB.connect()
host = uPEM.getHost(hostId)
commandText = commandText.replace(
"${", '$${'
) #for more details see following issue https://jira.int.zone/browse/POA-109131
rq = Request(user='******', group='root')
rq.command(commandText,
stdout='stdout',
stderr='stderr',
valid_exit_codes=[0])
if Const.isOsaWinPlatform(host.platform.os):
rq.export("default_shell_name", "cmd.exe", True)
rq.export("shell_cmd_switch", "/C", True)
else:
rq.export("default_shell_name", "/bin/sh", True)
rq.export("shell_cmd_switch", "-c", True)
rqRv = None
try:
rqRv = rq.send(host)
except uUtil.ExecFailed:
rqRv = rq.performRaw(host)
o = rqRv["stdout"]
if o: uLogging.debug(o)
return o
def perform(self, host_id=None):
host_id = host_id or self.__host_id
if host_id is None:
raise Exception("host_id not defined")
con = uSysDB.connect()
host = uPEM.getHost(host_id)
return self.performRaw(host)
def performCompat(self, host_id=None):
host_id = host_id or self.__host_id
if host_id is None:
raise Exception("host_id not defined")
con = uSysDB.connect()
host = uPEM.getHost(host_id)
try:
return self.performRaw(host)
except uUtil.ExecFailed:
return self.send(host)
def send(self, host):
from poaupdater import uConfig
status_code = -1
resp_out = ""
if self.__issuer is None:
self.__issuer = getHostCertificateDigest(uPEM.getHost(1))
config = uConfig.Config()
authToken = getHostAuthToken(host, config, self.__issuer)
host_ip = uPEM.getHostCommunicationIP(host.host_id)
uLogging.debug("Send REST HCL request to host %s (%s) ..." %
(str(host.host_id), host_ip))
# uLogging.debug("%s" % self.toxml())
headers = {
"Authorization": authToken,
"X-Auth-Host%s" % str(host.host_id): authToken
}
for d_host_id in self.__delegated_hosts:
if d_host_id != host.host_id:
headers["X-Auth-Host%s" % str(d_host_id)] = getHostAuthToken(
uPEM.getHost(d_host_id), config, self.__issuer)
http_req = urllib2.Request("https://%s:8352/process" % host_ip,
headers=headers,
data=None)
http_req.get_method = lambda: 'POST'
try:
if uLinux.SSL_VERIFICATION_ENFORCED:
resp = urllib2.urlopen(
http_req,
context=ssl._create_unverified_context(),
data=self.toxml())
else:
resp = urllib2.urlopen(http_req, data=self.toxml())
resp_out = resp.read()
try:
status_code = resp.getcode()
except AttributeError:
status_code = resp.code
except urllib2.HTTPError, error:
status_code = error.code
resp_out = error.read()
def removePgSlave(slaveHostID, masterRootPwd):
if not uLogging.logfile:
uLogging.init2("/var/log/pa/deregister_slave.log", True, False)
slave = uPEM.getHost(slaveHostID)
slaveCommunicationIP = uPEM.getHostCommunicationIP(slaveHostID)
runOnSlave = lambda cmd: uHCL.runHCLCmd(slaveHostID, cmd)
uLogging.info("Slave database server at %s (%s) is going to be removed.", slaveCommunicationIP, slave.name)
pghaSettings = getPghaSettings()
pgsqlOnSlave = None
try:
pgsqlOnSlave = uPgSQL.PostgreSQLConfig(commander = runOnSlave)
except Exception, e:
uLogging.info("Could not find slave database server at %s (%s): %s", slaveCommunicationIP, slave.name, str(e))
return
def upgrade_paagent_and_repourl(binfo, config, slave_hosts):
config.need_update_paagent = is_need_to_update_paagent(binfo)
if not config.need_update_paagent and not config.need_update_yum_repourl:
uLogging.info("No new agent RPMs in build, updating YUM repo URL is not needed also, skipping agent update.")
return
uLogging.debug(
"Need update YUM repo URL: {need_update_yum_repourl}; "
"there are new agents in build: {need_update_paagent}".format(
need_update_yum_repourl=config.need_update_yum_repourl, need_update_paagent=config.need_update_paagent))
uAction.progress.do("Updating agents on slave nodes")
config.mn_as_cert_issuer = uHCL.getHostCertificateDigest(uPEM.getHost(1))
# Filtering out non-upgradable slaves
slaves = [host for host in slave_hosts if host.host_id != 1]
uLogging.debug("Found slaves with agent: \n{slaves}".format(slaves=pformat(slaves)))
slaves = filter(lambda slave: is_slave_upgradable(slave, binfo, config), slaves)
uLogging.debug("All non-upgradable slaves are filtered out, actual slave-to-upgrade list now: "
"\n{slaves}".format(slaves=pformat(slaves)))
if not slaves:
uLogging.info("There is no slaves marked for update.")
uAction.progress.done()
return
# Preparing task pool for updating.
pool = preparePool(config)
slave_upgrade_monitoring = SlaveUpgradeMonitoring(pool, binfo, slaves)
# Filling task pool for slaves
for host in slaves:
try:
# Defining paagent url. If this will be eq None, package will not be updated
paagent_url = prepare_paagent_url(host.platform, binfo, config)
uLogging.debug("Slave '{host}' upgrade scheduled. Pa-agent URL is: '{agent_url}'"
.format(host=host, agent_url=paagent_url))
pool.put(TaskItem(host, paagent_url))
except uAction.ActionIgnored:
continue
thread_count = max_slave_upgrade_threads(hosts_to_upgrade_count=len(slaves),
slave_upgrade_threads=config.slave_upgrade_threads)
try:
pool.start(thread_count)
slave_upgrade_monitoring.start()
uLogging.info("Upgrade for agents on slaves started (parallel: {thread_count})".format(
thread_count=thread_count))
uAction.retriable(process_pool_results)(pool, binfo, config)
uLogging.info("All results of upgrading agents on slaves processed. ")
report_skipped_hosts(binfo)
except (Exception, KeyboardInterrupt) as e:
# to remember and raise what is happened under try statement if finally statement has failures too
uLogging.save_traceback()
pool.terminateLocked()
raise e
finally:
pool.terminate()
slave_upgrade_monitoring.terminate()
uAction.progress.done()
def deployPgSlave(slaveHostID, isBillingMaster, masterRootPwd, readOnlyUserType, additionalIPs, slaveScript, slaveScriptArgs):
if not uLogging.logfile:
uLogging.init2("/var/log/pa/register_slave.log", True, False)
uLogging.info("Deploying PostgreSQL slave server on PA service node #%d...", slaveHostID)
masterHostID = 0
pghaSettings = getPghaSettings()
if not isBillingMaster:
if slaveHostID == 1:
raise Exception("The target slave host is MN: no possibility to use MN node as a database replica.")
row = _getPgDbInfo()
databaseName = row[2]
if pghaSettings.isHa:
masterAddr = getHaMasterAddr(pghaSettings)
masterPort = pghaSettings.haBackendPort
targetReplicationSourceMasterAddr = pghaSettings.vip_2
else:
masterAddr = row[0]
masterPort = int(row[1])
targetReplicationSourceMasterAddr = masterAddr
uLogging.info("Master DB location: '%s at %d'" % (masterAddr, masterPort))
runOnMaster = _getCommandRunner(masterAddr, masterRootPwd)
if not runOnMaster.isLocal:
uLogging.info("Master is automation database server running remotely at %s:%d.", masterAddr, masterPort)
else:
uLogging.info("Master is automation database server running locally at %s:%d.", masterAddr, masterPort)
masterHostID = 1
else:
if slaveHostID in (b.get_host_id() for b in uBilling.get_billing_hosts()):
raise Exception("The target slave host is billing node: no possibility to use billing node as a database slave.")
dbParams = uBilling.PBAConf.getBillingDBPrams()
masterAddr = uBilling.PBAConf.getBillingDBHost()
masterPort = int(uBilling.PBAConf.getBillingDBPort())
databaseName = uBilling.PBAConf.getBillingDBName()
runOnMaster = _getCommandRunner(masterAddr, masterRootPwd)
targetReplicationSourceMasterAddr = masterAddr
uLogging.info("Master is billing database server running at %s:%d.", masterAddr, masterPort)
masterHostID = None
isPermitted = False
slave = uPEM.getHost(slaveHostID)
if not runOnMaster.isLocal:
try:
runCheck = lambda cmd: uUtil.runLocalCmd(cmd)
checkHostPermittedToBeReplicaOfDB(runCheck, slave.name)
isPermitted = True
except:
pass
if not isPermitted:
checkHostPermittedToBeReplicaOfDB(runOnMaster, slave.name)
slaveCommunicationIP = uPEM.getHostCommunicationIP(slaveHostID)
ipAddrJoined = ipAddrToUserUniqPostfix(slaveCommunicationIP)
replUserName = "******"+ipAddrJoined
replUserPwd = uUtil.generate_random_password(16)
runOnSlave = lambda cmd: uHCL.runHCLCmd(slaveHostID, cmd)
uLogging.info("Slave database server is going to be deployed at %s (%s)", slaveCommunicationIP, slave.name)
pgsqlOnMaster = uPgSQL.PostgreSQLConfig(commander = runOnMaster)
pgsqlVer = str(pgsqlOnMaster.get_version_as_int())
uLogging.info("Current running PostgreSQL version is '%s'" % pgsqlVer)
verifyPostgresCertificate(pgsqlOnMaster)
uLogging.info("Instaling PostgreSQL Server on the slave...")
runOnSlave("yum install -y odin-perftools postgresql%s postgresql%s-server postgresql%s-contrib" % (pgsqlVer, pgsqlVer, pgsqlVer))
runOnSlave("yum reinstall -y odin-perftools postgresql%s postgresql%s-server postgresql%s-contrib" % (pgsqlVer, pgsqlVer, pgsqlVer))
uLogging.info("Installation has finished!")
uLogging.info("Initializing database on slave...")
pgsqlOnSlave = uPgSQL.PostgreSQLConfig(commander = runOnSlave)
pgsqlOnSlave.cleanup()
pgsqlOnSlave.init_db()
uLinux.configureDatabaseImpl(pgsqlOnSlave, None, [])
uLogging.info("Saving some slave personal configuration files...")
slavePersonalFilesBu = []
slavePersonalFiles = (
pgsqlOnSlave.get_data_dir()+"/server.key",
pgsqlOnSlave.get_data_dir()+"/server.crt",
# pgsqlOnSlave.get_postgresql_conf(),
pgsqlOnSlave.get_pghba_conf()
)
slavePersonalDir = os.path.dirname(pgsqlOnSlave.get_data_dir().rstrip("/"))
for pf in slavePersonalFiles:
runOnSlave(""" su - postgres -c 'cp -f "%s" "%s/"' """ % (pf, slavePersonalDir))
slavePersonalFilesBu.append(os.path.join(slavePersonalDir, os.path.basename(pf)))
pgsqlOnSlave.stop()
uLogging.info("Database has been initialized!")
uLogging.info("Enabling replication connection from slave to master...")
runOnMaster(""" su - postgres -c "psql --port=%d -c \\"DROP ROLE IF EXISTS %s\\"" """ % (masterPort, replUserName,))
runOnMaster(""" su - postgres -c "psql --port=%d -c \\"CREATE ROLE %s WITH REPLICATION ENCRYPTED PASSWORD '%s' LOGIN CONNECTION LIMIT 8\\"" """ % (masterPort, replUserName, replUserPwd))
uLogging.info("Creating read-only user and users to be replicated from master to slave for farther readonly use on the slave node.")
roUserName = "******"+ipAddrJoined
roUserPwd = uUtil.generate_random_password(32)
# Provide the reentrancy, make sure the database doesn't contain objects created by possible previous launches
runOnMaster(""" su - postgres -c "psql --port=%d --dbname=%s -c \\"REVOKE SELECT ON ALL TABLES IN SCHEMA public FROM %s\\"" 2> /dev/null || echo -n """ % (masterPort, databaseName, roUserName))
runOnMaster(""" su - postgres -c "psql --port=%d -c \\"REVOKE EXECUTE ON FUNCTION func_stat_wal_receiver() from %s\\"" 2> /dev/null || echo -n """ % (masterPort, roUserName,))
runOnMaster(""" su - postgres -c "psql --port=%d -c \\"DROP ROLE IF EXISTS %s\\"" """ % (masterPort, roUserName,))
runOnMaster(""" su - postgres -c "psql --port=%d -c \\"CREATE ROLE %s WITH ENCRYPTED PASSWORD '%s' LOGIN\\"" """ % (masterPort, roUserName, roUserPwd))
#add ability to monitor replication status for RO user
def psql_as_postgres(input):
return r'su - postgres -c "psql --port=%d -c \"%s\""' % (masterPort, input)
runOnMaster(psql_as_postgres("DROP FUNCTION IF EXISTS func_stat_wal_receiver();"))
psql11_fields = []
if LooseVersion(pgsqlVer) >= LooseVersion("11"):
psql11_fields = ["cast('' as text) as sender_host", "-1 as sender_port"]
pg_stat_wal_receiver_sql = 'CREATE FUNCTION func_stat_wal_receiver() RETURNS SETOF pg_stat_wal_receiver as '
columns_to_select = ", ".join(["pid", "status", "receive_start_lsn", "receive_start_tli", "received_lsn", "received_tli",
"last_msg_send_time", "last_msg_receipt_time", "latest_end_lsn", "latest_end_time",
"cast('' as text) as slot_name"] +
psql11_fields +
["cast('' as text) as conninfo"])
pg_stat_wal_receiver_sql += r'\\$\\$ select %s from pg_stat_wal_receiver; \\$\\$ LANGUAGE sql SECURITY DEFINER;' % columns_to_select
runOnMaster(psql_as_postgres(pg_stat_wal_receiver_sql))
runOnMaster(psql_as_postgres("REVOKE EXECUTE ON FUNCTION func_stat_wal_receiver() FROM public;"))
runOnMaster(psql_as_postgres("GRANT EXECUTE ON FUNCTION func_stat_wal_receiver() to %s;" % roUserName))
if readOnlyUserType == "uinode":
uiBoosterTables = ("aps_resource", "aps_property_value", "aps_resource_link", "aps_application", "aps_property_info",
"aps_package", "aps_relation_info", "aps_relation_types", "aps_type_info", "aps_type_inheritance", "aps_package_series", "aps_package_service",
"aps_property_enum_info", "aps_type_info_to_package",
"aps_operation_param", "aps_operation_info")
runOnMaster(""" su - postgres -c "psql --port=%d --dbname=%s -c \\"GRANT SELECT ON TABLE %s TO %s\\"" """ % (masterPort, databaseName, ",".join(uiBoosterTables), roUserName))
else:
runOnMaster(""" su - postgres -c "psql --port=%d --dbname=%s -c \\"GRANT SELECT ON ALL TABLES IN SCHEMA public TO %s\\"" """ % (masterPort, databaseName, roUserName))
uLogging.info("Read-only user has been created.")
_tunePgHba(runOnMaster, "hostssl", "replication", replUserName, slaveCommunicationIP, pgsqlOnMaster.get_pghba_conf())
if int(getWalKeepSegments(runOnMaster, masterPort)) != 16384:
runOnMaster(""" sed -i '/^[ \t]*wal_keep_segments[ \t]*=.*/d' "%s" """ % (pgsqlOnMaster.get_postgresql_conf(),))
runOnMaster(""" sed -i -e '$,+0a\wal_keep_segments = 16384' "%s" """ % (pgsqlOnMaster.get_postgresql_conf(),))
#For more details see the following KB: https://kb.cloudblue.com/en/115916
#Chain called Postgres could be absent if KB is not applied, so that we have to add that rules only in case if KB applied
if runOnMaster(""" iptables -nL Postgres 2> /dev/null || echo -n """):
uLogging.info("Configuring iptables for replication access")
iptablesConfigAllowDb(run = runOnMaster, slaveCommunicationIP= slaveCommunicationIP, masterPort=masterPort)
uLogging.info("Configuring iptables on master done!")
if pghaSettings.isHa:
pghaSlaveAddr = pghaSettings.bDbNode if masterAddr == pghaSettings.aDbNode else pghaSettings.aDbNode
uLogging.info("Configuring iptables for replication access on PGHA slave '%s'" % pghaSlaveAddr)
runOnPghaSlave = uUtil.getSSHRemoteRunner(pghaSlaveAddr, masterRootPwd) # providing of password is an extra measure since SSH certificates are distributed
iptablesConfigAllowDb(run = runOnPghaSlave, slaveCommunicationIP= slaveCommunicationIP, masterPort=masterPort)
uLogging.info("Configuring iptables o PGHA slave done!")
pgsqlOnMaster.reload()
uLogging.info("Replication connection has been enabled!")
if pghaSettings.isHa:
forceHaMasterSyncConf(runOnMaster)
uLogging.info("Setting up initial database replication...")
cleanPgCertificate(pgsqlOnSlave.get_data_dir(), runOnSlave) # clean certificate if exists
baseBackupCmd = """ su - postgres -c 'PGPASSWORD=%s "%s/pg_basebackup" -X stream --host=%s --port=%s
"--pgdata=%s" "--username=%s" --write-recovery-conf --checkpoint=fast' """ % (replUserPwd, pgsqlOnSlave.get_bin_dir(), targetReplicationSourceMasterAddr, str(masterPort), pgsqlOnSlave.get_data_dir(), replUserName)
pgsqlOnSlave.cleanup()
#targeting errors like f.e. this-> ERROR: could not open file "./pg_hba.conf.bak": Permission denied
runOnMaster(""" chown -R postgres:postgres "%s" """ % (pgsqlOnMaster.get_data_dir(),))
runOnSlave(baseBackupCmd)
uLogging.info("Initial database replication has been done!")
uLogging.info("Doing post-configuration...")
dotPostgresDir = os.path.dirname(os.path.dirname(pgsqlOnSlave.get_data_dir().rstrip("/"))) + "/.postgresql"
runOnSlave(""" su - postgres -c 'mkdir -p "%s"' """ % (dotPostgresDir,))
runOnSlave(""" su - postgres -c 'cp -f "%s/%s" "%s/%s"' """ % (pgsqlOnSlave.get_data_dir(), "server.crt", dotPostgresDir, "root.crt"))
for i, pf in enumerate(slavePersonalFilesBu):
runOnSlave(""" su - postgres -c 'mv -f "%s" "%s/"' """ % (pf, os.path.dirname(slavePersonalFiles[i])))
uLinux.tunePostgresParamsImpl(pgsqlOnSlave)
runOnSlave("sed -i -E 's|(.*[ \\t]+sslmode[ \\t]*=[ \\t]*)prefer([ \\t]+.*)|\\1verify-ca\\2|g' \"%s/recovery.conf\" " % (pgsqlOnSlave.get_data_dir().rstrip("/"),))
#marking server as a hot standby
runOnSlave(""" sed -i '/^[ \t]*hot_standby[ \t]*=.*/d' "%s" """ % (pgsqlOnSlave.get_postgresql_conf(),))
runOnSlave(""" sed -i -e '$,+0a\\hot_standby = on' "%s" """ % (pgsqlOnSlave.get_postgresql_conf(),))
if additionalIPs is not None:
for ip in additionalIPs:
ipEsc = ip.replace(".", "\\.")
runOnSlave(""" sed -i -e '$,+0a\hostssl all all %s\/32 md5' "%s" """ % (ipEsc, pgsqlOnSlave.get_pghba_conf()))
runOnSlave(""" sed -i '/^listen_addresses/s/\*/127.0.0.1/g' "%s" """ % (pgsqlOnSlave.get_postgresql_conf(),))
uLogging.info("Post-configuration has been done!")
uLogging.info("Starting new slave database server!")
pgsqlOnSlave.restart()
pgsqlOnSlave.set_autostart()
waitSlaveRecoveryComplete(runOnSlave) # make sure recovery stage is complete
uLinux.tunePostgresLogs(runOnSlave)
pgsqlOnSlave.reload()
uLogging.info("New slave database server has started!")
if slaveScript:
uLogging.info("Running post configuration script on slave: %s", slaveScript)
cbCmd = """python "%s" connect_slave "%s" "%s" "%s" "%s" """ % (slaveScript, slaveCommunicationIP, databaseName, roUserName, roUserPwd)
for a in slaveScriptArgs:
cbCmd = cbCmd + ' "%s" ' % a
runOnSlave(cbCmd)
uLogging.info("Post configuration has been done!")
rv = DeployPgSlaveResult()
rv.replUserName = replUserName
rv.roUserName = roUserName
rv.masterHostID = masterHostID
rv.masterAddr = masterAddr
return rv