def getHostAuthToken(host, config, issuer):
kernel_pkey = config.kernel_priv_key
pkey_formatted = "-----BEGIN RSA PRIVATE KEY-----\n"
i = 0
while i < len(kernel_pkey):
pkey_formatted += kernel_pkey[i:i + 64] + "\n"
i += 64
pkey_formatted += "-----END RSA PRIVATE KEY-----"
host_ip = uPEM.getHostCommunicationIP(host.host_id)
digest = getHostCertificateDigest(host)
jose = {"alg": "RS256", "typ": "JWT"}
iat = int(time.mktime(time.localtime()))
token = {
"iss": issuer,
"aud": digest,
"nbf": iat,
"iat": iat,
"exp": iat + 3600,
"hcl_host_ip": host_ip
}
# uLogging.debug("JOSE: " + json.dumps(jose))
# uLogging.debug("Token: " + json.dumps(token))
header = str(base64.urlsafe_b64encode(json.dumps(jose))).rstrip("=") + "." \
+ str(base64.urlsafe_b64encode(json.dumps(token))).rstrip("=")
signature = _get_key_fingerprint(pkey_formatted, header, host)
signature = signature.decode("hex")
return header + "." + str(base64.urlsafe_b64encode(signature)).rstrip("=")
def check_update_windows_slave(task_item, config):
uLogging.debug("Checking if agent on Windows host {host} is updated".format(host=task_item.host))
auth_token = uHCL.getHostAuthToken(task_item.host, config, config.mn_as_cert_issuer)
remote_temp_dir = get_remote_temp_dir(task_item.host)
remote_async_exec_path = get_remote_async_exec_path(task_item.host)
# Asynchronous upgrade. If installation has been failed then PAgent.exe will be run again for product check only.
# The following cases can be:
# 1. exit code is 1604, it means installation is OK (manual upgrade is performed).
# 2. exit code is 0 , it's OK too.
# 3. exit code is 1603 (returned by bootstrapper when 'CheckWithoutUpgrade' built-in property
# is given at the command line. It means that agent not upgraded.
check_result_request = uHCL.Request(host_id=task_item.host.host_id, auto_export=True)
check_cmd_args = [remote_async_exec_path,
'--status',
'--anchor ' + agent_anchor_name,
'"{paagent_exe} /c /qn REBOOT=Suppress AUTO_UPGRADE=1 CheckWithoutRunUpgrade=1"'.format(
paagent_exe=PAAGENT_EXE_FILENAME
),
'--exit-codes 0,1604']
check_result_request.command(remote_async_exec_path, check_cmd_args, cwd=remote_temp_dir,
stderr="err", stdout="out", retvar="exit_code")
host_ip = uPEM.getHostCommunicationIP(task_item.host.host_id)
check_result_http_request = urllib2.Request("https://{host_ip}:8352/process".format(host_ip=host_ip),
headers={"Authorization": auth_token}, data=None)
check_result_http_request.get_method = lambda: 'POST'
uLogging.debug("Checking update status of Windows slave {host} via REST HCL request to {host_ip}".format(
host=task_item.host, host_ip=host_ip))
try:
if uLinux.SSL_VERIFICATION_ENFORCED:
http_respond = urllib2.urlopen(check_result_http_request, context=ssl._create_unverified_context(),
data=check_result_request.toxml())
else:
http_respond = urllib2.urlopen(check_result_http_request, data=check_result_request.toxml())
http_respond_out = http_respond.read()
try:
status_code = http_respond.getcode()
except AttributeError:
status_code = http_respond.code
except urllib2.HTTPError, error:
status_code = error.code
http_respond_out = error.read()
def removePgSlave(slaveHostID, masterRootPwd):
if not uLogging.logfile:
uLogging.init2("/var/log/pa/deregister_slave.log", True, False)
slave = uPEM.getHost(slaveHostID)
slaveCommunicationIP = uPEM.getHostCommunicationIP(slaveHostID)
runOnSlave = lambda cmd: uHCL.runHCLCmd(slaveHostID, cmd)
uLogging.info("Slave database server at %s (%s) is going to be removed.", slaveCommunicationIP, slave.name)
pghaSettings = getPghaSettings()
pgsqlOnSlave = None
try:
pgsqlOnSlave = uPgSQL.PostgreSQLConfig(commander = runOnSlave)
except Exception, e:
uLogging.info("Could not find slave database server at %s (%s): %s", slaveCommunicationIP, slave.name, str(e))
return
def send(self, host):
from poaupdater import uConfig
status_code = -1
resp_out = ""
if self.__issuer is None:
self.__issuer = getHostCertificateDigest(uPEM.getHost(1))
config = uConfig.Config()
authToken = getHostAuthToken(host, config, self.__issuer)
host_ip = uPEM.getHostCommunicationIP(host.host_id)
uLogging.debug("Send REST HCL request to host %s (%s) ..." %
(str(host.host_id), host_ip))
# uLogging.debug("%s" % self.toxml())
headers = {
"Authorization": authToken,
"X-Auth-Host%s" % str(host.host_id): authToken
}
for d_host_id in self.__delegated_hosts:
if d_host_id != host.host_id:
headers["X-Auth-Host%s" % str(d_host_id)] = getHostAuthToken(
uPEM.getHost(d_host_id), config, self.__issuer)
http_req = urllib2.Request("https://%s:8352/process" % host_ip,
headers=headers,
data=None)
http_req.get_method = lambda: 'POST'
try:
if uLinux.SSL_VERIFICATION_ENFORCED:
resp = urllib2.urlopen(
http_req,
context=ssl._create_unverified_context(),
data=self.toxml())
else:
resp = urllib2.urlopen(http_req, data=self.toxml())
resp_out = resp.read()
try:
status_code = resp.getcode()
except AttributeError:
status_code = resp.code
except urllib2.HTTPError, error:
status_code = error.code
resp_out = error.read()
def deployPgSlave(slaveHostID, isBillingMaster, masterRootPwd, readOnlyUserType, additionalIPs, slaveScript, slaveScriptArgs):
if not uLogging.logfile:
uLogging.init2("/var/log/pa/register_slave.log", True, False)
uLogging.info("Deploying PostgreSQL slave server on PA service node #%d...", slaveHostID)
masterHostID = 0
pghaSettings = getPghaSettings()
if not isBillingMaster:
if slaveHostID == 1:
raise Exception("The target slave host is MN: no possibility to use MN node as a database replica.")
row = _getPgDbInfo()
databaseName = row[2]
if pghaSettings.isHa:
masterAddr = getHaMasterAddr(pghaSettings)
masterPort = pghaSettings.haBackendPort
targetReplicationSourceMasterAddr = pghaSettings.vip_2
else:
masterAddr = row[0]
masterPort = int(row[1])
targetReplicationSourceMasterAddr = masterAddr
uLogging.info("Master DB location: '%s at %d'" % (masterAddr, masterPort))
runOnMaster = _getCommandRunner(masterAddr, masterRootPwd)
if not runOnMaster.isLocal:
uLogging.info("Master is automation database server running remotely at %s:%d.", masterAddr, masterPort)
else:
uLogging.info("Master is automation database server running locally at %s:%d.", masterAddr, masterPort)
masterHostID = 1
else:
if slaveHostID in (b.get_host_id() for b in uBilling.get_billing_hosts()):
raise Exception("The target slave host is billing node: no possibility to use billing node as a database slave.")
dbParams = uBilling.PBAConf.getBillingDBPrams()
masterAddr = uBilling.PBAConf.getBillingDBHost()
masterPort = int(uBilling.PBAConf.getBillingDBPort())
databaseName = uBilling.PBAConf.getBillingDBName()
runOnMaster = _getCommandRunner(masterAddr, masterRootPwd)
targetReplicationSourceMasterAddr = masterAddr
uLogging.info("Master is billing database server running at %s:%d.", masterAddr, masterPort)
masterHostID = None
isPermitted = False
slave = uPEM.getHost(slaveHostID)
if not runOnMaster.isLocal:
try:
runCheck = lambda cmd: uUtil.runLocalCmd(cmd)
checkHostPermittedToBeReplicaOfDB(runCheck, slave.name)
isPermitted = True
except:
pass
if not isPermitted:
checkHostPermittedToBeReplicaOfDB(runOnMaster, slave.name)
slaveCommunicationIP = uPEM.getHostCommunicationIP(slaveHostID)
ipAddrJoined = ipAddrToUserUniqPostfix(slaveCommunicationIP)
replUserName = "******"+ipAddrJoined
replUserPwd = uUtil.generate_random_password(16)
runOnSlave = lambda cmd: uHCL.runHCLCmd(slaveHostID, cmd)
uLogging.info("Slave database server is going to be deployed at %s (%s)", slaveCommunicationIP, slave.name)
pgsqlOnMaster = uPgSQL.PostgreSQLConfig(commander = runOnMaster)
pgsqlVer = str(pgsqlOnMaster.get_version_as_int())
uLogging.info("Current running PostgreSQL version is '%s'" % pgsqlVer)
verifyPostgresCertificate(pgsqlOnMaster)
uLogging.info("Instaling PostgreSQL Server on the slave...")
runOnSlave("yum install -y odin-perftools postgresql%s postgresql%s-server postgresql%s-contrib" % (pgsqlVer, pgsqlVer, pgsqlVer))
runOnSlave("yum reinstall -y odin-perftools postgresql%s postgresql%s-server postgresql%s-contrib" % (pgsqlVer, pgsqlVer, pgsqlVer))
uLogging.info("Installation has finished!")
uLogging.info("Initializing database on slave...")
pgsqlOnSlave = uPgSQL.PostgreSQLConfig(commander = runOnSlave)
pgsqlOnSlave.cleanup()
pgsqlOnSlave.init_db()
uLinux.configureDatabaseImpl(pgsqlOnSlave, None, [])
uLogging.info("Saving some slave personal configuration files...")
slavePersonalFilesBu = []
slavePersonalFiles = (
pgsqlOnSlave.get_data_dir()+"/server.key",
pgsqlOnSlave.get_data_dir()+"/server.crt",
# pgsqlOnSlave.get_postgresql_conf(),
pgsqlOnSlave.get_pghba_conf()
)
slavePersonalDir = os.path.dirname(pgsqlOnSlave.get_data_dir().rstrip("/"))
for pf in slavePersonalFiles:
runOnSlave(""" su - postgres -c 'cp -f "%s" "%s/"' """ % (pf, slavePersonalDir))
slavePersonalFilesBu.append(os.path.join(slavePersonalDir, os.path.basename(pf)))
pgsqlOnSlave.stop()
uLogging.info("Database has been initialized!")
uLogging.info("Enabling replication connection from slave to master...")
runOnMaster(""" su - postgres -c "psql --port=%d -c \\"DROP ROLE IF EXISTS %s\\"" """ % (masterPort, replUserName,))
runOnMaster(""" su - postgres -c "psql --port=%d -c \\"CREATE ROLE %s WITH REPLICATION ENCRYPTED PASSWORD '%s' LOGIN CONNECTION LIMIT 8\\"" """ % (masterPort, replUserName, replUserPwd))
uLogging.info("Creating read-only user and users to be replicated from master to slave for farther readonly use on the slave node.")
roUserName = "******"+ipAddrJoined
roUserPwd = uUtil.generate_random_password(32)
# Provide the reentrancy, make sure the database doesn't contain objects created by possible previous launches
runOnMaster(""" su - postgres -c "psql --port=%d --dbname=%s -c \\"REVOKE SELECT ON ALL TABLES IN SCHEMA public FROM %s\\"" 2> /dev/null || echo -n """ % (masterPort, databaseName, roUserName))
runOnMaster(""" su - postgres -c "psql --port=%d -c \\"REVOKE EXECUTE ON FUNCTION func_stat_wal_receiver() from %s\\"" 2> /dev/null || echo -n """ % (masterPort, roUserName,))
runOnMaster(""" su - postgres -c "psql --port=%d -c \\"DROP ROLE IF EXISTS %s\\"" """ % (masterPort, roUserName,))
runOnMaster(""" su - postgres -c "psql --port=%d -c \\"CREATE ROLE %s WITH ENCRYPTED PASSWORD '%s' LOGIN\\"" """ % (masterPort, roUserName, roUserPwd))
#add ability to monitor replication status for RO user
def psql_as_postgres(input):
return r'su - postgres -c "psql --port=%d -c \"%s\""' % (masterPort, input)
runOnMaster(psql_as_postgres("DROP FUNCTION IF EXISTS func_stat_wal_receiver();"))
psql11_fields = []
if LooseVersion(pgsqlVer) >= LooseVersion("11"):
psql11_fields = ["cast('' as text) as sender_host", "-1 as sender_port"]
pg_stat_wal_receiver_sql = 'CREATE FUNCTION func_stat_wal_receiver() RETURNS SETOF pg_stat_wal_receiver as '
columns_to_select = ", ".join(["pid", "status", "receive_start_lsn", "receive_start_tli", "received_lsn", "received_tli",
"last_msg_send_time", "last_msg_receipt_time", "latest_end_lsn", "latest_end_time",
"cast('' as text) as slot_name"] +
psql11_fields +
["cast('' as text) as conninfo"])
pg_stat_wal_receiver_sql += r'\\$\\$ select %s from pg_stat_wal_receiver; \\$\\$ LANGUAGE sql SECURITY DEFINER;' % columns_to_select
runOnMaster(psql_as_postgres(pg_stat_wal_receiver_sql))
runOnMaster(psql_as_postgres("REVOKE EXECUTE ON FUNCTION func_stat_wal_receiver() FROM public;"))
runOnMaster(psql_as_postgres("GRANT EXECUTE ON FUNCTION func_stat_wal_receiver() to %s;" % roUserName))
if readOnlyUserType == "uinode":
uiBoosterTables = ("aps_resource", "aps_property_value", "aps_resource_link", "aps_application", "aps_property_info",
"aps_package", "aps_relation_info", "aps_relation_types", "aps_type_info", "aps_type_inheritance", "aps_package_series", "aps_package_service",
"aps_property_enum_info", "aps_type_info_to_package",
"aps_operation_param", "aps_operation_info")
runOnMaster(""" su - postgres -c "psql --port=%d --dbname=%s -c \\"GRANT SELECT ON TABLE %s TO %s\\"" """ % (masterPort, databaseName, ",".join(uiBoosterTables), roUserName))
else:
runOnMaster(""" su - postgres -c "psql --port=%d --dbname=%s -c \\"GRANT SELECT ON ALL TABLES IN SCHEMA public TO %s\\"" """ % (masterPort, databaseName, roUserName))
uLogging.info("Read-only user has been created.")
_tunePgHba(runOnMaster, "hostssl", "replication", replUserName, slaveCommunicationIP, pgsqlOnMaster.get_pghba_conf())
if int(getWalKeepSegments(runOnMaster, masterPort)) != 16384:
runOnMaster(""" sed -i '/^[ \t]*wal_keep_segments[ \t]*=.*/d' "%s" """ % (pgsqlOnMaster.get_postgresql_conf(),))
runOnMaster(""" sed -i -e '$,+0a\wal_keep_segments = 16384' "%s" """ % (pgsqlOnMaster.get_postgresql_conf(),))
#For more details see the following KB: https://kb.cloudblue.com/en/115916
#Chain called Postgres could be absent if KB is not applied, so that we have to add that rules only in case if KB applied
if runOnMaster(""" iptables -nL Postgres 2> /dev/null || echo -n """):
uLogging.info("Configuring iptables for replication access")
iptablesConfigAllowDb(run = runOnMaster, slaveCommunicationIP= slaveCommunicationIP, masterPort=masterPort)
uLogging.info("Configuring iptables on master done!")
if pghaSettings.isHa:
pghaSlaveAddr = pghaSettings.bDbNode if masterAddr == pghaSettings.aDbNode else pghaSettings.aDbNode
uLogging.info("Configuring iptables for replication access on PGHA slave '%s'" % pghaSlaveAddr)
runOnPghaSlave = uUtil.getSSHRemoteRunner(pghaSlaveAddr, masterRootPwd) # providing of password is an extra measure since SSH certificates are distributed
iptablesConfigAllowDb(run = runOnPghaSlave, slaveCommunicationIP= slaveCommunicationIP, masterPort=masterPort)
uLogging.info("Configuring iptables o PGHA slave done!")
pgsqlOnMaster.reload()
uLogging.info("Replication connection has been enabled!")
if pghaSettings.isHa:
forceHaMasterSyncConf(runOnMaster)
uLogging.info("Setting up initial database replication...")
cleanPgCertificate(pgsqlOnSlave.get_data_dir(), runOnSlave) # clean certificate if exists
baseBackupCmd = """ su - postgres -c 'PGPASSWORD=%s "%s/pg_basebackup" -X stream --host=%s --port=%s
"--pgdata=%s" "--username=%s" --write-recovery-conf --checkpoint=fast' """ % (replUserPwd, pgsqlOnSlave.get_bin_dir(), targetReplicationSourceMasterAddr, str(masterPort), pgsqlOnSlave.get_data_dir(), replUserName)
pgsqlOnSlave.cleanup()
#targeting errors like f.e. this-> ERROR: could not open file "./pg_hba.conf.bak": Permission denied
runOnMaster(""" chown -R postgres:postgres "%s" """ % (pgsqlOnMaster.get_data_dir(),))
runOnSlave(baseBackupCmd)
uLogging.info("Initial database replication has been done!")
uLogging.info("Doing post-configuration...")
dotPostgresDir = os.path.dirname(os.path.dirname(pgsqlOnSlave.get_data_dir().rstrip("/"))) + "/.postgresql"
runOnSlave(""" su - postgres -c 'mkdir -p "%s"' """ % (dotPostgresDir,))
runOnSlave(""" su - postgres -c 'cp -f "%s/%s" "%s/%s"' """ % (pgsqlOnSlave.get_data_dir(), "server.crt", dotPostgresDir, "root.crt"))
for i, pf in enumerate(slavePersonalFilesBu):
runOnSlave(""" su - postgres -c 'mv -f "%s" "%s/"' """ % (pf, os.path.dirname(slavePersonalFiles[i])))
uLinux.tunePostgresParamsImpl(pgsqlOnSlave)
runOnSlave("sed -i -E 's|(.*[ \\t]+sslmode[ \\t]*=[ \\t]*)prefer([ \\t]+.*)|\\1verify-ca\\2|g' \"%s/recovery.conf\" " % (pgsqlOnSlave.get_data_dir().rstrip("/"),))
#marking server as a hot standby
runOnSlave(""" sed -i '/^[ \t]*hot_standby[ \t]*=.*/d' "%s" """ % (pgsqlOnSlave.get_postgresql_conf(),))
runOnSlave(""" sed -i -e '$,+0a\\hot_standby = on' "%s" """ % (pgsqlOnSlave.get_postgresql_conf(),))
if additionalIPs is not None:
for ip in additionalIPs:
ipEsc = ip.replace(".", "\\.")
runOnSlave(""" sed -i -e '$,+0a\hostssl all all %s\/32 md5' "%s" """ % (ipEsc, pgsqlOnSlave.get_pghba_conf()))
runOnSlave(""" sed -i '/^listen_addresses/s/\*/127.0.0.1/g' "%s" """ % (pgsqlOnSlave.get_postgresql_conf(),))
uLogging.info("Post-configuration has been done!")
uLogging.info("Starting new slave database server!")
pgsqlOnSlave.restart()
pgsqlOnSlave.set_autostart()
waitSlaveRecoveryComplete(runOnSlave) # make sure recovery stage is complete
uLinux.tunePostgresLogs(runOnSlave)
pgsqlOnSlave.reload()
uLogging.info("New slave database server has started!")
if slaveScript:
uLogging.info("Running post configuration script on slave: %s", slaveScript)
cbCmd = """python "%s" connect_slave "%s" "%s" "%s" "%s" """ % (slaveScript, slaveCommunicationIP, databaseName, roUserName, roUserPwd)
for a in slaveScriptArgs:
cbCmd = cbCmd + ' "%s" ' % a
runOnSlave(cbCmd)
uLogging.info("Post configuration has been done!")
rv = DeployPgSlaveResult()
rv.replUserName = replUserName
rv.roUserName = roUserName
rv.masterHostID = masterHostID
rv.masterAddr = masterAddr
return rv
k, v = kv.split("=", 1)
conInfoDict[k.strip()] = v.strip()
break
runOnMaster = None
recoveryHost = conInfoDict["host"]
if pghaSettings.isHa:
masterAddr = getHaMasterAddr(pghaSettings)
masterPort = pghaSettings.haBackendPort
else:
masterAddr = recoveryHost
masterPort = int(conInfoDict["port"])
if recoveryHost != uPEM.getHostCommunicationIP(1): #MN node?
billingHostID = None
runOnMaster = None
else:
uLogging.info("Master is automation database server running locally at %s:%d.", masterAddr, masterPort)
runOnMaster = lambda cmd: uUtil.runLocalCmd(cmd)
if runOnMaster is None: #Master is running as an external database server
uLogging.info("Master is automation database server running remotely at %s:%d.", masterAddr, masterPort)
runOnMaster = uUtil.getSSHRemoteRunner(masterAddr, masterRootPwd)
pgsqlOnMaster = uPgSQL.PostgreSQLConfig(commander = runOnMaster)
replUserName = conInfoDict["user"]
uLogging.info("Disabling replication connection from slave to master...")
ipEscpd = slaveCommunicationIP.replace(".", "\\.")
runOnMaster(""" sed -i '/^[ \t]*hostssl[ \t]\+replication[ \t]\\+%s[ \t]\+%s\/32[ \t]\+md5[ \t]*/d' "%s" """ % (replUserName, ipEscpd, pgsqlOnMaster.get_pghba_conf()))
runOnMaster(""" su - postgres -c "psql --port=%d -c \\"DROP ROLE IF EXISTS %s\\"" """ % (masterPort, replUserName,))