def query_data(self):
print "Testing Querying Data"
command = self.sql_command.format("select * from " + self.temp_table_name)
(retcode, out, err) = subprocess_command_with_results(self.test_user, command, self.active_master_host)
if retcode:
print_to_stderr_and_exit("Greenplum Smoke Tests Failed. Return Code: {0}. Out: {1} Error: {2}".format(retcode, out, err))
print out
def verify_segments_state(env, active_master_host):
import params
env.set_params(params)
command = "source /usr/local/greenplum-db/greenplum_path.sh; gpstate -d {0}/gpseg-1".format(
params.greenplum_master_dir
)
(retcode, out, err) = subprocess_command_with_results(params.greenplum_user, command, active_master_host)
if retcode:
print_to_stderr_and_exit(
"gpstate command returned non-zero result: {0}. Out: {1} Error: {2}".format(retcode, out, err)
)
Logger.info(
"Service check results:\nOutput of gpstate -d {0}\n".format(params.greenplum_master_data_dir) + str(out) + "\n"
)
if [
status_line
for status_line in out.split("\n")
if (status_line.startswith("gpseg") and status_line.split(" ")[1] == "d")
]:
print_to_stderr_and_exit(
"Service check detected that some of the Greenplum segments are down. run 'gpstate -t' on master for more info"
)
def insert_data(self):
print "Testing Insert Data"
command = self.sql_command.format("insert into " + self.temp_table_name + " select * from generate_series(1,10)")
(retcode, out, err) = subprocess_command_with_results(self.test_user, command, self.active_master_host)
if retcode:
print_to_stderr_and_exit("Greenplum Smoke Tests Failed. Return Code: {0}. Out: {1} Error: {2}".format(retcode, out, err))
print out
def create_table(self):
print "Testing Create Table"
command = self.sql_command.format("create table " + self.temp_table_name + "(a int)")
(retcode, out, err) = subprocess_command_with_results(self.test_user, command, self.active_master_host)
if retcode:
print_to_stderr_and_exit("Greenplum Smoke Tests Failed. Return Code: {0}. Out: {1} Error: {2}".format(retcode, out, err))
print out
def get_active_master_host():
"""
1. greenplum_master will always be the active master in greenplum cluster without
standby.
2. If cluster is configured with greenplum master & standby, but master data
directory does not exists on both the hosts, it suggests that greenplum database
is not initialized & cluster must be initialized with greenplum_master as the
active master.
"""
import params
active_master_host = None
if params.greenplum_standby is None or not is_datadir_existing_on_master_hosts(
):
active_master_host = params.greenplum_master
# If cluster has master and standby, ensure postmaster.opts exists
elif is_postmaster_opts_missing_on_master_hosts():
print_to_stderr_and_exit(
POSTMASTER_OPTS_MISSING.format(params.greenplum_master_data_dir))
else:
active_master_host = identify_active_master()
if active_master_host not in params.master_hosts:
print_to_stderr_and_exit(
"Host {0} not in the list of configured master hosts {1}. Please execute the requested operation from active greenplum master manually"
.format(active_master_host, " and ".join(params.master_hosts)))
return active_master_host
def check_data_correctness(self):
expected_data = "55"
print "Testing Correctness of Data. Finding sum of all the inserted entries. Expected output: " + expected_data
command = self.sql_noheader_command.format("select sum(a) from " + self.temp_table_name)
(retcode, out, err) = subprocess_command_with_results(self.test_user, command, self.active_master_host)
if retcode:
print_to_stderr_and_exit("Greenplum Smoke Tests Failed. Return Code: {0}. Out: {1} Error: {2}".format(retcode, out, err))
if expected_data != out.strip():
print_to_stderr_and_exit("Greenplum Smoke Tests Failed. Incorrect Data Returned. Expected Data: {0}. Acutal Data: {1} ".format(expected_data, out))
print out
def create_table(self):
print "Testing Create Table"
command = self.sql_command.format("create table " +
self.temp_table_name + "(a int)")
(retcode, out,
err) = subprocess_command_with_results(self.test_user, command,
self.active_master_host)
if retcode:
print_to_stderr_and_exit(
"Greenplum Smoke Tests Failed. Return Code: {0}. Out: {1} Error: {2}"
.format(retcode, out, err))
print out
def check_port_conflict():
import params
import subprocess
command = "netstat -tulpn | grep ':{0}\\b'".format(params.greenplum_master_port)
(stdoutdata, _) = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True).communicate()
if len(stdoutdata):
# we have a conflict with the greenplum master port.
message = "Conflict with Greenplum Master port. Either the service is already running or some other service is using port: {0}.\nProcess running on master port:\n{1}".format(
params.greenplum_master_port, stdoutdata
)
print_to_stderr_and_exit(message)
def query_data(self):
print "Testing Querying Data"
command = self.sql_command.format("select * from " +
self.temp_table_name)
(retcode, out,
err) = subprocess_command_with_results(self.test_user, command,
self.active_master_host)
if retcode:
print_to_stderr_and_exit(
"Greenplum Smoke Tests Failed. Return Code: {0}. Out: {1} Error: {2}"
.format(retcode, out, err))
print out
def check_port_conflict():
import params
import subprocess
command = "netstat -tulpn | grep ':{0}\\b'".format(
params.greenplum_master_port)
(stdoutdata, _) = subprocess.Popen(command,
stdout=subprocess.PIPE,
shell=True).communicate()
if (len(stdoutdata)):
# we have a conflict with the greenplum master port.
message = "Conflict with Greenplum Master port. Either the service is already running or some other service is using port: {0}.\nProcess running on master port:\n{1}".format(
params.greenplum_master_port, stdoutdata)
print_to_stderr_and_exit(message)
def insert_data(self):
print "Testing Insert Data"
command = self.sql_command.format(
"insert into " + self.temp_table_name +
" select * from generate_series(1,10)")
(retcode, out,
err) = subprocess_command_with_results(self.test_user, command,
self.active_master_host)
if retcode:
print_to_stderr_and_exit(
"Greenplum Smoke Tests Failed. Return Code: {0}. Out: {1} Error: {2}"
.format(retcode, out, err))
print out
def check_standby_activation_prereq():
"""
Check the state of greenplum standby master (i.e params.greenplum_standby), and return an exception if it does not appear to be in a valid state.
"""
import params
GPACTIVATESTANDBY_INSTRUCTIONS = "\nSteps to execute gpactivatestandby.\n1. ssh to the standby host\n2. Login as gpadmin user: su - gpadmin\n3. Execute the command: gpactivatestandby -a -f -d {0}".format(params.greenplum_master_data_dir)
if not params.greenplum_standby:
print_to_stderr_and_exit("Standby is not configured in this cluster, activate standby command is used in clusters which has standby configured.")
# Check if there is postgres process running on master port, if running it indicates that standby has already been activated to active master
command = "netstat -tulpn | grep ':{0}\s' | grep postgres".format(params.greenplum_master_port)
returncode, stdoutdata, _ = subprocess_command_with_results(params.root_user, command, params.hostname)
if returncode == 0:
print_to_stderr_and_exit("Active greenplum master process appears to be already running on host {0} on port {1}, standby activation is not required. Process details:\n{2}".format(params.greenplum_standby, params.greenplum_master_port, stdoutdata))
# Check if postmaster.opts file exists, it helps to identify if.greenplum_standby is acting as standby
if not os.path.isfile(params.postmaster_opts_filepath):
EXCEPTION_MSG = "{0} file does not exists on the host {1}, cannot continue with activating standby. Please activate standby manually from command line until its fixed.".format(params.postmaster_opts_filepath, params.hostname) + GPACTIVATESTANDBY_INSTRUCTIONS + "\nNote: postmaster.opts is created automatically during start of greenplum master."
print_to_stderr_and_exit(EXCEPTION_MSG)
# Read postmaster.opts file into a list
with open(params.postmaster_opts_filepath, "r") as fh:
postmaster_content = fh.read().split()
"""
Raise exception if postmaster.opts content indicate that this host is not configured as standby
Case 1. If contents of postmaster.opts has flag "-x", it indicates that it is not configured as a standby.
Example contents of standby postmaster.opts:
postgres "-D" "/data/greenplum/master/gpseg-1" "-p" "5432" "-b" "12" "-C" "-1" "-z" "10" "-i"
Case 2. If standby activation was triggered but it failed, content of postmaster.opts will include "-x" flag but with another option "gp_role=utility". In this case, we should allow gpactivatestandby to be executed again
/usr/local/greenplum-db/bin/postgres "-D" "/data/greenplum/master/gpseg-1" "-p" "5433" "-b" "1" "-z" "1" "--silent-mode=true" "-i" "-M" "master" "-C" "-1" "-x" "0" "-c" "gp_role=utility"
"""
if '"-x"' in postmaster_content and not '"gp_role=utility"' in postmaster_content:
print_to_stderr_and_exit("Contents of {0} on host {1} indicate that it is not currently acting as standby greenplum master.\nActivateStandby from UI can only be run if host {1} is acting as standby greenplum master. Please verify if you have already activated it to active greenplum master.\nIf host {2} is acting as standby greenplum master and needs to be activated, please execute gpactivatestandby manually from command line.".format(params.postmaster_opts_filepath, params.greenplum_standby, params.greenplum_master) + GPACTIVATESTANDBY_INSTRUCTIONS)
"""
If a lock file /tmp/.s.PGSQL.<master_port>.lock is found on.greenplum_master host, activate standby will fail stating that an active postgres process in running on it.
Ensure that gpactivatestandby is triggered only if the lock file is not existing. If not checked, gpactivatestandby will be executed and the contents
of postmaster.opts will get inconsistent impacting start / stop operation from ambari ui, so avoid such cases.
"""
lock_file_name = "/tmp/.s.PGSQL.{0}.lock".format(params.greenplum_master_port)
# ssh to the.greenplum_master and identify if file exists or not
if not active_master_helper.is_file_missing(params.greenplum_master, lock_file_name):
print_to_stderr_and_exit("Lock file /tmp/.s.PGSQL.{0}.lock exists on host {1} suggesting that active postgres process is running on it.\nIf greenplum master process is running on host {1}, please stop the database before retrying activate standby.\nIf greenplum master is not running on host {1}, please delete /tmp/.s.PGSQL.{0}, /tmp/.s.PGSQL.{0}.* and {2}/postmaster.pid files on it before retrying activate standby.".format(params.greenplum_master_port, params.greenplum_master, params.greenplum_master_data_dir))
def check_data_correctness(self):
expected_data = "55"
print "Testing Correctness of Data. Finding sum of all the inserted entries. Expected output: " + expected_data
command = self.sql_noheader_command.format("select sum(a) from " +
self.temp_table_name)
(retcode, out,
err) = subprocess_command_with_results(self.test_user, command,
self.active_master_host)
if retcode:
print_to_stderr_and_exit(
"Greenplum Smoke Tests Failed. Return Code: {0}. Out: {1} Error: {2}"
.format(retcode, out, err))
if expected_data != out.strip():
print_to_stderr_and_exit(
"Greenplum Smoke Tests Failed. Incorrect Data Returned. Expected Data: {0}. Acutal Data: {1} "
.format(expected_data, out))
print out
def verify_segments_state(env, active_master_host):
import params
env.set_params(params)
command = "source /usr/local/greenplum-db/greenplum_path.sh; gpstate -d {0}/gpseg-1".format(
params.greenplum_master_dir)
(retcode, out,
err) = subprocess_command_with_results(params.greenplum_user, command,
active_master_host)
if retcode:
print_to_stderr_and_exit(
"gpstate command returned non-zero result: {0}. Out: {1} Error: {2}"
.format(retcode, out, err))
Logger.info("Service check results:\nOutput of gpstate -d {0}\n".format(
params.greenplum_master_data_dir) + str(out) + "\n")
if [
status_line for status_line in out.split('\n') if
(status_line.startswith('gpseg') and status_line.split(" ")[1] == 'd')
]:
print_to_stderr_and_exit(
"Service check detected that some of the Greenplum segments are down. run 'gpstate -t' on master for more info"
)
def get_active_master_host():
"""
1. greenplum_master will always be the active master in greenplum cluster without
standby.
2. If cluster is configured with greenplum master & standby, but master data
directory does not exists on both the hosts, it suggests that greenplum database
is not initialized & cluster must be initialized with greenplum_master as the
active master.
"""
import params
active_master_host = None
if params.greenplum_standby is None or not is_datadir_existing_on_master_hosts():
active_master_host = params.greenplum_master
# If cluster has master and standby, ensure postmaster.opts exists
elif is_postmaster_opts_missing_on_master_hosts():
print_to_stderr_and_exit(POSTMASTER_OPTS_MISSING.format(params.greenplum_master_data_dir))
else:
active_master_host = identify_active_master()
if active_master_host not in params.master_hosts:
print_to_stderr_and_exit("Host {0} not in the list of configured master hosts {1}. Please execute the requested operation from active greenplum master manually".format(active_master_host, " and ".join(params.master_hosts)))
return active_master_host
def identify_active_master():
"""
Example contents of postmaster.opts in 2 different cases
Case 1: Master configured with Standby:
Contents on master postmaster.opts
postgres "-D" "/data/greenplum/master/gpseg-1" "-p" "5432" "-b" "1" "-z" "10" "--silent-mode=true" "-i" "-M" "master" "-C" "-1" "-x" "12" "-E"
Contents of standby postmaster.opts:
postgres "-D" "/data/greenplum/master/gpseg-1" "-p" "5432" "-b" "12" "-C" "-1" "-z" "10" "-i"
Case 2: Master configured without standby:
postgres "-D" "/data/greenplum/master/gpseg-1" "-p" "5432" "-b" "1" "-z" "10" "--silent-mode=true" "-i" "-M" "master" "-C" "-1" "-x" "0" "-E"
Interpretation:
postmaster.opts (if present) contains information for the segment startup parameters. Flag "-x" can indicate if the server is a master (with or without standby) or standby
-x = 0 : Master server (Standby is not configured)
-x = n : Master server (Standby is configured)
-x flag not available: Standby server
"""
import params
hostname = socket.gethostname()
master_postmaster_opts_content = convert_postmaster_content_to_list(
params.greenplum_master, params.postmaster_opts_filepath)
standby_postmaster_opts_content = convert_postmaster_content_to_list(
params.greenplum_standby, params.postmaster_opts_filepath)
if '"gp_role=utility"' in master_postmaster_opts_content or '"gp_role=utility"' in standby_postmaster_opts_content:
print_to_stderr_and_exit(
MASTER_STARTED_IN_UTILITY_MODE.format(
params.greenplum_master_data_dir))
_x_in_master_contents = '"-x"' in master_postmaster_opts_content
_x_in_standby_contents = '"-x"' in standby_postmaster_opts_content
if _x_in_master_contents and not _x_in_standby_contents:
return params.greenplum_master
if not _x_in_master_contents and _x_in_standby_contents:
return params.greenplum_standby
if _x_in_master_contents and _x_in_standby_contents:
return identify_active_master_by_timestamp(hostname)
"""
If control reaches here, it indicates that an active master cannot be identified. Return appropriate failure message
"""
if not _x_in_master_contents and not _x_in_standby_contents:
print_to_stderr_and_exit(
BOTH_MASTER_HAS_STANDBY_CONTENT.format(
params.greenplum_master_data_dir))
print_to_stderr_and_exit(
UNABLE_TO_IDENTIFY_ACTIVE_MASTER.format(
params.greenplum_master_data_dir))
def identify_active_master():
"""
Example contents of postmaster.opts in 2 different cases
Case 1: Master configured with Standby:
Contents on master postmaster.opts
postgres "-D" "/data/greenplum/master/gpseg-1" "-p" "5432" "-b" "1" "-z" "10" "--silent-mode=true" "-i" "-M" "master" "-C" "-1" "-x" "12" "-E"
Contents of standby postmaster.opts:
postgres "-D" "/data/greenplum/master/gpseg-1" "-p" "5432" "-b" "12" "-C" "-1" "-z" "10" "-i"
Case 2: Master configured without standby:
postgres "-D" "/data/greenplum/master/gpseg-1" "-p" "5432" "-b" "1" "-z" "10" "--silent-mode=true" "-i" "-M" "master" "-C" "-1" "-x" "0" "-E"
Interpretation:
postmaster.opts (if present) contains information for the segment startup parameters. Flag "-x" can indicate if the server is a master (with or without standby) or standby
-x = 0 : Master server (Standby is not configured)
-x = n : Master server (Standby is configured)
-x flag not available: Standby server
"""
import params
hostname = socket.gethostname()
master_postmaster_opts_content = convert_postmaster_content_to_list(params.greenplum_master, params.postmaster_opts_filepath)
standby_postmaster_opts_content = convert_postmaster_content_to_list(params.greenplum_standby, params.postmaster_opts_filepath)
if '"gp_role=utility"' in master_postmaster_opts_content or '"gp_role=utility"' in standby_postmaster_opts_content:
print_to_stderr_and_exit(MASTER_STARTED_IN_UTILITY_MODE.format(params.greenplum_master_data_dir))
_x_in_master_contents = '"-x"' in master_postmaster_opts_content
_x_in_standby_contents = '"-x"' in standby_postmaster_opts_content
if _x_in_master_contents and not _x_in_standby_contents:
return params.greenplum_master
if not _x_in_master_contents and _x_in_standby_contents:
return params.greenplum_standby
if _x_in_master_contents and _x_in_standby_contents:
return identify_active_master_by_timestamp(hostname)
"""
If control reaches here, it indicates that an active master cannot be identified. Return appropriate failure message
"""
if not _x_in_master_contents and not _x_in_standby_contents:
print_to_stderr_and_exit(BOTH_MASTER_HAS_STANDBY_CONTENT.format(params.greenplum_master_data_dir))
print_to_stderr_and_exit(UNABLE_TO_IDENTIFY_ACTIVE_MASTER.format(params.greenplum_master_data_dir))