def get_download_url_from_build_prod(self, product_version, gppkg):
# defaults to be 4.2
gpdb_version = '4.2'
if product_version.startswith('4.3'):
gpdb_version = '4.3'
orca = ""
if product_version >= '4.3.5':
orca = 'orca'
os_, platform_ = self.get_os_platform()
compatiable = self.check_os_compatibility(os_, gppkg)
if not compatiable:
logger.error("the package %s is not compatiable with the os %s, please make sure the compatiable package exists" %(gppkg, os_))
return(-1, None, None)
build_prod_host = self.DEFAULT_BUILD_PROD_URL
gppkg_config = self.getconfig(product_version=gpdb_version, gppkg=gppkg)
gppkg_config['pkg'] = gppkg
gppkg_config['gpdbversion'] = gpdb_version
gppkg_config['os'] = self.failover_gppkg_to_os_version(os_, gppkg)
gppkg_config['platform'] = platform_
gppkg_config['type'] = 'gppkg'
gppkg_config['orca'] = orca
#GPDB 4.2 and 4.3 is having different nameing format for gppkg
if 'gpdbversion' in gppkg_config and 'ossversion' in gppkg_config:
gppkg_name = "%(pkg)s-ossv%(ossversion)s_pv%(version)s_gpdb%(gpdbversion)s%(orca)s-%(os)s-%(platform)s.%(type)s" % gppkg_config
elif gpdb_version == '4.3':
gppkg_name = "%(pkg)s-pv%(version)s_gpdb%(gpdbversion)s%(orca)s-%(os)s-%(platform)s.%(type)s" % gppkg_config
else:
gppkg_name = "%(pkg)s-%(version)s-%(os)s-%(platform)s.%(type)s" % gppkg_config
download_url = build_prod_host + '/gppkg/%(pkg)s/'%gppkg_config + gppkg_name
return (0, download_url, gppkg_name)
def get_download_url_from_build_prod(self, product_version, gppkg):
# defaults to be 4.2
gpdb_version = '4.2'
if product_version.startswith('4.3'):
gpdb_version = '4.3'
orca = ""
if product_version >= '4.3.5':
orca = 'orca'
os_, platform_ = self.get_os_platform()
compatiable = self.check_os_compatibility(os_, gppkg)
if not compatiable:
logger.error(
"the package %s is not compatiable with the os %s, please make sure the compatiable package exists"
% (gppkg, os_))
return (-1, None, None)
build_prod_host = self.DEFAULT_BUILD_PROD_URL
gppkg_config = self.getconfig(product_version=gpdb_version,
gppkg=gppkg)
gppkg_config['pkg'] = gppkg
gppkg_config['gpdbversion'] = gpdb_version
gppkg_config['os'] = self.failover_gppkg_to_os_version(os_, gppkg)
gppkg_config['platform'] = platform_
gppkg_config['type'] = 'gppkg'
gppkg_config['orca'] = orca
#GPDB 4.2 and 4.3 is having different nameing format for gppkg
if 'gpdbversion' in gppkg_config and 'ossversion' in gppkg_config:
gppkg_name = "%(pkg)s-ossv%(ossversion)s_pv%(version)s_gpdb%(gpdbversion)s%(orca)s-%(os)s-%(platform)s.%(type)s" % gppkg_config
elif gpdb_version == '4.3':
gppkg_name = "%(pkg)s-pv%(version)s_gpdb%(gpdbversion)s%(orca)s-%(os)s-%(platform)s.%(type)s" % gppkg_config
else:
gppkg_name = "%(pkg)s-%(version)s-%(os)s-%(platform)s.%(type)s" % gppkg_config
download_url = build_prod_host + '/gppkg/%(pkg)s/' % gppkg_config + gppkg_name
return (0, download_url, gppkg_name)
def get_download_url_from_build_prod(self, product_version, gppkg):
# defaults to be 4.2
gpdb_version = '4.2'
if product_version.startswith('4.3'):
gpdb_version = '4.3'
orca = ""
try:
minor_version = float(
re.compile('.*\d+\.\d+\.(\d+\.\d+)').match(
product_version).group(1))
if minor_version >= float(
5.0
): #minor version grabbed from 4.3.5.0 when orca was introduced
orca = 'orca'
except Exception as e:
logger.error("%s" % str(e))
raise Exception('Unable to parse product_version: %s' %
product_version)
os_, platform_ = self.get_os_platform()
compatiable = self.check_os_compatibility(os_, gppkg)
if not compatiable:
logger.error(
"the package %s is not compatiable with the os %s, please make sure the compatiable package exists"
% (gppkg, os_))
return (-1, None, None)
build_prod_host = self.DEFAULT_BUILD_PROD_URL
gppkg_config = self.getconfig(product_version=gpdb_version,
gppkg=gppkg)
gppkg_config['pkg'] = gppkg
gppkg_config['gpdbversion'] = gpdb_version
gppkg_config['os'] = self.failover_gppkg_to_os_version(os_, gppkg)
gppkg_config['platform'] = platform_
gppkg_config['type'] = 'gppkg'
gppkg_config['orca'] = orca
#GPDB 4.2 and 4.3 is having different nameing format for gppkg
if 'gpdbversion' in gppkg_config and 'ossversion' in gppkg_config:
gppkg_name = "%(pkg)s-ossv%(ossversion)s_pv%(version)s_gpdb%(gpdbversion)s%(orca)s-%(os)s-%(platform)s.%(type)s" % gppkg_config
elif gpdb_version == '4.3':
gppkg_name = "%(pkg)s-pv%(version)s_gpdb%(gpdbversion)s%(orca)s-%(os)s-%(platform)s.%(type)s" % gppkg_config
else:
gppkg_name = "%(pkg)s-%(version)s-%(os)s-%(platform)s.%(type)s" % gppkg_config
download_url = build_prod_host + '/gppkg/%(pkg)s/' % gppkg_config + gppkg_name
return (0, download_url, gppkg_name)
def gpdb_functionality_validation(self):
"""
Verify that we can create a table, insert data,
select data and drop data.
"""
filename = self._get_absolute_filename('test_basic_gpdb_functionality.sql')
logger.info('Validating that Greenplum Database is still functional ...')
assert PSQL.run_sql_file(sql_file=filename, dbname=self.test_database)
answer_file = re.sub('sql$', 'ans', filename)
output_file = re.sub('sql$', 'out', filename)
if not Gpdiff.are_files_equal(output_file, answer_file):
logger.error('Could not validate gpdb functionality')
return False
return True
def test_pg_inherits(self):
"""
Change order of children in pg_inherits on segments. Alter should not
cause inconsistent OIDs.
"""
# Create paritioned table.
sql = local_path("create_part_table.sql")
out = local_path("create_part_table.out")
ans = local_path("create_part_table.ans")
PSQL.run_sql_file(sql, out)
assert Gpdiff.are_files_equal(out, ans)
# Change order of children in pg_inherits on segments but not
# on master.
sql = local_path("reorder_pg_inherits.sql")
out = local_path("reorder_pg_inherits.out")
ans = local_path("reorder_pg_inherits.ans")
segments = [
seg for seg in self.gparray.getSegDbList() if seg.role == "p"
]
assert len(segments) > 0, "No primary segments found."
primary = segments[0]
PSQL.run_sql_file(sql,
out,
host=primary.hostname,
port=primary.port,
PGOPTIONS=("-c allow_system_table_mods=dml "
"-c gp_session_role=utility"))
assert Gpdiff.are_files_equal(out, ans)
# Alter the partitioned table so that it's rewritten.
with dbconn.connect(dbconn.DbURL()) as conn:
dbconn.execSQL(conn, "ALTER TABLE co1 ALTER COLUMN c2 TYPE int8")
conn.commit()
# Run gpcheckcat
result = GpdbVerify().gpcheckcat(testname="inconsistent")
# Test return code
if result[0] != 0:
logger.error(result[2]) # log output
self.fail("gpcheckcat 'inconsistent' test failed")
def run_data_validation(self):
"""
Validate data by executing a SQL file and comparing results
with the answer file.
"""
filename = self._get_absolute_filename(self.select_file)
logger.info("Validating data using '%s' ..." % filename)
if not PSQL.run_sql_file(sql_file=filename, dbname=self.dbname):
raise Exception("failed querying data pre-expansion: '%s'" % filename)
if not filename.endswith('sql'):
raise Exception("The filename must end in .sql extension")
answer_file = re.sub('sql$', 'ans', filename)
output_file = re.sub('sql$', 'out', filename)
if not Gpdiff.are_files_equal(output_file, answer_file):
logger.error("files don't match pre-expansion: '%s' and '%s'" % (answer_file, output_file))
return False
return True
def check_random_dist_tuple_count_skew(self, tname, values):
"""
max - min should not exceed 5% of the Maximum number of tuples.
@return: False if there is any error
True otherwise
"""
if not values:
return True
max_tuple_count, min_tuple_count = max(values), min(values)
diff = max_tuple_count - min_tuple_count
pct = float(diff) / float(max_tuple_count) * 100.0
if pct > 5:
logger.error("MAX (%d) MIN (%d) DIFF (%d) PCT(%f)" % (max_tuple_count, min_tuple_count, diff, pct))
return False
logger.info("OK: Table (%s) Max (%d) Min (%d) tuples per segdb" % (tname, max_tuple_count, min_tuple_count))
return True
def test_pg_inherits(self):
"""
Change order of children in pg_inherits on segments. Alter should not
cause inconsistent OIDs.
"""
# Create paritioned table.
sql = local_path("create_part_table.sql")
out = local_path("create_part_table.out")
ans = local_path("create_part_table.ans")
PSQL.run_sql_file(sql, out)
assert Gpdiff.are_files_equal(out, ans)
# Change order of children in pg_inherits on segments but not
# on master.
sql = local_path("reorder_pg_inherits.sql")
out = local_path("reorder_pg_inherits.out")
ans = local_path("reorder_pg_inherits.ans")
segments = [seg for seg in self.gparray.getSegDbList()
if seg.role == "p"]
assert len(segments) > 0, "No primary segments found."
primary = segments[0]
PSQL.run_sql_file(
sql, out, host=primary.hostname, port=primary.port,
PGOPTIONS=("-c allow_system_table_mods=dml "
"-c gp_session_role=utility"))
assert Gpdiff.are_files_equal(out, ans)
# Alter the partitioned table so that it's rewritten.
with dbconn.connect(dbconn.DbURL()) as conn:
dbconn.execSQL(conn, "ALTER TABLE co1 ALTER COLUMN c2 TYPE int8")
conn.commit()
# Run gpcheckcat
result = GpdbVerify().gpcheckcat(testname="inconsistent")
# Test return code
if result[0] != 0:
logger.error(result[2]) # log output
self.fail("gpcheckcat 'inconsistent' test failed")
def get_download_url_from_build_prod(self, product_version, gppkg):
# defaults to be 4.2
gpdb_version = '4.2'
if product_version.startswith('4.3'):
gpdb_version = '4.3'
orca = ""
try:
minor_version = float(re.compile('.*\d+\.\d+\.(\d+\.\d+)').match(product_version).group(1))
if minor_version >= float(5.0): #minor version grabbed from 4.3.5.0 when orca was introduced
orca = 'orca'
except Exception as e:
logger.error("%s" % str(e))
raise Exception('Unable to parse product_version: %s' % product_version)
os_, platform_ = self.get_os_platform()
compatiable = self.check_os_compatibility(os_, gppkg)
if not compatiable:
logger.error("the package %s is not compatiable with the os %s, please make sure the compatiable package exists" %(gppkg, os_))
return(-1, None, None)
build_prod_host = self.DEFAULT_BUILD_PROD_URL
gppkg_config = self.getconfig(product_version=gpdb_version, gppkg=gppkg)
gppkg_config['pkg'] = gppkg
gppkg_config['gpdbversion'] = gpdb_version
gppkg_config['os'] = self.failover_gppkg_to_os_version(os_, gppkg)
gppkg_config['platform'] = platform_
gppkg_config['type'] = 'gppkg'
gppkg_config['orca'] = orca
#GPDB 4.2 and 4.3 is having different nameing format for gppkg
if 'gpdbversion' in gppkg_config and 'ossversion' in gppkg_config:
gppkg_name = "%(pkg)s-ossv%(ossversion)s_pv%(version)s_gpdb%(gpdbversion)s%(orca)s-%(os)s-%(platform)s.%(type)s" % gppkg_config
elif gpdb_version == '4.3':
gppkg_name = "%(pkg)s-pv%(version)s_gpdb%(gpdbversion)s%(orca)s-%(os)s-%(platform)s.%(type)s" % gppkg_config
else:
gppkg_name = "%(pkg)s-%(version)s-%(os)s-%(platform)s.%(type)s" % gppkg_config
download_url = build_prod_host + '/gppkg/%(pkg)s/'%gppkg_config + gppkg_name
return (0, download_url, gppkg_name)
def catalog_validation(self):
"""
Validate that there are no inconsistencies in the catalog
@return: True if there are no inconsistencies
False otherwise
"""
logger.info("Running gpcheckcat to validate catalog ...")
# Fetch the count of databases using gpcheckcat that pass the catalog check test
out_file = self._get_absolute_filename('gpcheckcat.out')
assert self.db_port is not None
cmd_str = '$GPHOME/bin/lib/gpcheckcat -A -O -p %s &> %s' % (self.db_port, out_file)
cmd = Command('run gpcheckcat', cmd_str)
cmd.run(validateAfter=True)
line_no = 0
with open(out_file) as fp:
for line in fp:
if 'Found no catalog issue' in line:
line_no += 1
count_db = 0
# fetch the database count on the host using pg_catalog
with dbconn.connect(dbconn.DbURL()) as conn:
row = dbconn.execSQLForSingleton(conn, "select count(*) from pg_database")
# -1 because gpcheckcat does not run against template0
count_db = row - 1
# Check if the numbers match else expansion dint go through fine return false
if line_no != count_db:
failed_dbs = self._get_checkcat_failed_dbs(out_file)
logger.error('gpcheckcat failed for the following databases %s' % failed_dbs)
return False
return True
def test_xlogPreparedXactSeg(self):
"""
Test to verify the xlog on segment gets cleaned-up only till point of oldest prepared transaction.
The flow of this test is as follows.
1. Initiate the Standby using the Master (primary) postmaster
paramerters.
2. A: Inject the fault to suspend Mater after Prepare done.
3. A: Now execute a transaction and commit it. This transaction will be blocked.
4. B: Inject the fault to suspend Mater after Commit done.
5. B: Now execute a transaction and commit it. This master will be blocked.
6. Promote the standby.
7. Verify the result, transaction A results should not be visible and
transaction B results should be visible.
"""
PSQL.run_sql_command('DROP table if exists xansrep_prepare')
PSQL.run_sql_command(
'DROP table if exists xansrep1, xansrep2, xansrep3, xansrep4')
fault = Gpfault()
# 2. Inject fault at prepared state
result = fault.suspend_at(
'transaction_abort_after_distributed_prepared')
logger.info(result.stdout)
self.assertEqual(result.rc, 0, result.stdout)
# 3. Now execute a transaction and commit it. The backend is expected
# be blocked.
logger.info('Create table xansrep_prepare...')
create_xansprep_prepare_query = 'create table xansprep_prepare (a int)'
# Due to the suspend, we don't wait for the result
subprocess.Popen([
'psql', '-c', create_xansprep_prepare_query, '-p',
os.environ.get('PGPORT')
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
logger.info('Check if suspend fault is hit ...')
if not self.check_pg_stat_activity(create_xansprep_prepare_query):
logger.error('transaction has not been started yet')
triggered = fault.wait_triggered(
'transaction_abort_after_distributed_prepared')
self.assertTrue(triggered, 'Fault was not triggered')
# Lets trigger switch to xlog on segment
PSQL.run_sql_command_utility_mode(sql_cmd='select pg_switch_xlog()',
port=2100)
PSQL.run_sql_command_utility_mode(sql_cmd='checkpoint', port=2100)
# Generate more records on xlog
PSQL.run_sql_command('create table xansrep1 (a int)')
PSQL.run_sql_command('create table xansrep2 (a int)')
PSQL.run_sql_command_utility_mode(sql_cmd='select pg_switch_xlog()',
port=2100)
PSQL.run_sql_command_utility_mode(sql_cmd='checkpoint', port=2100)
PSQL.run_sql_command('create table xansrep3 (a int)')
PSQL.run_sql_command('create table xansrep4 (a int)')
PSQL.run_sql_command_utility_mode(sql_cmd='select pg_switch_xlog()',
port=2100)
PSQL.run_sql_command_utility_mode(sql_cmd='checkpoint', port=2100)
cmd = Command('gpstop', 'gpstop -air')
cmd.run(validateAfter=True)
def test_xlogPreparedXactSeg(self):
"""
Test to verify the xlog on segment gets cleaned-up only till point of oldest prepared transaction.
The flow of this test is as follows.
1. Initiate the Standby using the Master (primary) postmaster
paramerters.
2. A: Inject the fault to suspend Mater after Prepare done.
3. A: Now execute a transaction and commit it. This transaction will be blocked.
4. B: Inject the fault to suspend Mater after Commit done.
5. B: Now execute a transaction and commit it. This master will be blocked.
6. Promote the standby.
7. Verify the result, transaction A results should not be visible and
transaction B results should be visible.
"""
PSQL.run_sql_command('DROP table if exists xansrep_prepare')
PSQL.run_sql_command('DROP table if exists xansrep1, xansrep2, xansrep3, xansrep4')
fault = Gpfault()
# 2. Inject fault at prepared state
result = fault.suspend_at(
'transaction_abort_after_distributed_prepared')
logger.info(result.stdout)
self.assertEqual(result.rc, 0, result.stdout)
# 3. Now execute a transaction and commit it. The backend is expected
# be blocked.
logger.info('Create table xansrep_prepare...')
create_xansprep_prepare_query = 'create table xansprep_prepare (a int)'
# Due to the suspend, we don't wait for the result
subprocess.Popen(['psql',
'-c', create_xansprep_prepare_query,
'-p', os.environ.get('PGPORT')],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
logger.info('Check if suspend fault is hit ...')
if not self.check_pg_stat_activity(create_xansprep_prepare_query):
logger.error('transaction has not been started yet')
triggered = fault.wait_triggered(
'transaction_abort_after_distributed_prepared')
self.assertTrue(triggered, 'Fault was not triggered')
# Lets trigger switch to xlog on segment
PSQL.run_sql_command_utility_mode(sql_cmd='select pg_switch_xlog()', port=2100)
PSQL.run_sql_command_utility_mode(sql_cmd='checkpoint', port=2100)
# Generate more records on xlog
PSQL.run_sql_command('create table xansrep1 (a int)')
PSQL.run_sql_command('create table xansrep2 (a int)')
PSQL.run_sql_command_utility_mode(sql_cmd='select pg_switch_xlog()', port=2100)
PSQL.run_sql_command_utility_mode(sql_cmd='checkpoint', port=2100)
PSQL.run_sql_command('create table xansrep3 (a int)')
PSQL.run_sql_command('create table xansrep4 (a int)')
PSQL.run_sql_command_utility_mode(sql_cmd='select pg_switch_xlog()', port=2100)
PSQL.run_sql_command_utility_mode(sql_cmd='checkpoint', port=2100)
cmd = Command('gpstop', 'gpstop -air')
cmd.run(validateAfter=True)
