def get_download_url_from_build_prod(self, product_version, gppkg): # defaults to be 4.2 gpdb_version = '4.2' if product_version.startswith('4.3'): gpdb_version = '4.3' orca = "" if product_version >= '4.3.5': orca = 'orca' os_, platform_ = self.get_os_platform() compatiable = self.check_os_compatibility(os_, gppkg) if not compatiable: logger.error("the package %s is not compatiable with the os %s, please make sure the compatiable package exists" %(gppkg, os_)) return(-1, None, None) build_prod_host = self.DEFAULT_BUILD_PROD_URL gppkg_config = self.getconfig(product_version=gpdb_version, gppkg=gppkg) gppkg_config['pkg'] = gppkg gppkg_config['gpdbversion'] = gpdb_version gppkg_config['os'] = self.failover_gppkg_to_os_version(os_, gppkg) gppkg_config['platform'] = platform_ gppkg_config['type'] = 'gppkg' gppkg_config['orca'] = orca #GPDB 4.2 and 4.3 is having different nameing format for gppkg if 'gpdbversion' in gppkg_config and 'ossversion' in gppkg_config: gppkg_name = "%(pkg)s-ossv%(ossversion)s_pv%(version)s_gpdb%(gpdbversion)s%(orca)s-%(os)s-%(platform)s.%(type)s" % gppkg_config elif gpdb_version == '4.3': gppkg_name = "%(pkg)s-pv%(version)s_gpdb%(gpdbversion)s%(orca)s-%(os)s-%(platform)s.%(type)s" % gppkg_config else: gppkg_name = "%(pkg)s-%(version)s-%(os)s-%(platform)s.%(type)s" % gppkg_config download_url = build_prod_host + '/gppkg/%(pkg)s/'%gppkg_config + gppkg_name return (0, download_url, gppkg_name)
def get_download_url_from_build_prod(self, product_version, gppkg): # defaults to be 4.2 gpdb_version = '4.2' if product_version.startswith('4.3'): gpdb_version = '4.3' orca = "" if product_version >= '4.3.5': orca = 'orca' os_, platform_ = self.get_os_platform() compatiable = self.check_os_compatibility(os_, gppkg) if not compatiable: logger.error( "the package %s is not compatiable with the os %s, please make sure the compatiable package exists" % (gppkg, os_)) return (-1, None, None) build_prod_host = self.DEFAULT_BUILD_PROD_URL gppkg_config = self.getconfig(product_version=gpdb_version, gppkg=gppkg) gppkg_config['pkg'] = gppkg gppkg_config['gpdbversion'] = gpdb_version gppkg_config['os'] = self.failover_gppkg_to_os_version(os_, gppkg) gppkg_config['platform'] = platform_ gppkg_config['type'] = 'gppkg' gppkg_config['orca'] = orca #GPDB 4.2 and 4.3 is having different nameing format for gppkg if 'gpdbversion' in gppkg_config and 'ossversion' in gppkg_config: gppkg_name = "%(pkg)s-ossv%(ossversion)s_pv%(version)s_gpdb%(gpdbversion)s%(orca)s-%(os)s-%(platform)s.%(type)s" % gppkg_config elif gpdb_version == '4.3': gppkg_name = "%(pkg)s-pv%(version)s_gpdb%(gpdbversion)s%(orca)s-%(os)s-%(platform)s.%(type)s" % gppkg_config else: gppkg_name = "%(pkg)s-%(version)s-%(os)s-%(platform)s.%(type)s" % gppkg_config download_url = build_prod_host + '/gppkg/%(pkg)s/' % gppkg_config + gppkg_name return (0, download_url, gppkg_name)
def get_download_url_from_build_prod(self, product_version, gppkg): # defaults to be 4.2 gpdb_version = '4.2' if product_version.startswith('4.3'): gpdb_version = '4.3' orca = "" try: minor_version = float( re.compile('.*\d+\.\d+\.(\d+\.\d+)').match( product_version).group(1)) if minor_version >= float( 5.0 ): #minor version grabbed from 4.3.5.0 when orca was introduced orca = 'orca' except Exception as e: logger.error("%s" % str(e)) raise Exception('Unable to parse product_version: %s' % product_version) os_, platform_ = self.get_os_platform() compatiable = self.check_os_compatibility(os_, gppkg) if not compatiable: logger.error( "the package %s is not compatiable with the os %s, please make sure the compatiable package exists" % (gppkg, os_)) return (-1, None, None) build_prod_host = self.DEFAULT_BUILD_PROD_URL gppkg_config = self.getconfig(product_version=gpdb_version, gppkg=gppkg) gppkg_config['pkg'] = gppkg gppkg_config['gpdbversion'] = gpdb_version gppkg_config['os'] = self.failover_gppkg_to_os_version(os_, gppkg) gppkg_config['platform'] = platform_ gppkg_config['type'] = 'gppkg' gppkg_config['orca'] = orca #GPDB 4.2 and 4.3 is having different nameing format for gppkg if 'gpdbversion' in gppkg_config and 'ossversion' in gppkg_config: gppkg_name = "%(pkg)s-ossv%(ossversion)s_pv%(version)s_gpdb%(gpdbversion)s%(orca)s-%(os)s-%(platform)s.%(type)s" % gppkg_config elif gpdb_version == '4.3': gppkg_name = "%(pkg)s-pv%(version)s_gpdb%(gpdbversion)s%(orca)s-%(os)s-%(platform)s.%(type)s" % gppkg_config else: gppkg_name = "%(pkg)s-%(version)s-%(os)s-%(platform)s.%(type)s" % gppkg_config download_url = build_prod_host + '/gppkg/%(pkg)s/' % gppkg_config + gppkg_name return (0, download_url, gppkg_name)
def gpdb_functionality_validation(self): """ Verify that we can create a table, insert data, select data and drop data. """ filename = self._get_absolute_filename('test_basic_gpdb_functionality.sql') logger.info('Validating that Greenplum Database is still functional ...') assert PSQL.run_sql_file(sql_file=filename, dbname=self.test_database) answer_file = re.sub('sql$', 'ans', filename) output_file = re.sub('sql$', 'out', filename) if not Gpdiff.are_files_equal(output_file, answer_file): logger.error('Could not validate gpdb functionality') return False return True
def test_pg_inherits(self): """ Change order of children in pg_inherits on segments. Alter should not cause inconsistent OIDs. """ # Create paritioned table. sql = local_path("create_part_table.sql") out = local_path("create_part_table.out") ans = local_path("create_part_table.ans") PSQL.run_sql_file(sql, out) assert Gpdiff.are_files_equal(out, ans) # Change order of children in pg_inherits on segments but not # on master. sql = local_path("reorder_pg_inherits.sql") out = local_path("reorder_pg_inherits.out") ans = local_path("reorder_pg_inherits.ans") segments = [ seg for seg in self.gparray.getSegDbList() if seg.role == "p" ] assert len(segments) > 0, "No primary segments found." primary = segments[0] PSQL.run_sql_file(sql, out, host=primary.hostname, port=primary.port, PGOPTIONS=("-c allow_system_table_mods=dml " "-c gp_session_role=utility")) assert Gpdiff.are_files_equal(out, ans) # Alter the partitioned table so that it's rewritten. with dbconn.connect(dbconn.DbURL()) as conn: dbconn.execSQL(conn, "ALTER TABLE co1 ALTER COLUMN c2 TYPE int8") conn.commit() # Run gpcheckcat result = GpdbVerify().gpcheckcat(testname="inconsistent") # Test return code if result[0] != 0: logger.error(result[2]) # log output self.fail("gpcheckcat 'inconsistent' test failed")
def run_data_validation(self): """ Validate data by executing a SQL file and comparing results with the answer file. """ filename = self._get_absolute_filename(self.select_file) logger.info("Validating data using '%s' ..." % filename) if not PSQL.run_sql_file(sql_file=filename, dbname=self.dbname): raise Exception("failed querying data pre-expansion: '%s'" % filename) if not filename.endswith('sql'): raise Exception("The filename must end in .sql extension") answer_file = re.sub('sql$', 'ans', filename) output_file = re.sub('sql$', 'out', filename) if not Gpdiff.are_files_equal(output_file, answer_file): logger.error("files don't match pre-expansion: '%s' and '%s'" % (answer_file, output_file)) return False return True
def check_random_dist_tuple_count_skew(self, tname, values): """ max - min should not exceed 5% of the Maximum number of tuples. @return: False if there is any error True otherwise """ if not values: return True max_tuple_count, min_tuple_count = max(values), min(values) diff = max_tuple_count - min_tuple_count pct = float(diff) / float(max_tuple_count) * 100.0 if pct > 5: logger.error("MAX (%d) MIN (%d) DIFF (%d) PCT(%f)" % (max_tuple_count, min_tuple_count, diff, pct)) return False logger.info("OK: Table (%s) Max (%d) Min (%d) tuples per segdb" % (tname, max_tuple_count, min_tuple_count)) return True
def test_pg_inherits(self): """ Change order of children in pg_inherits on segments. Alter should not cause inconsistent OIDs. """ # Create paritioned table. sql = local_path("create_part_table.sql") out = local_path("create_part_table.out") ans = local_path("create_part_table.ans") PSQL.run_sql_file(sql, out) assert Gpdiff.are_files_equal(out, ans) # Change order of children in pg_inherits on segments but not # on master. sql = local_path("reorder_pg_inherits.sql") out = local_path("reorder_pg_inherits.out") ans = local_path("reorder_pg_inherits.ans") segments = [seg for seg in self.gparray.getSegDbList() if seg.role == "p"] assert len(segments) > 0, "No primary segments found." primary = segments[0] PSQL.run_sql_file( sql, out, host=primary.hostname, port=primary.port, PGOPTIONS=("-c allow_system_table_mods=dml " "-c gp_session_role=utility")) assert Gpdiff.are_files_equal(out, ans) # Alter the partitioned table so that it's rewritten. with dbconn.connect(dbconn.DbURL()) as conn: dbconn.execSQL(conn, "ALTER TABLE co1 ALTER COLUMN c2 TYPE int8") conn.commit() # Run gpcheckcat result = GpdbVerify().gpcheckcat(testname="inconsistent") # Test return code if result[0] != 0: logger.error(result[2]) # log output self.fail("gpcheckcat 'inconsistent' test failed")
def get_download_url_from_build_prod(self, product_version, gppkg): # defaults to be 4.2 gpdb_version = '4.2' if product_version.startswith('4.3'): gpdb_version = '4.3' orca = "" try: minor_version = float(re.compile('.*\d+\.\d+\.(\d+\.\d+)').match(product_version).group(1)) if minor_version >= float(5.0): #minor version grabbed from 4.3.5.0 when orca was introduced orca = 'orca' except Exception as e: logger.error("%s" % str(e)) raise Exception('Unable to parse product_version: %s' % product_version) os_, platform_ = self.get_os_platform() compatiable = self.check_os_compatibility(os_, gppkg) if not compatiable: logger.error("the package %s is not compatiable with the os %s, please make sure the compatiable package exists" %(gppkg, os_)) return(-1, None, None) build_prod_host = self.DEFAULT_BUILD_PROD_URL gppkg_config = self.getconfig(product_version=gpdb_version, gppkg=gppkg) gppkg_config['pkg'] = gppkg gppkg_config['gpdbversion'] = gpdb_version gppkg_config['os'] = self.failover_gppkg_to_os_version(os_, gppkg) gppkg_config['platform'] = platform_ gppkg_config['type'] = 'gppkg' gppkg_config['orca'] = orca #GPDB 4.2 and 4.3 is having different nameing format for gppkg if 'gpdbversion' in gppkg_config and 'ossversion' in gppkg_config: gppkg_name = "%(pkg)s-ossv%(ossversion)s_pv%(version)s_gpdb%(gpdbversion)s%(orca)s-%(os)s-%(platform)s.%(type)s" % gppkg_config elif gpdb_version == '4.3': gppkg_name = "%(pkg)s-pv%(version)s_gpdb%(gpdbversion)s%(orca)s-%(os)s-%(platform)s.%(type)s" % gppkg_config else: gppkg_name = "%(pkg)s-%(version)s-%(os)s-%(platform)s.%(type)s" % gppkg_config download_url = build_prod_host + '/gppkg/%(pkg)s/'%gppkg_config + gppkg_name return (0, download_url, gppkg_name)
def catalog_validation(self): """ Validate that there are no inconsistencies in the catalog @return: True if there are no inconsistencies False otherwise """ logger.info("Running gpcheckcat to validate catalog ...") # Fetch the count of databases using gpcheckcat that pass the catalog check test out_file = self._get_absolute_filename('gpcheckcat.out') assert self.db_port is not None cmd_str = '$GPHOME/bin/lib/gpcheckcat -A -O -p %s &> %s' % (self.db_port, out_file) cmd = Command('run gpcheckcat', cmd_str) cmd.run(validateAfter=True) line_no = 0 with open(out_file) as fp: for line in fp: if 'Found no catalog issue' in line: line_no += 1 count_db = 0 # fetch the database count on the host using pg_catalog with dbconn.connect(dbconn.DbURL()) as conn: row = dbconn.execSQLForSingleton(conn, "select count(*) from pg_database") # -1 because gpcheckcat does not run against template0 count_db = row - 1 # Check if the numbers match else expansion dint go through fine return false if line_no != count_db: failed_dbs = self._get_checkcat_failed_dbs(out_file) logger.error('gpcheckcat failed for the following databases %s' % failed_dbs) return False return True
def test_xlogPreparedXactSeg(self): """ Test to verify the xlog on segment gets cleaned-up only till point of oldest prepared transaction. The flow of this test is as follows. 1. Initiate the Standby using the Master (primary) postmaster paramerters. 2. A: Inject the fault to suspend Mater after Prepare done. 3. A: Now execute a transaction and commit it. This transaction will be blocked. 4. B: Inject the fault to suspend Mater after Commit done. 5. B: Now execute a transaction and commit it. This master will be blocked. 6. Promote the standby. 7. Verify the result, transaction A results should not be visible and transaction B results should be visible. """ PSQL.run_sql_command('DROP table if exists xansrep_prepare') PSQL.run_sql_command( 'DROP table if exists xansrep1, xansrep2, xansrep3, xansrep4') fault = Gpfault() # 2. Inject fault at prepared state result = fault.suspend_at( 'transaction_abort_after_distributed_prepared') logger.info(result.stdout) self.assertEqual(result.rc, 0, result.stdout) # 3. Now execute a transaction and commit it. The backend is expected # be blocked. logger.info('Create table xansrep_prepare...') create_xansprep_prepare_query = 'create table xansprep_prepare (a int)' # Due to the suspend, we don't wait for the result subprocess.Popen([ 'psql', '-c', create_xansprep_prepare_query, '-p', os.environ.get('PGPORT') ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) logger.info('Check if suspend fault is hit ...') if not self.check_pg_stat_activity(create_xansprep_prepare_query): logger.error('transaction has not been started yet') triggered = fault.wait_triggered( 'transaction_abort_after_distributed_prepared') self.assertTrue(triggered, 'Fault was not triggered') # Lets trigger switch to xlog on segment PSQL.run_sql_command_utility_mode(sql_cmd='select pg_switch_xlog()', port=2100) PSQL.run_sql_command_utility_mode(sql_cmd='checkpoint', port=2100) # Generate more records on xlog PSQL.run_sql_command('create table xansrep1 (a int)') PSQL.run_sql_command('create table xansrep2 (a int)') PSQL.run_sql_command_utility_mode(sql_cmd='select pg_switch_xlog()', port=2100) PSQL.run_sql_command_utility_mode(sql_cmd='checkpoint', port=2100) PSQL.run_sql_command('create table xansrep3 (a int)') PSQL.run_sql_command('create table xansrep4 (a int)') PSQL.run_sql_command_utility_mode(sql_cmd='select pg_switch_xlog()', port=2100) PSQL.run_sql_command_utility_mode(sql_cmd='checkpoint', port=2100) cmd = Command('gpstop', 'gpstop -air') cmd.run(validateAfter=True)
def test_xlogPreparedXactSeg(self): """ Test to verify the xlog on segment gets cleaned-up only till point of oldest prepared transaction. The flow of this test is as follows. 1. Initiate the Standby using the Master (primary) postmaster paramerters. 2. A: Inject the fault to suspend Mater after Prepare done. 3. A: Now execute a transaction and commit it. This transaction will be blocked. 4. B: Inject the fault to suspend Mater after Commit done. 5. B: Now execute a transaction and commit it. This master will be blocked. 6. Promote the standby. 7. Verify the result, transaction A results should not be visible and transaction B results should be visible. """ PSQL.run_sql_command('DROP table if exists xansrep_prepare') PSQL.run_sql_command('DROP table if exists xansrep1, xansrep2, xansrep3, xansrep4') fault = Gpfault() # 2. Inject fault at prepared state result = fault.suspend_at( 'transaction_abort_after_distributed_prepared') logger.info(result.stdout) self.assertEqual(result.rc, 0, result.stdout) # 3. Now execute a transaction and commit it. The backend is expected # be blocked. logger.info('Create table xansrep_prepare...') create_xansprep_prepare_query = 'create table xansprep_prepare (a int)' # Due to the suspend, we don't wait for the result subprocess.Popen(['psql', '-c', create_xansprep_prepare_query, '-p', os.environ.get('PGPORT')], stdout=subprocess.PIPE, stderr=subprocess.PIPE) logger.info('Check if suspend fault is hit ...') if not self.check_pg_stat_activity(create_xansprep_prepare_query): logger.error('transaction has not been started yet') triggered = fault.wait_triggered( 'transaction_abort_after_distributed_prepared') self.assertTrue(triggered, 'Fault was not triggered') # Lets trigger switch to xlog on segment PSQL.run_sql_command_utility_mode(sql_cmd='select pg_switch_xlog()', port=2100) PSQL.run_sql_command_utility_mode(sql_cmd='checkpoint', port=2100) # Generate more records on xlog PSQL.run_sql_command('create table xansrep1 (a int)') PSQL.run_sql_command('create table xansrep2 (a int)') PSQL.run_sql_command_utility_mode(sql_cmd='select pg_switch_xlog()', port=2100) PSQL.run_sql_command_utility_mode(sql_cmd='checkpoint', port=2100) PSQL.run_sql_command('create table xansrep3 (a int)') PSQL.run_sql_command('create table xansrep4 (a int)') PSQL.run_sql_command_utility_mode(sql_cmd='select pg_switch_xlog()', port=2100) PSQL.run_sql_command_utility_mode(sql_cmd='checkpoint', port=2100) cmd = Command('gpstop', 'gpstop -air') cmd.run(validateAfter=True)