def copy_files_to_segments(filename, location): config = GPDBConfig() hosts = config.get_hosts(segments=True) for host in hosts: cmd = 'gpssh -h %s -e "scp %s %s:%s/" ' % (host, filename, host, location) tinctest.logger.debug(cmd) res = {'rc':0, 'stderr':'', 'stdout':''} run_shell_command (cmd, 'run scp', res) if res['rc']>0: raise Exception('Copying to host %s failed' % host)
def copy_files_to_segments(filename, location): config = GPDBConfig() hosts = config.get_hosts(segments=True) for host in hosts: cmd = 'gpssh -h %s -e "scp %s %s:%s/" ' % (host, filename, host, location) tinctest.logger.debug(cmd) res = {"rc": 0, "stderr": "", "stdout": ""} run_shell_command(cmd, "run scp", res) if res["rc"] > 0: raise Exception("Copying to host %s failed" % host)
def setUpClass(cls): super(PreExpansionWorkloadTests, cls).setUpClass() # gpscp the script required for external table in create_base_workload scp_file = os.path.join(cls.get_sql_dir(), 'datagen.py') gpconfig = GPDBConfig() hosts = gpconfig.get_hosts() hosts_file = os.path.join(cls.get_out_dir(), 'hostfile') with open(hosts_file, 'w') as f: f.write('\n'.join(hosts)) res = {'rc':0, 'stderr':'', 'stdout':''} run_shell_command("gpscp -f %s %s =:$GPHOME/bin" %(hosts_file, scp_file), 'gpscp script', res) if res['rc'] > 0: tinctest.logger.warning("Failed to gpscp the required script to all the segments for external table queries. The script might already exist !")
def setUpClass(cls): super(PreExpansionWorkloadTests, cls).setUpClass() # gpscp the script required for external table in create_base_workload scp_file = os.path.join(cls.get_sql_dir(), 'datagen.py') gpconfig = GPDBConfig() hosts = gpconfig.get_hosts() hosts_file = os.path.join(cls.get_out_dir(), 'hostfile') with open(hosts_file, 'w') as f: f.write('\n'.join(hosts)) res = {'rc':0, 'stderr':'', 'stdout':''} run_shell_command("gpscp -f %s %s =:$GPHOME/bin" %(hosts_file, scp_file), 'gpscp script', res) if res['rc'] > 0: tinctest.logger.warning("Failed to gpscp the required script to all the segments for external table queries. The script might already exist !")
class GPDBConfigRegressionTests(unittest.TestCase): def __init__(self, methodName): self.gpconfig = GPDBConfig() super(GPDBConfigRegressionTests, self).__init__(methodName) def test_get_countprimarysegments(self): nprimary = self.gpconfig.get_countprimarysegments() self.assertTrue(nprimary > 0) def test_get_hostandport_of_segment(self): (host, port) = self.gpconfig.get_hostandport_of_segment(psegmentNumber=-1, pRole='p') myhost = socket.gethostname() self.assertEquals(host, myhost) def test_get_count_segments(self): seg_count = self.gpconfig.get_count_segments() self.assertTrue(seg_count.strip() > 0) def test_seghostnames(self): hostlist = self.gpconfig.get_hosts() self.assertTrue(len(hostlist) > 0) def test_hostnames(self): hostlist = self.gpconfig.get_hosts(segments=False) self.assertTrue(len(hostlist) > 0) def tes_get_masterhost(self): master_host = self.gpconfig.get_masterhost() myhost = socket.gethostname() self.assertEquals(master_host, myhost) def test_get_masterdata_directory(self): master_dd = self.gpconfig.get_masterdata_directory() my_mdd = os.getenv("MASTER_DATA_DIRECTORY") self.assertEquals(master_dd, my_mdd)
class GPDBConfigRegressionTests(unittest.TestCase): def __init__(self, methodName): self.gpconfig = GPDBConfig() super(GPDBConfigRegressionTests,self).__init__(methodName) def test_get_countprimarysegments(self): nprimary = self.gpconfig.get_countprimarysegments() self.assertTrue(nprimary > 0) def test_get_hostandport_of_segment(self): (host,port) = self.gpconfig.get_hostandport_of_segment(psegmentNumber = -1, pRole = 'p') myhost = socket.gethostname() self.assertEquals(host, myhost) def test_get_count_segments(self): seg_count = self.gpconfig.get_count_segments() self.assertTrue(seg_count.strip() >0) def test_seghostnames(self): hostlist = self.gpconfig.get_hosts() self.assertTrue(len(hostlist) >0) def test_hostnames(self): hostlist = self.gpconfig.get_hosts(segments=False) self.assertTrue(len(hostlist) >0) def tes_get_masterhost(self): master_host = self.gpconfig.get_masterhost() myhost = socket.gethostname() self.assertEquals(master_host, myhost) def test_get_masterdata_directory(self): master_dd = self.gpconfig.get_masterdata_directory() my_mdd = os.getenv("MASTER_DATA_DIRECTORY") self.assertEquals(master_dd, my_mdd)
class GPAddmirrorsTestCase(MPPTestCase): def __init__(self, methodName): self.config = GPDBConfig() self.mdd = os.environ.get('MASTER_DATA_DIRECTORY') self.seg_prefix = os.path.basename(self.mdd).split('-')[0] self.master_host = self.config.get_masterhost() self.gpinitconfig_template = local_path( 'configs/gpinitconfig_template') self.datadir_config_file = local_path('configs/datadir_config_file') self.mirror_config_file = local_path('configs/mirror_config_file') self.gpinitconfig_file = local_path('configs/gpinitconfig') self.host_file = local_path('configs/hosts') self.hosts = self.config.get_hosts(segments=True) self.port_base = '40000' self.master_port = os.environ.get('PGPORT', '5432') self.primary_data_dir = self.config.get_host_and_datadir_of_segment( dbid=2)[1] # initially set the mirror data dir same to primary's self.mirror_data_dir = os.path.join( os.path.dirname(os.path.dirname(self.primary_data_dir)), 'mirror') self.gpinitsystem = True self.number_of_segments = self.config.get_countprimarysegments() self.number_of_segments_per_host = self.number_of_segments / len( self.hosts) self.standby_enabled = False self.number_of_parallelism = 4 self.fs_location = [] super(GPAddmirrorsTestCase, self).__init__(methodName) def setUp(self): super(GPAddmirrorsTestCase, self).setUp() def _setup_gpaddmirrors(self, port_offset=1000): """ Takes care of creating all the directories required for gpaddmirrors and generating input files for gpaddmirrors """ # Generate gpaddmirrors config files try: self._generate_gpaddmirrors_input_files(port_offset) except Exception, e: tinctest.logger.exception( "Encountered exception during generation of input files: %s" % e) raise
class GPAddmirrorsTestCase(MPPTestCase): def __init__(self, methodName): self.config = GPDBConfig() self.mdd = os.environ.get('MASTER_DATA_DIRECTORY') self.seg_prefix = os.path.basename(self.mdd).split('-')[0] self.master_host = self.config.get_masterhost() self.gpinitconfig_template = local_path('configs/gpinitconfig_template') self.datadir_config_file = local_path('configs/datadir_config_file') self.mirror_config_file = local_path('configs/mirror_config_file') self.gpinitconfig_file = local_path('configs/gpinitconfig') self.host_file = local_path('configs/hosts') self.hosts = self.config.get_hosts(segments = True) self.port_base = '40000' self.master_port = os.environ.get('PGPORT', '5432') self.primary_data_dir = self.config.get_host_and_datadir_of_segment(dbid = 2)[1] # initially set the mirror data dir same to primary's self.mirror_data_dir = os.path.join(os.path.dirname(os.path.dirname(self.primary_data_dir)), 'mirror') self.gpinitsystem = True self.number_of_segments = self.config.get_countprimarysegments() self.number_of_segments_per_host = self.number_of_segments / len(self.hosts) self.standby_enabled = False self.number_of_parallelism = 4 self.fs_location = [] super(GPAddmirrorsTestCase, self).__init__(methodName) def setUp(self): super(GPAddmirrorsTestCase, self).setUp() def _setup_gpaddmirrors(self, port_offset=1000): """ Takes care of creating all the directories required for gpaddmirrors and generating input files for gpaddmirrors """ # Generate gpaddmirrors config files try: self._generate_gpaddmirrors_input_files(port_offset) except Exception, e: tinctest.logger.exception("Encountered exception during generation of input files: %s" % e) raise
class GpinitStandby(object): '''Class for gpinitstandby operations Disclaimer: Some of these may repeat with the mpp/lib version''' def __init__(self): self.stdby = StandbyVerify() self.runmixin = StandbyRunMixin() self.runmixin.createdb(dbname='walrepl') self.mdd = os.environ.get('MASTER_DATA_DIRECTORY') self.config = GPDBConfig() self.pgutil = GpUtility() self.host = socket.gethostname() def run(self, option=''): '''Runs gpinitstandby and returns True if successfull ''' gpinitstandby_cmd = 'gpinitstandby -a %s' % option cmd = Command(name='Running Gpinitstandby', cmdStr="%s" % gpinitstandby_cmd) tinctest.logger.info(" %s" % cmd) cmd.run(validateAfter=False) result = cmd.get_results() if result.rc != 0: return False return True def verify_gpinitstandby(self, primary_pid): '''Verify the presence of standby in recovery mode ''' if (self.stdby.check_gp_segment_config()) and ( self.stdby.check_pg_stat_replication()) and ( self.stdby.check_standby_processes() ) and self.compare_primary_pid(primary_pid): return True return False def get_masterhost(self): std_sql = "select hostname from gp_segment_configuration where content=-1 and role='p';" master_host = PSQL.run_sql_command(std_sql, flags='-q -t', dbname='postgres') return master_host.strip() def get_standbyhost(self): std_sql = "select hostname from gp_segment_configuration where content='-1' and role='m';" standby_host = PSQL.run_sql_command(std_sql, flags='-q -t', dbname='postgres') return standby_host.strip() def get_filespace_location(self): fs_sql = "select fselocation from pg_filespace_entry where fselocation like '%fs_walrepl_a%' and fsedbid=1;" filespace_loc = PSQL.run_sql_command(fs_sql, flags='-q -t', dbname='postgres') return filespace_loc.strip() def get_standbyhostnode(self): ''' Function used to obtain the hostname of one of the segment node inorder to use it as the standby master node" @return : returns the hostname of the segment node which can be used as the standby master node ''' hostlist = self.config.get_hosts() standby = '' for host in hostlist: if host.strip() != self.host: standby = host.strip() if len(standby) > 0: return standby else: tinctest.logger.error( 'No segment host other than master available to have remote standby' ) def get_primary_pid(self): pid = self.pgutil.get_pid_by_keyword(pgport=os.environ.get('PGPORT'), keyword=self.mdd) if int(pid) == -1: raise WalReplException( 'Unable to get pid of primary master process') else: return int(pid) def compare_primary_pid(self, initial_pid): final_pid = self.get_primary_pid() if initial_pid == final_pid: return True return False def create_dir_on_standby(self, standby, location): fs_cmd = "gpssh -h %s -e 'rm -rf %s; mkdir -p %s' " % ( standby, location, location) cmd = Command( name='Make dierctory on standby before running the command', cmdStr=fs_cmd) tinctest.logger.info('%s' % cmd) cmd.run(validateAfter=True) result = cmd.get_results() if result.rc != 0: raise WalReplException('Unable to create directory on standby') else: return True def initstand_by_with_default(self): master_host = self.get_masterhost() gp_cmd = "/bin/bash -c 'gpinitstandby -s %s'" % (master_host) cmd = Command(name='Running the command', cmdStr=gp_cmd) tinctest.logger.info('%s' % cmd) cmd.run(validateAfter=False) sleep(2) result = cmd.get_results() lines = result.stdout.splitlines() for line in lines: if 'Data directory already exists' in line: return True return False def init_with_prompt(self, filespace_loc): standby = self.get_standbyhostnode() gp_cmd = "/bin/bash -c 'gpinitstandby -s %s -a'" % (standby) logfile = open(local_path('install2.log'), 'w') child = pexpect.spawn(gp_cmd, timeout=400) child.logfile = logfile sleep(5) check = child.expect( ['.* Enter standby filespace location for filespace.*', ' ']) child.sendline(filespace_loc) sleep(10) check = child.expect(['.*Successfully created standby master.*']) if check != 0: tinctest.logger.error('gpinitstandy failed') return False child.close() return True
def _generate_gpexpand_input_files(self): config = GPDBConfig() # use the last segment as a template # TODO: This logic needs fixing when we enable mirror tests seg = (sorted(config.record, lambda a, b: b.content - a.content)).pop(0) # Find max db id for generating gpexpand input file max_db_id = max([r.dbid for r in config.record]) self.assertNotEqual(seg.content, -1) def next_datadir(datadir, i=1): # /path/to/foo/seg12 -> /path/to/foo/seg13 lefthand = datadir.rstrip('0123456789') segnum = int(datadir[len(lefthand):]) return lefthand + str(segnum + i) with open(self.testcase_gpexpand_file, 'w') as f: # For existing hosts, add self.number_of_expansion_segments # For new hosts add existing number of segments + number_of_expansion_segments # Existing hosts existing_hosts = config.get_hosts(segments=True) existing_number_of_segments = 0 cnt = 1 for host in existing_hosts: # Find the existing number of segments in the existing hosts existing_number_of_segments = config.get_segments_count_per_host()[host] for i in range(self.number_of_expansion_segments): f.write( "{host}:{addr}:{port}:{datadir}:{dbid}:{content}:{role}\n".format( host=host, addr=host, port= int(seg.port) + cnt, datadir=next_datadir(seg.datadir, cnt), dbid=int(max_db_id) + cnt, content= int(seg.content) + cnt, role='p' )) cnt += 1 # New hosts if self.number_of_expansion_hosts > 0: new_expansion_hosts = list(set(self.hosts) - existing_hosts) if not new_expansion_hosts: raise GPExpandTestCaseException("No new hosts available for expansion based on the environment variable GPEXPAND_HOSTS: %s" %os.environ.get("GPEXPAND_HOSTS")) for host in new_expansion_hosts[0:self.number_of_expansion_hosts]: for i in range(existing_number_of_segments + self.number_of_expansion_segments): f.write( "{host}:{addr}:{port}:{datadir}:{dbid}:{content}:{role}\n".format( host=host, addr=host, port= int(seg.port) + cnt, datadir=next_datadir(seg.datadir, cnt), dbid=int(seg.dbid) + cnt, content= int(seg.content) + cnt, role='p' )) cnt += 1
class GpinitStandby(object): '''Class for gpinitstandby operations Disclaimer: Some of these may repeat with the mpp/lib version''' def __init__(self): self.stdby = StandbyVerify() self.runmixin = StandbyRunMixin() self.runmixin.createdb(dbname='walrepl') self.mdd = os.environ.get('MASTER_DATA_DIRECTORY') self.config = GPDBConfig() self.pgutil = GpUtility() self.host = socket.gethostname() def run(self, option = ''): '''Runs gpinitstandby and returns True if successfull ''' gpinitstandby_cmd = 'gpinitstandby -a %s' % option cmd = Command(name='Running Gpinitstandby', cmdStr="%s" % gpinitstandby_cmd) tinctest.logger.info(" %s" % cmd) cmd.run(validateAfter=False) result = cmd.get_results() if result.rc != 0: return False return True def verify_gpinitstandby(self, primary_pid): '''Verify the presence of standby in recovery mode ''' if (self.stdby.check_gp_segment_config()) and (self.stdby.check_pg_stat_replication()) and (self.stdby.check_standby_processes())and self.compare_primary_pid(primary_pid) : return True return False def get_masterhost(self): std_sql = "select hostname from gp_segment_configuration where content=-1 and role='p';" master_host = PSQL.run_sql_command(std_sql, flags = '-q -t', dbname= 'postgres') return master_host.strip() def get_standbyhost(self): std_sql = "select hostname from gp_segment_configuration where content='-1' and role='m';" standby_host = PSQL.run_sql_command(std_sql, flags = '-q -t', dbname= 'postgres') return standby_host.strip() def get_filespace_location(self): fs_sql = "select fselocation from pg_filespace_entry where fselocation like '%fs_walrepl_a%' and fsedbid=1;" filespace_loc = PSQL.run_sql_command(fs_sql, flags = '-q -t', dbname= 'postgres') return filespace_loc.strip() def get_standbyhostnode(self): ''' Function used to obtain the hostname of one of the segment node inorder to use it as the standby master node" @return : returns the hostname of the segment node which can be used as the standby master node ''' hostlist = self.config.get_hosts() standby = '' for host in hostlist: if host.strip() != self.host: standby = host.strip() if len(standby) > 0 : return standby else: tinctest.logger.error('No segment host other than master available to have remote standby') def get_primary_pid(self): pid = self.pgutil.get_pid_by_keyword(pgport=os.environ.get('PGPORT'), keyword=self.mdd) if int(pid) == -1: raise WalReplException('Unable to get pid of primary master process') else: return int(pid) def compare_primary_pid(self, initial_pid): final_pid = self.get_primary_pid() if initial_pid == final_pid : return True return False def create_dir_on_standby(self, standby, location): fs_cmd = "gpssh -h %s -e 'rm -rf %s; mkdir -p %s' " % (standby, location, location) cmd = Command(name='Make dierctory on standby before running the command', cmdStr = fs_cmd) tinctest.logger.info('%s' % cmd) cmd.run(validateAfter=True) result = cmd.get_results() if result.rc != 0: raise WalReplException('Unable to create directory on standby') else: return True def initstand_by_with_default(self): master_host = self.get_masterhost() gp_cmd = "/bin/bash -c 'gpinitstandby -s %s'" % (master_host) logfile = open(local_path('install.log'),'w') child = pexpect.spawn(gp_cmd, timeout=400) child.logfile = logfile sleep(2) check = child.expect(['.* Enter standby filespace location for filespace pg_system .*', ' ']) if check != 0: child.close() l_file = open(local_path('install.log'),'r') lines = l_file.readlines() for line in lines: if 'default: NA' in line: return True return False def init_with_prompt(self,filespace_loc): standby = self.get_standbyhostnode() gp_cmd = "/bin/bash -c 'gpinitstandby -s %s -a'" % (standby) logfile = open(local_path('install2.log'),'w') child = pexpect.spawn(gp_cmd, timeout=400) child.logfile = logfile sleep(5) check = child.expect(['.* Enter standby filespace location for filespace.*', ' ']) child.sendline(filespace_loc) sleep(10) check = child.expect(['.*Successfully created standby master.*']) if check != 0: tinctest.logger.error('gpinitstandy failed') return False child.close() return True