def __init__(self, methodName):
self.gp = GpactivateStandby()
self.verify = StandbyVerify()
self.config = GPDBConfig()
self.disk = Disk()
self.sdby_mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.pgport = os.environ.get('PGPORT')
super(OODClass, self).__init__(methodName)
def __init__(self,methodName):
self.gp = GpactivateStandby()
self.verify = StandbyVerify()
self.config = GPDBConfig()
self.disk = Disk()
self.sdby_mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.pgport = os.environ.get('PGPORT')
super(OODClass,self).__init__(methodName)
def __init__(self):
self.disk = Disk()
class Steps():
def __init__(self):
self.disk = Disk()
def get_segments(self):
get_seg_sql = "select distinct(hostname) from gp_segment_configuration;"
res = PSQL.run_sql_command(get_seg_sql, dbname = 'template1', flags ='-q -t')
return res.strip().split('\n')
def checkDiskUsage(self, host, disk): # This now checks for only /data
(rc, result) = self.disk.get_disk_usage(host, '/'+disk)
if rc != 0:
raise Exception ("The specified mount /data is not present for the device")
else:
available_usage = result
return available_usage
def fillDisk(self, filename, host):
cmd_prefix = "ssh " +host+ " \""
cmd_postfix = "\""
location = os.getcwd()
if not os.path.isdir('%s/diskfill/' % location):
os.makedirs('%s/diskfill/' % location)
cmd_str = cmd_prefix + "dd if=/dev/zero bs=1024K count=1000 of=" +location+ "/diskfill/" + filename +cmd_postfix
results={'rc':0, 'stdout':'', 'stderr':''}
run_shell_command(cmd_str, results=results)
if int(results['rc']) !=0:
raise Exception('disk fill not working')
def changetracking(self, type = 'mirror'):
''' Routine to inject fault that places system in change tracking'''
tinctest.logger.info("Put system in changetracking ")
cmd_str = 'gpfaultinjector -f filerep_consumer -m async -y fault -r %s -H ALL' %type
results={'rc':0, 'stdout':'', 'stderr':''}
run_shell_command(cmd_str, results=results)
return results['stdout']
def is_dbdown(self):
'''return true if system is the system is down'''
tinctest.logger.info("checking if segments are up and in sync")
cmd_str = "select count(*) from gp_segment_configuration where mode = 'c'"
res = PSQL.run_sql_command(cmd_str, dbname = 'template1', flags ='-q -t')
if int(res.strip()) > 0:
return True
return False
def checkLogMessage(self):
''' Select from gp_toolkit log message to see if the concurrent test run resulted in No space left on device'''
log_sql = "select substring(logmessage from 0 for 60) from gp_toolkit.__gp_log_segment_ext where logmessage like '%disabled%';"
for i in range(1,25):
result = PSQL.run_sql_command(log_sql, dbname = 'template1', flags ='-q -t')
result= res.strip().split('\n')
log_chk = False
for logmsg in result:
if ('write error for change tracking full log' in logmsg[0] ):
log_chk = True
break
if log_chk == True:
break
sleep(2)
if log_chk == False:
raise Exception("Message 'changetracking disabled' not found in the logs")
return True
def remove_fillfiles(self, filename, host):
location = os.getcwd()
cmd_str = "ssh %s rm %s/diskfill/%s*" % (host,location, filename)
results={'rc':0, 'stdout':'', 'stderr':''}
run_shell_command(cmd_str, results=results)
if int(results['rc']) !=0:
raise Exception('Unable to delete the fill files')
return
def recover_segments(self):
'''call gprecoverseg -a to recover down segments'''
tinctest.logger.info("call gprecoverseg -a to recover down segments")
cmd_str = 'gprecoverseg -a'
results={'rc':0, 'stdout':'', 'stderr':''}
run_shell_command(cmd_str, results=results)
class OODClass(MPPTestCase):
def __init__(self,methodName):
self.gp = GpactivateStandby()
self.verify = StandbyVerify()
self.config = GPDBConfig()
self.disk = Disk()
self.sdby_mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.pgport = os.environ.get('PGPORT')
super(OODClass,self).__init__(methodName)
def initiate_standby(self):
self.gp.create_standby(local='no')
def check_standby(self):
self.assertFalse(self.verify.check_standby_processes())
def get_standby_dbid(self):
std_sql = "select dbid from gp_segment_configuration where content='-1' and role='m';"
standby_dbid = PSQL.run_sql_command(std_sql, flags = '-q -t', dbname= 'template1')
return standby_dbid.strip()
def restart_standby(self):
sdby_host = self.config.get_master_standbyhost()
stdby_dbid = self.get_standby_dbid()
cmd="pg_ctl -D %s -o '-p %s --gp_dbid=%s --gp_num_contents_in_cluster=2 --silent-mode=true -i -M master --gp_contentid=-1 -x 0 -E' start &"%(self.sdby_mdd, self.pgport, stdby_dbid)
self.assertTrue(self.gp.run_remote(sdby_host,cmd, self.pgport, self.sdby_mdd))
self.assertTrue(self.verify.check_standby_processes())
def check_diskusage(self, host): # This now checks for only /data
(rc, result) = self.disk.get_disk_usage(host, '/data')
if rc != 0:
raise Exception ("The specified mount /data is not present for the device")
else:
available_usage = result
return available_usage
def _fill(self, filename, host):
cmd_prefix = "ssh " +host+ " \""
cmd_postfix = "\""
location = '/data'
if not os.path.isdir('%s/diskfill/' % location):
os.makedirs('%s/diskfill/' % location)
cmd_str = cmd_prefix + "dd if=/dev/zero bs=16384K count=2000 of=" +location+ "/diskfill/" + filename +cmd_postfix
cmd = Command(name='Fill Disk', cmdStr=cmd_str)
tinctest.logger.info(" %s" % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
if result.rc !=0:
tinctest.logger.error('disk fill not working. Its already full')
def filldisk(self):
host = self.config.get_master_standbyhost()
disk_usage = self.check_diskusage(host)
i = 0
while(int(disk_usage.strip()) >1000000):
filename = 'new_space_%s' % i
self._fill(filename, host)
i +=1
disk_usage = self.check_diskusage(host)
def remove_fillfiles(self, filename, host):
location = '/data'
cmd_str = "ssh %s rm %s/diskfill/%s*" % (host,location, filename)
cmd = Command(name='Remove fill files', cmdStr=cmd_str)
tinctest.logger.info(" %s" % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
if result.rc !=0:
raise Exception('Unable to delete the fill files')
return
def cleanup(self):
host = self.config.get_master_standbyhost()
self.remove_fillfiles('new_space', host)
#Recover segemnts in case segments and standby were on the same host
cmd = Command(name='gprecoverseg', cmdStr='gprecoverseg -a')
tinctest.logger.info(" %s" % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
if result.rc !=0:
raise Exception('gprecoverseg failed')
while(self.config.is_not_insync_segments() == False):
tinctest.logger.info('Waiting for DB to be in sync')
def __init__(self):
self.disk = Disk()
class Steps():
def __init__(self):
self.disk = Disk()
def get_segments(self):
get_seg_sql = "select distinct(hostname) from gp_segment_configuration;"
res = PSQL.run_sql_command(get_seg_sql, dbname = 'template1', flags ='-q -t')
return res.strip().split('\n')
def checkDiskUsage(self, host, disk): # This now checks for only /data
(rc, result) = self.disk.get_disk_usage(host, '/'+disk)
if rc != 0:
raise Exception ("The specified mount /data is not present for the device")
else:
available_usage = result
return available_usage
def fillDisk(self, filename, host):
cmd_prefix = "ssh " +host+ " \""
cmd_postfix = "\""
location = os.getcwd()
if not os.path.isdir('%s/diskfill/' % location):
os.makedirs('%s/diskfill/' % location)
cmd_str = cmd_prefix + "dd if=/dev/zero bs=1024K count=1000 of=" +location+ "/diskfill/" + filename +cmd_postfix
results={'rc':0, 'stdout':'', 'stderr':''}
run_shell_command(cmd_str, results=results)
if int(results['rc']) !=0:
raise Exception('disk fill not working')
def changetracking(self, type = 'mirror'):
''' Routine to inject fault that places system in change tracking'''
tinctest.logger.info("Put system in changetracking ")
cmd_str = 'gpfaultinjector -f filerep_consumer -m async -y fault -r %s -H ALL' %type
results={'rc':0, 'stdout':'', 'stderr':''}
run_shell_command(cmd_str, results=results)
return results['stdout']
def is_dbdown(self):
'''return true if system is the system is down'''
tinctest.logger.info("checking if segments are up and in sync")
cmd_str = "select count(*) from gp_segment_configuration where mode = 'c'"
res = PSQL.run_sql_command(cmd_str, dbname = 'template1', flags ='-q -t')
if int(res.strip()) > 0:
return True
return False
def checkLogMessage(self):
''' Select from gp_toolkit log message to see if the concurrent test run resulted in No space left on device'''
log_sql = "select substring(logmessage from 0 for 60) from gp_toolkit.__gp_log_segment_ext where logmessage like '%disabled%';"
for i in range(1,25):
result = PSQL.run_sql_command(log_sql, dbname = 'template1', flags ='-q -t')
result= res.strip().split('\n')
log_chk = False
for logmsg in result:
if ('write error for change tracking full log' in logmsg[0] ):
log_chk = True
break
if log_chk == True:
break
sleep(2)
if log_chk == False:
raise Exception("Message 'changetracking disabled' not found in the logs")
return True
def remove_fillfiles(self, filename, host):
location = os.getcwd()
cmd_str = "ssh %s rm %s/diskfill/%s*" % (host,location, filename)
results={'rc':0, 'stdout':'', 'stderr':''}
run_shell_command(cmd_str, results=results)
if int(results['rc']) !=0:
raise Exception('Unable to delete the fill files')
return
def recover_segments(self):
'''call gprecoverseg -a to recover down segments'''
tinctest.logger.info("call gprecoverseg -a to recover down segments")
cmd_str = 'gprecoverseg -a'
results={'rc':0, 'stdout':'', 'stderr':''}
run_shell_command(cmd_str, results=results)
class OODClass(MPPTestCase):
def __init__(self, methodName):
self.gp = GpactivateStandby()
self.verify = StandbyVerify()
self.config = GPDBConfig()
self.disk = Disk()
self.sdby_mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.pgport = os.environ.get('PGPORT')
super(OODClass, self).__init__(methodName)
def initiate_standby(self):
self.gp.create_standby(local='no')
def check_standby(self):
self.assertFalse(self.verify.check_standby_processes())
def get_standby_dbid(self):
std_sql = "select dbid from gp_segment_configuration where content='-1' and role='m';"
standby_dbid = PSQL.run_sql_command(std_sql,
flags='-q -t',
dbname='template1')
return standby_dbid.strip()
def restart_standby(self):
sdby_host = self.config.get_master_standbyhost()
stdby_dbid = self.get_standby_dbid()
cmd = "pg_ctl -D %s -o '-p %s -b %s -z 2 --silent-mode=true -i -M master -C -1 -x 0 -E' start &" % (
self.sdby_mdd, self.pgport, stdby_dbid)
self.assertTrue(
self.gp.run_remote(sdby_host, cmd, self.pgport, self.sdby_mdd))
self.assertTrue(self.verify.check_standby_processes())
def check_diskusage(self, host): # This now checks for only /data
(rc, result) = self.disk.get_disk_usage(host, '/data')
if rc != 0:
raise Exception(
"The specified mount /data is not present for the device")
else:
available_usage = result
return available_usage
def _fill(self, filename, host):
cmd_prefix = "ssh " + host + " \""
cmd_postfix = "\""
location = '/data'
if not os.path.isdir('%s/diskfill/' % location):
os.makedirs('%s/diskfill/' % location)
cmd_str = cmd_prefix + "dd if=/dev/zero bs=16384K count=2000 of=" + location + "/diskfill/" + filename + cmd_postfix
cmd = Command(name='Fill Disk', cmdStr=cmd_str)
tinctest.logger.info(" %s" % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
if result.rc != 0:
tinctest.logger.error('disk fill not working. Its already full')
def filldisk(self):
host = self.config.get_master_standbyhost()
disk_usage = self.check_diskusage(host)
i = 0
while (int(disk_usage.strip()) > 1000000):
filename = 'new_space_%s' % i
self._fill(filename, host)
i += 1
disk_usage = self.check_diskusage(host)
def remove_fillfiles(self, filename, host):
location = '/data'
cmd_str = "ssh %s rm %s/diskfill/%s*" % (host, location, filename)
cmd = Command(name='Remove fill files', cmdStr=cmd_str)
tinctest.logger.info(" %s" % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
if result.rc != 0:
raise Exception('Unable to delete the fill files')
return
def cleanup(self):
host = self.config.get_master_standbyhost()
self.remove_fillfiles('new_space', host)
#Recover segemnts in case segments and standby were on the same host
cmd = Command(name='gprecoverseg', cmdStr='gprecoverseg -a')
tinctest.logger.info(" %s" % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
if result.rc != 0:
raise Exception('gprecoverseg failed')
while (self.config.is_not_insync_segments() == False):
tinctest.logger.info('Waiting for DB to be in sync')
