def _run_test(self, test_name, test_type):
"""Run the actual test.
@param test_type pg_dump or gp_dump
"""
base = self._base_file_path()
base = os.path.join(base, test_name)
self._sql_file = base + '.sql'
self._init_file = base + '_init.sql'
self._init_out_file = base + '.' + test_type + '_init.out'
self._ans_file = base + '.ans'
self._dump_file = base + '.dump' # for pg_dump only
self._pre_out_file = base + '.' + test_type + '.out1'
self._post_out_file = base + '.' + test_type + '.out2'
logger.debug("Refreshing database")
self._refresh_database()
self._run_sql(self._init_file, self._init_out_file)
self._run_sql(self._sql_file, self._pre_out_file)
self._validate(self._pre_out_file, self._ans_file)
if test_type == 'pg_dump':
self._pg_dump()
elif test_type == 'gp_dump':
self._gp_dump()
else:
assert False, "unknown test_type " + test_type
self._refresh_database()
if test_type == 'pg_dump':
self._pg_restore()
elif test_type == 'gp_dump':
self._gp_restore()
self._run_sql(self._sql_file, self._post_out_file)
self._validate(self._post_out_file, self._ans_file)
def _run_test(self, test_name, test_type):
"""Run the actual test.
@param test_type pg_dump or gp_dump
"""
base = self._base_file_path()
base = os.path.join(base, test_name)
self._sql_file = base + '.sql'
self._init_file = base + '_init.sql'
self._init_out_file = base + '.' + test_type + '_init.out'
self._ans_file = base + '.ans'
self._dump_file = base + '.dump' # for pg_dump only
self._pre_out_file = base + '.' + test_type + '.out1'
self._post_out_file = base + '.' + test_type + '.out2'
logger.debug("Refreshing database")
self._refresh_database()
self._run_sql(self._init_file, self._init_out_file)
self._run_sql(self._sql_file, self._pre_out_file)
self._validate(self._pre_out_file, self._ans_file)
if test_type == 'pg_dump':
self._pg_dump()
elif test_type == 'gp_dump':
self._gp_dump()
else:
assert False, "unknown test_type " + test_type
self._refresh_database()
if test_type == 'pg_dump':
self._pg_restore()
elif test_type == 'gp_dump':
self._gp_restore()
self._run_sql(self._sql_file, self._post_out_file)
self._validate(self._post_out_file, self._ans_file)
def _refresh_database(self):
"Drop (if exists) and create a database."
self._drop_database()
psql = RetryPSQL(sql_cmd="""
CREATE DATABASE {dbname};
""".format(dbname=self.dbname), dbname="postgres")
assert psql.run(), "CREATE DATABASE failed"
logger.debug("Database created")
def _refresh_database(self):
"Drop (if exists) and create a database."
self._drop_database()
psql = RetryPSQL(sql_cmd="""
CREATE DATABASE {dbname};
""".format(dbname=self.dbname),
dbname="postgres")
assert psql.run(), "CREATE DATABASE failed"
logger.debug("Database created")
def check_pkg_exists(self, pkgname):
cmd = 'gppkg -q --all'
res = {'rc': 0, 'stderr': '', 'stdout': ''}
run_shell_command(cmd, 'run gppkg', res)
logger.debug(res['stdout'])
pkgs = res['stdout'].strip().split('\n')[1:]
for pkg in pkgs:
if pkgname in pkg:
return (True, pkg)
return (False, None)
def check_pkg_exists(self, pkgname):
cmd = 'gppkg -q --all'
res = {'rc':0, 'stderr':'', 'stdout':''}
run_shell_command (cmd, 'run gppkg', res)
logger.debug(res['stdout'])
pkgs = res['stdout'].strip().split('\n')[1:]
for pkg in pkgs:
if pkgname in pkg:
return (True, pkg)
return (False, None)
def run(self):
"""Run psql and see if stderr contains "ERROR" string.
If it was an error, re-try with given interval (1 sec by default)
and repeat as many as retry parameter (5 by default). If retry
count exceeds, return False. Otherwise, True.
"""
retry = self.retry
for i in range(self.retry):
cmd = PSQL(**self.psql_args)
logger.debug("Running command: %s" % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
if "ERROR" not in result.stderr:
return True
time.sleep(self.interval)
return False
def run(self):
"""Run psql and see if stderr contains "ERROR" string.
If it was an error, re-try with given interval (1 sec by default)
and repeat as many as retry parameter (5 by default). If retry
count exceeds, return False. Otherwise, True.
"""
retry = self.retry
for i in range(self.retry):
cmd = PSQL(**self.psql_args)
logger.debug("Running command: %s" % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
if "ERROR" not in result.stderr:
return True
time.sleep(self.interval)
return False
def download_pkg(self, product_version, gppkg):
"""
Download gppkg from artifacts server.
"""
target_dir = local_path('download/')
if not os.path.exists(target_dir):
os.makedirs(target_dir)
(rc, download_link, package_name) = self.get_download_url_from_build_prod(product_version, gppkg)
if rc != 0:
return (-1, None, None)
wget_cmd = 'wget --html-extension %s -O %s`basename %s`' % (download_link, target_dir, download_link)
logger.debug('Download link: %s' % wget_cmd)
res = {'rc':0, 'stderr':'', 'stdout':''}
run_shell_command (wget_cmd, 'run wget', res)
if res['rc'] > 0:
raise Exception("Gppkg download failed")
return (0, target_dir, package_name)
def run(self):
"""
Run gpinitsystem
rc=0, gpinitsystem has no warning(s) or error(s)
rc=1, gpinitsystem has warning(s) but no error(s)
rc=2, gpinitsystem has error(s)
"""
self.create_datadir()
cmd = Command(name='run gpinitsystem', cmdStr='source %s/%s; gpinitsystem -a -c %s' %\
(self.dir, self.source_path, self.config_file))
cmd.run()
result = cmd.get_results()
if result.rc > 1:
msg = "stdout:%s\nstderr:%s" % (result.stdout, result.stderr)
raise GpinitsystemError("gpinitsystem failed (%d): %s" % (result.rc, msg))
logger.debug("Successfully ran gpinitsystem ...")
def download_pkg(self, product_version, gppkg):
"""
Download gppkg from artifacts server.
"""
target_dir = local_path('download/')
if not os.path.exists(target_dir):
os.makedirs(target_dir)
(rc, download_link, package_name) = self.get_download_url_from_build_prod(product_version, gppkg)
if rc != 0:
return (-1, None, None)
wget_cmd = 'wget --html-extension %s -O %s`basename %s`' % (download_link, target_dir, download_link)
logger.debug('Download link: %s' % wget_cmd)
res = {'rc':0, 'stderr':'', 'stdout':''}
run_shell_command (wget_cmd, 'run wget', res)
if res['rc'] > 0:
raise Exception("Gppkg download failed")
return (0, target_dir, package_name)
def run_gppkg_install(self, pkgfile):
"""
@summary: Runs gppkg -i to install a gppkg. Output is written to gppkg_i.log file in current directory.
@param pkgdir: The directory containing the gppkg file
@param pkgfile: The name of the .gppkg file in pkgdir
@raise GppkgUtilError: If gppkg install fails or if pkgfile specified does not exist
"""
if os.path.isfile(pkgfile):
logger.debug( '\nGppkgUtil: Installing gppkg using gppkg file: %s' % (pkgfile))
cmd = 'gppkg -i %s' % pkgfile
res = {'rc':0, 'stderr':'', 'stdout':''}
run_shell_command (cmd, 'run gppkg', res)
logger.debug(res)
if res['rc']> 0:
tinctest.logger.info('result from install package %s' % res['stdout'])
raise Exception('Failed to install the package')
self.check_and_install_sql(res['stdout'])
else:
raise Exception("*** ERROR: .gppkg file not found '. Make sure %s exists." % (pkgfile))
def run_gppkg_install(self, pkgfile):
"""
@summary: Runs gppkg -i to install a gppkg. Output is written to gppkg_i.log file in current directory.
@param pkgdir: The directory containing the gppkg file
@param pkgfile: The name of the .gppkg file in pkgdir
@raise GppkgUtilError: If gppkg install fails or if pkgfile specified does not exist
"""
if os.path.isfile(pkgfile):
logger.debug( '\nGppkgUtil: Installing gppkg using gppkg file: %s' % (pkgfile))
cmd = 'gppkg -i %s' % pkgfile
res = {'rc':0, 'stderr':'', 'stdout':''}
run_shell_command (cmd, 'run gppkg', res)
logger.debug(res)
if res['rc']> 0:
tinctest.logger.info('result from install package %s' % res['stdout'])
raise Exception('Failed to install the package')
self.check_and_install_sql(res['stdout'])
else:
raise Exception("*** ERROR: .gppkg file not found '. Make sure %s exists." % (pkgfile))
def run_gppkg_uninstall(self, pkgname):
"""
@summary: Runs gppkg -r to uninstall a gppkg. Output is written to gppkg_r.log file in current directory.
@param pkgfile: The name of the .gppkg file
@raise GppkgUtilError: If gppkg uninstall fails
"""
(existed, pkg) = self.check_pkg_exists(pkgname)
if not existed:
logger.info('the package does not exist, no need to remove, %s'%pkgname)
return True
logger.debug( '\nGppkgUtil: Uninstalling gppkg using gppkg file: %s' % (pkg))
cmd = 'gppkg -r %s' % pkg
res = {'rc':0, 'stderr':'', 'stdout':''}
run_shell_command (cmd, 'run gppkg', res)
logger.debug(res)
if res['rc']> 0:
logger.info('Failed to Uninstall the package, %s' % pkgname)
return False
else:
return True
def run_gppkg_uninstall(self, pkgname):
"""
@summary: Runs gppkg -r to uninstall a gppkg. Output is written to gppkg_r.log file in current directory.
@param pkgfile: The name of the .gppkg file
@raise GppkgUtilError: If gppkg uninstall fails
"""
(existed, pkg) = self.check_pkg_exists(pkgname)
if not existed:
logger.info('the package does not exist, no need to remove, %s'%pkgname)
return True
logger.debug( '\nGppkgUtil: Uninstalling gppkg using gppkg file: %s' % (pkg))
cmd = 'gppkg -r %s' % pkg
res = {'rc':0, 'stderr':'', 'stdout':''}
run_shell_command (cmd, 'run gppkg', res)
logger.debug(res)
if res['rc']> 0:
logger.info('Failed to Uninstall the package, %s' % pkgname)
return False
else:
return True
def get_os_platform(self):
from sys import platform as _platform
machine = ''
if _platform == 'linux' or _platform == 'linux2': # Both SuSE and RHEL returns linux
if os.path.exists("/etc/SuSE-release"):
machine = 'suse'
else:
machine = 'redhat'
elif _platform == 'sunos5':
machine = 'solaris'
if not machine:
raise Exception('unable to determine the platform')
cmd = 'cat '
res = {'rc':0, 'stderr':'', 'stdout':''}
if machine.lower() == 'suse':
cmd = cmd + '/etc/SuSE-release'
run_shell_command (cmd, 'check os kernel version', res)
if 'SUSE Linux Enterprise Server 11' in res['stdout']:
os_ = 'suse11'
elif 'SUSE Linux Enterprise Server 10' in res['stdout']:
os_ = 'suse10'
elif machine.lower() == 'redhat':
cmd = cmd + '/etc/redhat-release'
run_shell_command (cmd, 'check os kernel version', res)
if 'Linux Server release 5.' in res['stdout']:
os_ = 'rhel5'
elif 'Linux Server release 6.' in res['stdout']:
os_ = 'rhel6'
elif machine.lower() == 'solaris':
cmd = cmd + '/etc/release'
run_shell_command (cmd, 'check os kernel version', res)
if 'Solaris 10' in res['stdout']:
os_ = 'sol10'
elif 'Solaris 11' in res['stdout']:
os_ = 'sol11'
logger.debug(res['stdout'])
return os_, platform.machine()
def get_os_platform(self):
from sys import platform as _platform
machine = ''
if _platform == 'linux' or _platform == 'linux2': # Both SuSE and RHEL returns linux
if os.path.exists("/etc/SuSE-release"):
machine = 'suse'
else:
machine = 'redhat'
elif _platform == 'sunos5':
machine = 'solaris'
if not machine:
raise Exception('unable to determine the platform')
cmd = 'cat '
res = {'rc': 0, 'stderr': '', 'stdout': ''}
if machine.lower() == 'suse':
cmd = cmd + '/etc/SuSE-release'
run_shell_command(cmd, 'check os kernel version', res)
if 'SUSE Linux Enterprise Server 11' in res['stdout']:
os_ = 'suse11'
elif 'SUSE Linux Enterprise Server 10' in res['stdout']:
os_ = 'suse10'
elif machine.lower() == 'redhat':
cmd = cmd + '/etc/redhat-release'
run_shell_command(cmd, 'check os kernel version', res)
if 'Linux Server release 5.' in res['stdout']:
os_ = 'rhel5'
elif 'Linux Server release 6.' in res['stdout']:
os_ = 'rhel6'
elif machine.lower() == 'solaris':
cmd = cmd + '/etc/release'
run_shell_command(cmd, 'check os kernel version', res)
if 'Solaris 10' in res['stdout']:
os_ = 'sol10'
elif 'Solaris 11' in res['stdout']:
os_ = 'sol11'
logger.debug(res['stdout'])
return os_, platform.machine()
