def test_extensions_prepare(self, request):
"""
Scenario:
1. Install current version
:return:
"""
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
# Step 1
pginst = PgInstall(product=name, edition=edition,
version=version, milestone=milestone,
branch=branch, windows=(self.system == 'Windows'))
request.cls.pginst = pginst
pginst.setup_repo()
print("Running on %s." % target)
if pginst.os.is_altlinux() and pginst.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
if self.system != 'Windows':
pginst.install_full()
pginst.initdb_start()
else:
pginst.install_postgres_win()
def test_clean_install(self, request):
"""
Scenario:
1. Install current version
2. Check that setup successfull (select version)
:return:
"""
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
# Step 1
pginst = PgInstall(product=name,
edition=edition,
version=version,
milestone=milestone,
branch=branch,
windows=(self.system == 'Windows'))
if pginst.os.is_altlinux() and pginst.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
request.cls.pginst = pginst
pginst.setup_repo()
print("Running on %s." % target)
print("Minor product version is: %s\n" %
pginst.get_product_minor_version())
if self.system != 'Windows':
pginst.install_base()
pginst.initdb_start()
else:
pginst.install_postgres_win()
server_version = pginst.get_server_version()
client_version = pginst.get_psql_version()
print("Server version:\n%s\nClient version:\n%s" %
(server_version, client_version))
print("OK")
def test_hotstandby_compat(self, request):
"""
Scenario:
1. Install current version
2. Check that setup successfull (select version)
:return:
"""
global windows_os
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
windows_os = True
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
if name != 'postgrespro':
print("Hot Standby compatibility test is only for postgrespro.")
return
if edition == "ent":
archive_url = PGPRO_ARCHIVE_ENTERPRISE
elif edition == "std":
archive_url = PGPRO_ARCHIVE_STANDARD
else:
raise Exception("Unsupported postgrespro edition (%s)." % edition)
print("Running on %s." % target)
# Choose two versions -- newest and oldest supported
soup = get_soup(archive_url)
arcversions = []
startswith = 'pgproee-' if edition == 'ent' else \
('pgpro-' if edition == 'std' else 'pg1c-')
for link in soup.findAll('a'):
href = link.get('href')
if href.startswith(startswith) and href.endswith('/'):
vere = re.search(r'\w+-([0-9.]+)/', href)
if vere:
if vere.group(1).startswith(version):
arcvers = vere.group(1)
if version == '9.6':
# Due to CATALOG_VERSION_NO change
# we don't support lower 9.6 versions
if compare_versions(arcvers, '9.6.4.1') < 0:
arcvers = None
# PGPRO-3227, PGPRO-3834
if windows_os and version == '10':
if compare_versions(arcvers, '10.11.1'):
arcvers = None
if windows_os and version == '11':
if compare_versions(arcvers, '11.6.1') < 0:
arcvers = None
if arcvers:
arcversions.append(arcvers)
arcversions.sort(key=extend_ver)
if not arcversions:
print("No previous minor versions found. Test skipped.")
return
# Choose first and last versions
testversions = [arcversions[0], arcversions[-1]]
if testversions[0] == testversions[1]:
testversions = [testversions[0]]
# Workaround for unsupported libpq options
fix_extra_libpq_options = False
if edition == 'ent' and version == '10':
if compare_versions(arcversions[0], '10.4.1') < 0:
fix_extra_libpq_options = True
pre12version = version in ['9.6', '10', '11']
if windows_os:
waldir = r'C:\tmp\pgwal'
srcdir = r'C:\tmp'
else:
waldir = os.path.join(tempfile.gettempdir(), 'pgwal')
srcdir = '/var/src'
pgsrcdir = None
for oldversion in testversions:
print("Installing", oldversion)
pgold = PgInstall(product=name, edition=edition,
version=oldversion, milestone='archive',
branch=None, windows=windows_os)
pgold.setup_repo()
if not windows_os:
pgold.install_base()
pgold.initdb_start()
else:
pgold.install_postgres_win()
setup_pgpass('replicator', 'replicator')
server_version = pgold.get_server_version()
client_version = pgold.get_psql_version()
print("Old server version:\n%s\nOld client version:\n%s" %
(server_version, client_version))
pgold.exec_psql('ALTER SYSTEM SET port=15432')
oldpgprefix = pgold.get_pg_prefix()
olddatadir = pgold.get_datadir()
if oldpgprefix == '/usr':
raise Exception("/usr as postgres prefix is not supported.")
if pgold.get_configdir() != pgold.get_datadir():
raise Exception("Separate config dir is not supported.")
pgold.stop_service()
time.sleep(5)
if not windows_os:
subprocess.check_call('cp -a "%s" "%s.old"' %
(oldpgprefix, oldpgprefix), shell=True)
subprocess.check_call('cp -a "%s" "%s.old"' %
(olddatadir, olddatadir), shell=True)
oldpgprefix += ".old"
olddatadir += ".old"
else:
print('xcopy /S /E /O /X /I /Q "%s" "%s.old"' %
(oldpgprefix, oldpgprefix))
subprocess.check_call('xcopy /S /E /O /X /I /Q "%s" "%s.old"' %
(oldpgprefix, oldpgprefix), shell=True)
oldpgprefix += ".old"
olddatadir = os.path.join(oldpgprefix, 'data')
if os.path.exists(os.path.join(olddatadir,
'postgresql.conf.old')):
os.remove(os.path.join(olddatadir, 'postgresql.conf.old'))
pgold.remove_full(remove_data=True)
pgold.pg_prefix = oldpgprefix
pgold.datadir = olddatadir
pgold.configdir = olddatadir
pgold.port = 15432
if not windows_os:
pgold.pg_preexec = 'sudo -u postgres ' \
'LD_LIBRARY_PATH=${LD_LIBRARY_PATH} '
old_env = os.environ.copy()
old_env["LD_LIBRARY_PATH"] = os.path.join(oldpgprefix, 'lib')
pgold.env = old_env
else:
subprocess.check_call(
'sc create postgres-old binpath= '
'"\\"%s\\" runservice -N postgres-old -D \\"%s\\" -w"'
' start= demand obj= "NT Authority\\NetworkService" ' %
(os.path.join(oldpgprefix, 'bin', 'pg_ctl'), olddatadir),
shell=True)
pgold.service_name = 'postgres-old'
pgnew = PgInstall(product=name, edition=edition,
version=version, milestone=milestone,
branch=branch, windows=windows_os)
pgnew.setup_repo()
if not windows_os:
pgnew.install_base()
pgnew.initdb_start()
else:
pgnew.install_postgres_win()
if not pgsrcdir:
pgsrcdir = prepare_pg_regress(pgnew, srcdir)
pgnew.stop_service()
pgnew.remove_data()
# Test replication from old to new
setup_sender(pgold, waldir, pgnew.get_datadir())
start_receiver(pgnew, waldir, pre12version)
run_hs_test(pgold, pgnew, pgsrcdir)
do_server_action(pgnew, "stop")
do_server_action(pgold, "stop")
# Test replication from new to old
pgnew.init_cluster(force_remove=True)
pgold.remove_data()
setup_sender(pgnew, waldir, pgold.get_datadir())
if fix_extra_libpq_options:
workaround_for_extra_libpq_options(pgold)
start_receiver(pgold, waldir, pre12version)
run_hs_test(pgnew, pgold, pgsrcdir)
do_server_action(pgnew, "stop")
do_server_action(pgold, "stop")
pgnew.remove_full(remove_data=True)
shutil.rmtree(olddatadir)
shutil.rmtree(oldpgprefix)
if windows_os:
subprocess.check_call('sc delete postgres-old', shell=True)
print("OK")
def test_full_install(self, request):
"""
Scenario:
1. Install current version
2. Check that setup successfull
:return:
"""
dist = ""
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
request.cls.pgid = '%s-%s' % (edition, version)
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
# Step 1
pginst = PgInstall(product=name,
edition=edition,
version=version,
milestone=milestone,
branch=branch,
windows=(self.system == 'Windows'))
# This is a workaround for the problem described in PGPRO-3596
if pginst.os.is_altlinux() and pginst.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
request.cls.pginst = pginst
pginst.setup_repo()
print("Running on %s." % target)
all_available_packages = pginst.all_packages_in_repo
print("All available packages in repo %s:\n" % pginst.reponame,
"\n".join(all_available_packages))
if self.system != 'Windows':
pginst.install_full()
pginst.install_package(" ".join(all_available_packages))
check_package_contents(pginst, all_available_packages)
check_executables(pginst, all_available_packages)
pginst.initdb_start()
else:
pginst.install_perl_win()
pginst.install_postgres_win()
server_version = pginst.get_server_version()
client_version = pginst.get_psql_version()
print("Server version:\n%s\nClient version:\n%s" %
(server_version, client_version))
if name == 'postgrespro' and not (edition in ['1c', 'sql']):
ppversion = pginst.exec_psql_select("SELECT pgpro_version()")
# PGPRO-3760
assert ppversion.startswith('PostgresPro ')
# assert ppversion.startswith('PostgresPro ' + version)
ppedition = pginst.exec_psql_select("SELECT pgpro_edition()")
if edition == 'ent':
assert ppedition == 'enterprise'
elif edition == 'ent-cert':
assert ppedition == 'enterprise'
else:
assert ppedition == 'standard'
print('pgpro_source_id:',
pginst.exec_psql_select("SELECT pgpro_build()"))
if version not in ["9.6", "10", "13"]:
pginst.env = {}
for var in os.environ:
pginst.env[var] = str(os.environ[var])
pginst.env["LANG"] = 'C'
# PGPRO-4100 TODO: Use pgpro_controldata
cdout = pginst.exec_server_bin('pg_controldata', '"%s"' %
pginst.get_datadir()).split('\n')
if name == 'postgrespro' and not (edition in ['1c', 'sql']):
assert cdout[0].startswith('pg_control edition:')
cdedition = cdout[0].replace('pg_control edition:', '').strip()
if edition == 'ent':
assert cdedition == 'Postgres Pro Enterprise'
elif edition == 'std':
assert cdedition == 'Postgres Pro Standard'
print("OK")
def test_dev_usage(self, request):
"""
Scenario:
1. Install only -dev package
2. Try to build PGXS extension
:return:
"""
dist = ""
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
# Step 1
pginst = PgInstall(product=name,
edition=edition,
version=version,
milestone=milestone,
branch=branch,
windows=(self.system == 'Windows'))
pginst.setup_repo()
print("Running on %s." % target)
if pginst.os.is_altlinux() and pginst.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
if self.system != 'Windows':
pginst.install_server_dev()
pg_bin_path = pginst.get_default_bin_path()
curpath = os.path.dirname(os.path.abspath(__file__))
test_script = r"""
set -e
if which apt-get; then
apt-get install -y gcc || true
apt-get install -y make
grep -E '(Debian GNU/Linux 9|Debian GNU/Linux 10|'\
'"Ubuntu [0-9]+\.[0-9]+|"OSNova Linux|'\
'"Astra Linux \(Smolensk 1.6\)"|"Astra Linux \(Orel\)")'\
/etc/os-release >/dev/null 2>/dev/null && \
apt install -y libdpkg-perl
elif which zypper; then
zypper install -y gcc make
elif which yum; then
yum install -y gcc make redhat-rpm-config
fi
tar fax ../extras/pg_wait_sampling.tar.gz -C /tmp && \
cd /tmp/pg_wait_sampling*/
export PATH=%s:$PATH
make USE_PGXS=1
make USE_PGXS=1 install
chmod 777 .
""" % (pg_bin_path)
subprocess.check_call(test_script, cwd=curpath, shell=True)
pginst.install_full()
pginst.initdb_start()
pginst.exec_psql('ALTER SYSTEM SET shared_preload_libraries = '
'pg_wait_sampling')
pginst.restart_service()
test_script = r"""
cd /tmp/pg_wait_sampling*/
sudo -u postgres sh -c "export PATH=%s:$PATH; make USE_PGXS=1 installcheck"
""" % (pg_bin_path)
subprocess.check_call(test_script, shell=True)
else:
pginst.install_postgres_win()
pginst.install_perl_win()
print("OK")
def test_make_check(self, request):
"""
Scenario:
1. Install current version
2. Check that setup successfull
We need to perform the test on Windows in two stages:
First we setup the server and prepare environment,
then we exclude current user from the Administrators group.
Second we execute `make installcheck` without admin rights.
:return:
"""
dist = ""
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
pgid = '%s-%s' % (edition, version)
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
# Step 1
pginst = PgInstall(product=name,
edition=edition,
version=version,
milestone=milestone,
branch=branch,
windows=(self.system == 'Windows'))
request.cls.pginst = pginst
pginst.make_check_passed = False
curpath = os.path.dirname(os.path.abspath(__file__))
if pginst.os.is_altlinux() and pginst.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
if self.system == 'Windows':
if os.path.exists(pginst.get_default_bin_path()):
# Refresh environment to get correct PYTHONHOME
refresh_env_win()
# The instance is already installed and
# installcheck environment is presumably prepared,
# so just run make_installcheck (once more)
subprocess.check_call(
'"%s" "%s" "%s"' %
(os.path.join(curpath, 'make_installcheck.cmd'),
get_pg_prefix(pginst), pginst.service_name),
shell=True)
pginst.make_check_passed = True
return
pginst.setup_repo()
print("Running on %s." % target)
tarball = pginst.download_source()
tar = tarfile.open(tarball, 'r:bz2')
tar.extractall()
tar.close()
for comp in [
'orafce', 'plv8', 'pgpro-stats', 'pgpro-pwr',
'pgpro-controldata', 'pg-filedump', 'pg-portal-modify',
'pg-repack'
]:
pkgname = '%s-%s-%s' % (comp, edition, version)
if pkgname not in pginst.get_packages_in_repo():
pkgname = comp
if pkgname not in pginst.get_packages_in_repo():
continue
pginst.download_source(comp.replace('-', '_'),
pginst.get_package_version(pkgname),
'tar.gz')
if self.system != 'Windows':
pginst.install_full()
pginst.initdb_start()
else:
pginst.install_perl_win()
pginst.install_postgres_win(port=55432)
try:
pginst.exec_psql('CREATE EXTENSION plpython3u')
except Exception:
subprocess.check_call('SETX PYTHONHOME C:\\Python27 -m',
shell=True)
if version != "9.6" or self.system == 'Windows' or \
(edition == '1c' and pginst.os_name not in DEBIAN_BASED):
buildinfo = os.path.join(pginst.get_pg_prefix(), 'doc',
'buildinfo.txt')
else:
buildinfo = subprocess.check_output(
'ls /usr/share/doc/postgres*pro*/buildinfo.txt',
shell=True).decode(ConsoleEncoding).strip()
with open(buildinfo, 'r') as bi:
bitxt = bi.read()
assert (re.search(r'^Documentation translation', bitxt,
re.MULTILINE))
assert (re.search(r'^Source', bitxt, re.MULTILINE))
assert (re.search(r'^SPEC', bitxt, re.MULTILINE))
print("The binary package buildinfo:\n%s\n" % bitxt)
pginst.install_default_config()
pginst.exec_psql("ALTER SYSTEM SET max_worker_processes = 16")
pginst.exec_psql("ALTER SYSTEM SET lc_messages = 'C'")
# Prepare pg_hba.conf for src/interfaces/ecpg/test/connect/test5
with open(os.path.join(pginst.get_configdir(), 'pg_hba.conf'),
'r+') as conf:
hba = re.sub(r'^(local\s+all\s+all\s+peer)$',
"local all regress_ecpg_user1 md5\n"
"local all regress_ecpg_user2 md5\n"
"local all regress_hacker trust\n"
"local all regress_superuser trust\n"
"local all nosuper trust\n"
r'\1',
conf.read(),
flags=re.MULTILINE)
conf.seek(0)
conf.write(hba)
pginst.load_shared_libraries(restart_service=False)
pginst.restart_service()
if self.system != 'Windows':
subprocess.check_call(
'"%s" "%s" "%s"' %
(os.path.join(curpath, 'make_installcheck.sh'),
get_pg_prefix(pginst), pginst.service_name),
shell=True)
pginst.make_check_passed = True
else:
# First run is performed to setup the environment
subprocess.check_call(
'"%s" "%s" "%s"' %
(os.path.join(curpath, 'make_installcheck.cmd'),
get_pg_prefix(pginst), pginst.service_name),
shell=True)
request.session.customexitstatus = 222
def test_upgrade_minor(self, request):
"""
Scenario:
1. Install current version
2. Check that setup successfull (select version)
:return:
"""
global windows_os
distro = get_distro()
if distro[2] == 'x86_64' or self.system == 'Windows':
distro = distro[:-1]
dist = " ".join(distro)
if self.system == 'Linux':
windows_os = False
elif self.system == 'Windows':
windows_os = True
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
if edition not in ['std', 'ent', '1c']:
print("Minor upgrade only for 1c, std and ent")
return
if name != 'postgrespro':
print("Minor upgrade test is only for postgrespro.")
return
small_key = "-".join([name, edition, version])
specified_version = False
if dist in ARCHIVE_VERSIONS \
and small_key in ARCHIVE_VERSIONS[dist]:
specified_version = ARCHIVE_VERSIONS[dist][small_key]
if specified_version is None:
return "%s %s %s does not support archived versions on %s." % \
(name, edition, version, dist)
print("specified version is %s" % specified_version)
print("Running on %s." % target)
pgnew = PgInstall(product=name,
edition=edition,
version=version,
milestone=milestone,
branch=branch,
windows=windows_os)
if pgnew.os.is_altlinux() and pgnew.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
pgnew.setup_repo()
if not windows_os:
pgnew.install_full()
subprocess.check_call('cp -a "%s" "%s"' %
(pgnew.get_pg_prefix(), client_dir),
shell=True)
pgnew.remove_full(False, True)
pgnew.remove_data(True)
# PGPRO-3310
if pgnew.os_name in DEBIAN_BASED:
remove_alternatives()
else:
pgnew.install_postgres_win()
pgnew.stop_service()
subprocess.check_call('xcopy /S /E /O /X /I /Q "%s" "%s"' %
(pgnew.get_pg_prefix(), client_dir),
shell=True)
pgnew.remove_full(True)
pgconfig = subprocess.check_output(
'"%s"' % os.path.join(client_dir, 'bin', 'pg_config'),
shell=True).decode(ConsoleEncoding)
vere = re.search(r'PGPRO\_VERSION\s=\s([0-9.]+)', pgconfig)
if (vere):
current_ver = vere.group(1)
else:
vere = re.search(r'VERSION\s=\s\w+\s([0-9.]+)', pgconfig)
current_ver = vere.group(1)
print("Current version is %s" % current_ver)
test_versions = get_test_versions(edition, version, specified_version,
current_ver)
if test_versions is None:
print("No archive versions found.")
return
print(test_versions)
dump_file_name = download_dump(name, edition, version + '-old',
tempdir)
for oldversion in test_versions:
print("Installing", oldversion)
key = "-".join([name, edition, oldversion])
pgold = PgInstall(product=name,
edition=edition,
version=oldversion,
milestone='archive',
branch=None,
windows=windows_os)
if pgold.os.is_altlinux() and pgold.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
pgold.setup_repo()
if not windows_os:
# PGPRO-3889
if (pgold.os_name.startswith('CentOS') or
pgold.os_name.startswith('Red Hat') or
pgold.os_name.startswith('Oracle Linux')) and \
pgold.os_version.startswith('8'):
for pkg in pgold.all_packages_in_repo[:]:
if ('jit' in pkg):
pgold.all_packages_in_repo.remove(pkg)
if (pgold.os_name.startswith('SLES')
and pgold.os_version.startswith('15')):
for pkg in pgold.all_packages_in_repo[:]:
if 'zstd' in pkg:
pgold.all_packages_in_repo.remove(pkg)
# PGPRO-2954
for pkg in pgold.all_packages_in_repo[:]:
if 'bouncer' in pkg or 'badger' in pkg:
pgold.all_packages_in_repo.remove(pkg)
if pgnew.os_name == 'ROSA Enterprise Linux Server' \
and pgnew.os_version.startswith('7.3') \
and edition == 'std' \
and version == '9.6' \
and compare_versions(oldversion, '9.6.13.1') == 0:
for pkg in pgold.all_packages_in_repo[:]:
if 'probackup' in pkg:
pgold.all_packages_in_repo.remove(pkg)
pgold.install_full()
pgold.initdb_start()
else:
pgold.install_postgres_win()
pgold.load_shared_libraries()
with open(os.path.join(tempdir, 'load-%s.log' % oldversion),
'wb') as out:
pgold.exec_psql_file(
dump_file_name,
'-q%s' %
('' if pgold.os_arch == 'x86' else ' -v ON_ERROR_STOP=1'),
stdout=out)
expected_file_name = os.path.join(tempdir, "%s-expected.sql" % key)
dumpall(pgold, expected_file_name)
pgold.delete_repo()
pgnew = PgInstall(product=name,
edition=edition,
version=version,
milestone=milestone,
branch=None,
windows=windows_os)
if pgnew.os.is_altlinux() and pgnew.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
pgnew.setup_repo()
pgold.stop_service()
if not windows_os:
pgnew.update_all_packages()
pgnew.start_service()
else:
pgnew.install_postgres_win()
result_file_name = os.path.join(tempdir, "%s-result.sql" % key)
dumpall(pgnew, result_file_name)
diff_dbs(expected_file_name, result_file_name,
os.path.join(tempdir, "%s.sql.diff" % key))
pgnew.stop_service()
repo_diff = list(
set(pgold.all_packages_in_repo) -
set(pgnew.all_packages_in_repo))
print("repo diff is %s" % repo_diff)
for package in repo_diff:
try:
pgold.remove_package(package)
except Exception:
pass
pgnew.remove_full(True)
# PGPRO-3310
if pgnew.os_name in DEBIAN_BASED:
remove_alternatives()
if pgold.os_name in DEBIAN_BASED and version == '9.6':
try:
subprocess.check_call("apt-get purge -y 'postgres*'",
shell=True)
except Exception:
pass
# PGPRO-2563
if pgold.os_name == 'Ubuntu' and version == '9.6' and \
edition == 'ent':
time.sleep(20)
def install_server(product,
edition,
version,
milestone,
branch,
windows,
old=False):
pg = PgInstall(product=product,
edition=edition,
version=version,
milestone=milestone,
branch=branch,
windows=windows)
pg.setup_repo()
if pg.os.is_altlinux() and pg.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
if not windows:
if old and pg.os_name == 'SLES' and pg.os_version.startswith('12.'):
for pkg in pg.all_packages_in_repo[:]:
if ('libzstd' in pkg):
pg.all_packages_in_repo.remove(pkg)
# PGPRO-3889
if (pg.os_name.startswith('CentOS') or
pg.os_name.startswith('Red Hat') or
pg.os_name.startswith('Oracle Linux')) and \
pg.os_version.startswith('8'):
for pkg in pg.all_packages_in_repo[:]:
if ('jit' in pkg):
pg.all_packages_in_repo.remove(pkg)
pg.install_full_topless()
# PGPRO-2136
if pg.os_name in ALT_BASED:
with open('/etc/sysconfig/i18n', 'r') as file:
for line in file:
kv = line.split('=')
if len(kv) == 2:
os.environ[kv[0]] = kv[1].strip()
else:
pg.install_postgres_win()
pg.client_path_needed = True
pg.server_path_needed = True
pg.install_default_config()
pg.load_shared_libraries()
return pg
def test_multimaster_install(self, request):
"""
Scenario:
1. Install current version
2. Check that setup successfull (select version)
:return:
"""
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
if version.startswith('9.') or version == '10' \
or version == '13' \
or not edition.startswith('ent'):
print('Version %s %s is not supported' % (edition, version))
return
elif version == '11' and dist == 'Windows':
print('Windows is not supported under version %s' % version)
return
# Step 1
pginst = PgInstall(product=name,
edition=edition,
version=version,
milestone=milestone,
branch=branch,
windows=(self.system == 'Windows'))
request.cls.pginst = pginst
if pginst.os.is_altlinux() and pginst.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
pginst.setup_repo()
print("Running on %s." % target)
if self.system != 'Windows':
pginst.install_full_topless()
else:
pginst.install_postgres_win()
pginst.stop_service()
mm = Multimaster(size=2,
pginst=pginst,
rootdir=os.path.abspath(
os.path.join(pginst.get_datadir(), os.pardir)))
mm.start()
for i, cl_state in mm.get_cluster_state_all().items():
if int(cl_state[0]) != i:
raise Exception('Wrong ID on node %i! %i != %i' %
(i, i, cl_state[0]))
if cl_state[1].lower() != 'online':
print(cl_state)
raise Exception('Node %i is not online!' % i)
print(mm.get_nodes_state_all())
for i, ns_all in mm.get_nodes_state_all().items():
for node_state in ns_all:
if node_state[1].rstrip() != 't':
raise Exception('Node %s is not enabled! (on node %i)' %
(node_state[0], i))
pgbench = {}
for i, node in mm.nodes.items():
pgbench[i] = mm.pgbench(i,
type='simple-update',
duration=60,
max_tries=100)
pgbench[1].init()
for i in range(1, mm.size + 1):
if not mm.nodes[i].referee:
print('Running pgbench on node%i' % i)
pgbench[i].start()
mm.wait_for_txid(600)
print('Cluster state:')
print(mm.get_cluster_state_all())
print(mm.get_nodes_state_all())
print('Current TXID (after pgbench): %s' %
mm.psql('SELECT txid_current()', '-Aqt'))
print('Isolating node 2...')
mm.isolate(2)
if mm.has_referee:
mm.wait_for_referee(1, 60)
pgbench[1].wait()
pgbench[1].start()
mm.wait_for_txid(2700)
print('Cluster state:')
print(mm.get_cluster_state_all(allow_fail=True))
print(mm.get_nodes_state_all(allow_fail=True))
print('De-isolating node 2...')
mm.deisolate(2)
print('Current TXID (after isolation): %s' %
mm.psql('SELECT txid_current()', '-Aqt'))
if not mm.check(2):
print('Try to recover node 2')
mm.wait(2, 600)
print('Cluster state:')
print(mm.get_cluster_state_all(allow_fail=True))
print(mm.get_nodes_state_all(allow_fail=True))
mm.wait_for_txid(3500)
for i in range(1, mm.size):
if not mm.nodes[i].referee:
print('Pgbench %i terminated rc=%i' % (i, pgbench[i].stop()))
mm.sure_pgbench_is_dead(i)
pgbench_tables = ('pgbench_branches', 'pgbench_tellers',
'pgbench_accounts', 'pgbench_history')
print('Current TXID (after pgbench termination): %s' %
mm.psql('SELECT txid_current()', '-Aqt'))
for table in pgbench_tables:
result1 = mm.pg_dump(1, [table])
for i in range(2, mm.size + 1):
if not mm.nodes[i].referee:
result = mm.pg_dump(i, [table])
diff_dbs(result1, result, '%s_1_%i.diff' % (table, i))
def test_install(self, request):
""" Install pg_probackup utility and configure postgresql
for running pg_probackup.
Scenario:
1. Install postgrespro version
2. Check version function
3. Check help function
"""
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
request.cls.skip = False
if name != 'postgrespro' or edition == '1c' or version == '13':
# TODO: Enable this test for v13
print("PgProBackup test is only for postgrespro std and ent.")
request.cls.skip = True
return
# Step 1
self.pginst = PgInstall(product=name, edition=edition, version=version,
milestone=milestone, branch=branch,
windows=(self.system == 'Windows'))
self.pginst.setup_repo()
if self.pginst.os.is_altlinux()and self.pginst.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
if self.system != 'Windows':
self.pginst.install_full()
self.pginst.initdb_start()
else:
self.pginst.install_postgres_win()
self.fix_permissions(tempdir)
self.bindir = self.pginst.get_bin_path() + os.sep
print('Bindir is %s' % self.bindir)
self.pginst.load_shared_libraries(restart_service=True)
request.cls.pginst = self.pginst
# Step 2
assert self.execute_pg_probackup("--help")
ver = self.execute_pg_probackup("--version")
print('Version is %s' % ver)
vere = re.search(r'[^(]+\s([0-9.]+)\s.*', ver)
self.version = vere.group(1)
assert self.version
request.cls.version = self.version
class TestPgprobackup():
system = platform.system()
bindir = ''
def fix_permissions(self, dir):
if self.system != 'Windows':
import pwd
import grp
os.chown(dir,
pwd.getpwnam("postgres").pw_uid,
grp.getgrnam("postgres").gr_gid)
for root, dirs, files in os.walk(dir):
for d in dirs:
os.chown(os.path.join(root, d),
pwd.getpwnam("postgres").pw_uid,
grp.getgrnam("postgres").gr_gid)
for f in files:
os.chown(os.path.join(root, f),
pwd.getpwnam("postgres").pw_uid,
grp.getgrnam("postgres").gr_gid)
else:
# Grant Full Access to "Network Service" and Users
subprocess.check_call(
'icacls "%s" /grant *S-1-5-32-545:(OI)(CI)F /T' % dir,
shell=True)
subprocess.check_call(
'icacls "%s" /grant *S-1-5-20:(OI)(CI)F /T' % dir,
shell=True)
subprocess.check_call(
r'icacls "%s\*" /grant *S-1-5-32-545:(OI)(CI)F /T' % dir,
shell=True)
subprocess.check_call(
r'icacls "%s\*" /grant *S-1-5-20:(OI)(CI)F /T' % dir,
shell=True)
def execute_pg_probackup(self, *options):
"""
:param options: strings with params for pg_probackup
:return: output of command execution
"""
cmd = '%s"%spg_probackup" %s' % (
self.pginst.pg_sudo_cmd, self.bindir, " ".join(list(options)))
ret = subprocess.check_output(cmd, shell=True).decode()
return ret
def download_source(self):
if self.pginst.milestone == 'alpha':
baseurl = PGPRO_DEV_SOURCES_BASE
elif self.pginst.milestone == 'beta':
baseurl = PGPRO_STABLE_SOURCES_BASE
else:
baseurl = PGPRO_ARCHIVE_SOURCES_BASE
tar_href = 'pg_probackup-%s.tar.gz' % self.version
# tar_file = os.path.join(tempdir, tar_href)
urlretrieve(baseurl + '/' + tar_href, tar_href)
return tar_href
def test_install(self, request):
""" Install pg_probackup utility and configure postgresql
for running pg_probackup.
Scenario:
1. Install postgrespro version
2. Check version function
3. Check help function
"""
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
request.cls.skip = False
if name != 'postgrespro' or edition == '1c' or version == '13':
# TODO: Enable this test for v13
print("PgProBackup test is only for postgrespro std and ent.")
request.cls.skip = True
return
# Step 1
self.pginst = PgInstall(product=name, edition=edition, version=version,
milestone=milestone, branch=branch,
windows=(self.system == 'Windows'))
self.pginst.setup_repo()
if self.pginst.os.is_altlinux()and self.pginst.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
if self.system != 'Windows':
self.pginst.install_full()
self.pginst.initdb_start()
else:
self.pginst.install_postgres_win()
self.fix_permissions(tempdir)
self.bindir = self.pginst.get_bin_path() + os.sep
print('Bindir is %s' % self.bindir)
self.pginst.load_shared_libraries(restart_service=True)
request.cls.pginst = self.pginst
# Step 2
assert self.execute_pg_probackup("--help")
ver = self.execute_pg_probackup("--version")
print('Version is %s' % ver)
vere = re.search(r'[^(]+\s([0-9.]+)\s.*', ver)
self.version = vere.group(1)
assert self.version
request.cls.version = self.version
def test_pgprobackup_internal(self, request):
if request.cls.skip:
print("PgProBackup test is only for postgrespro std and ent.")
return
self.pginst = request.cls.pginst
if sys.version_info > (3, 0) and \
compare_versions(self.version, '2.4.5') < 0:
print("Only 2 python temporary supported")
return
if self.system == 'Windows':
print("Only linuxes temporary supported")
return
self.version = request.cls.version
self.bindir = self.pginst.get_bin_path() + os.sep
os.environ['PATH'] = os.environ['PATH'] + ':' + self.bindir
tar_file = self.download_source()
print(tar_file)
tar = tarfile.open(tar_file, "r:gz")
tar.extractall()
tar.close()
dir = '.'.join(tar_file.split('.')[:-2])
print(dir)
# Patch tests for 2.4.2
if self.version == '2.4.2':
urlretrieve('https://github.com/postgrespro/pg_probackup/raw/8147'
'001/tests/incr_restore.py',
os.path.join(dir, 'tests', 'incr_restore.py'))
self.fix_permissions(dir)
subprocess.check_call('pip%s install testgres%s' %
(sys.version_info[0],
'==1.8.2' if sys.version_info[0] == 2 else ''),
shell=True)
# PGPRO-4108 wait ptrack2.0 in 10
cmd = "%s sh -c 'PG_CONFIG=\"%s/pg_config\"" \
" LANG=C PG_PROBACKUP_PTRACK=%s " \
" PG_PROBACKUP_TEST_BASIC=ON python%s -m unittest -v tests'" \
% (self.pginst.pg_sudo_cmd, self.pginst.get_bin_path(),
'ON' if compare_versions(self.pginst.version, '10') > 0
else 'OFF',
'2.7' if self.pginst.os_name in REDHAT_BASED and
self.pginst.os_version.startswith('6') else
sys.version_info[0])
print(subprocess.check_output(cmd, cwd=dir, shell=True).decode())
print("OK")
def test_pgprobackup_simple_backup_restore(self, request):
"""Test pg_probackup with compression feature
and full continous backup
Scenario:
1. Create backup dir
3. Init backup dir
4. Set options for pg_probackup
5. Create tablespace with compression
6. Create table in compression tablespace
7. Edit pg_hba_config
8. Make full backup
9. Get backup id
10. Check that backup status is OK
11. Check that backup validation is OK
"""
if request.cls.skip:
print("PgProBackup test is only for postgrespro std and ent.")
return
self.pginst = request.cls.pginst
self.bindir = self.pginst.get_bin_path() + os.sep
# Step 1
backup_dir = os.path.join(tempdir, 'backup')
os.mkdir(backup_dir)
self.fix_permissions(backup_dir)
# Step 3
self.execute_pg_probackup("init", "-B", '"%s"' % backup_dir)
self.execute_pg_probackup("add-instance", "-B", '"%s"' % backup_dir,
"-D", '"%s"' %
self.pginst.get_default_datadir(),
"--instance", "main")
# Step 5
if self.pginst.edition == 'ent':
self.pginst.exec_psql("ALTER SYSTEM SET cfs_gc_workers TO '0'")
if self.pginst.version == '9.6':
with open(os.path.join(self.pginst.get_default_configdir(),
"pg_hba.conf"), "a") as hba:
hba.write("host replication all all trust") \
if self.system == 'Windows' else \
hba.write("local replication all trust")
self.pginst.exec_psql("ALTER SYSTEM SET wal_level TO 'replica'")
self.pginst.exec_psql("ALTER SYSTEM SET max_wal_senders TO '1'")
self.pginst.restart_service()
dump_file_name = download_dump(self.pginst.product,
self.pginst.edition,
self.pginst.version, tempdir)
with open(os.path.join(tempdir, 'load-dump.log'), 'wb') as out:
self.pginst.exec_psql_file(dump_file_name, '-q',
stdout=out)
# Step 6
tablespace_path = os.path.join(tempdir, 'pgprobackup')
os.mkdir(tablespace_path)
self.fix_permissions(tablespace_path)
create_command = 'CREATE TABLESPACE pgprobackup ' \
'LOCATION \'%s\'%s;' % \
(tablespace_path,
' WITH(compression = true)' if
self.pginst.edition == 'ent' else '')
self.pginst.exec_psql(create_command)
# Step 7
self.pginst.exec_psql(
"CREATE TABLE tbl TABLESPACE pgprobackup"
" AS SELECT i, rpad('',30,'a')"
" FROM generate_series(0,100000) AS i;")
before_backup_file = os.path.join(tempdir, 'before.sql')
self.pginst.do_in_all_dbs(truncate_unlogged_sql)
self.pginst.exec_client_bin('pg_dumpall', '-f "%s"' %
before_backup_file)
# Step 8
self.execute_pg_probackup("backup", "-b", "full", "-B",
'"%s"' % backup_dir, "-d", "postgres",
"-U", "postgres", "--instance",
"main", "--stream", "" if
self.pginst.get_default_datadir() ==
self.pginst.get_default_configdir()
else "--external-dirs=%s" %
self.pginst.get_default_configdir()
)
# Step 9
# Get last backup id and get out for show command with this backup
pgprobackup_show = json.loads(
self.execute_pg_probackup("show", "-B", '"%s"' % backup_dir, "-U",
"postgres", "--format=json"))
backup_id = pgprobackup_show[0]['backups'][0]['id']
# Step 10
assert pgprobackup_show[0]['backups'][0]['status'] == 'OK'
# Step 11
print("Validating backup %s" % backup_id)
self.execute_pg_probackup("validate", "-i", backup_id, "-B",
'"%s"' % backup_dir, '--instance', 'main')
# Step 12
print("Drop data and restore backup")
self.pginst.stop_service()
self.pginst.remove_data()
shutil.rmtree(tablespace_path)
self.execute_pg_probackup("restore", "-i", backup_id, "-B",
'"%s"' % backup_dir, '--instance', 'main')
# PBCKP-91
if self.system == 'Windows':
self.fix_permissions(self.pginst.get_default_datadir())
self.fix_permissions(tablespace_path)
self.pginst.start_service()
after_backup_file = os.path.join(tempdir, 'after.sql')
self.pginst.exec_client_bin('pg_dumpall', '-f "%s"' %
after_backup_file)
diff_dbs(before_backup_file, after_backup_file,
os.path.join(tempdir, 'diff.sql'))