def __init__(self, remote=False, host=None):
self.remote = remote
self.host = host
self.dist_info = get_distro(remote, host)
self.os_name = self.dist_info[0]
self.os_version = self.dist_info[1]
self.os_arch = self.dist_info[2]
def getDistribution():
if sys.platform == "win32":
return platform.win32_ver()
elif sys.platform == "linux" or sys.platform == "linux2":
distname, version, id = get_distro()
return "%s %s" % (distname, version)
elif sys.platform == "darwin":
release, versioninfo, machine = platform.mac_ver()
return "%s %s" % (release, versioninfo)
else:
return "Undefined"
def test_clean_install(self, request):
"""
Scenario:
1. Install current version
2. Check that setup successfull (select version)
:return:
"""
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
# Step 1
pginst = PgInstall(product=name,
edition=edition,
version=version,
milestone=milestone,
branch=branch,
windows=(self.system == 'Windows'))
if pginst.os.is_altlinux() and pginst.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
request.cls.pginst = pginst
pginst.setup_repo()
print("Running on %s." % target)
print("Minor product version is: %s\n" %
pginst.get_product_minor_version())
if self.system != 'Windows':
pginst.install_base()
pginst.initdb_start()
else:
pginst.install_postgres_win()
server_version = pginst.get_server_version()
client_version = pginst.get_psql_version()
print("Server version:\n%s\nClient version:\n%s" %
(server_version, client_version))
print("OK")
def test_extensions_prepare(self, request):
"""
Scenario:
1. Install current version
:return:
"""
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
# Step 1
pginst = PgInstall(product=name, edition=edition,
version=version, milestone=milestone,
branch=branch, windows=(self.system == 'Windows'))
request.cls.pginst = pginst
pginst.setup_repo()
print("Running on %s." % target)
if pginst.os.is_altlinux() and pginst.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
if self.system != 'Windows':
pginst.install_full()
pginst.initdb_start()
else:
pginst.install_postgres_win()
def test_hotstandby_compat(self, request):
"""
Scenario:
1. Install current version
2. Check that setup successfull (select version)
:return:
"""
global windows_os
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
windows_os = True
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
if name != 'postgrespro':
print("Hot Standby compatibility test is only for postgrespro.")
return
if edition == "ent":
archive_url = PGPRO_ARCHIVE_ENTERPRISE
elif edition == "std":
archive_url = PGPRO_ARCHIVE_STANDARD
else:
raise Exception("Unsupported postgrespro edition (%s)." % edition)
print("Running on %s." % target)
# Choose two versions -- newest and oldest supported
soup = get_soup(archive_url)
arcversions = []
startswith = 'pgproee-' if edition == 'ent' else \
('pgpro-' if edition == 'std' else 'pg1c-')
for link in soup.findAll('a'):
href = link.get('href')
if href.startswith(startswith) and href.endswith('/'):
vere = re.search(r'\w+-([0-9.]+)/', href)
if vere:
if vere.group(1).startswith(version):
arcvers = vere.group(1)
if version == '9.6':
# Due to CATALOG_VERSION_NO change
# we don't support lower 9.6 versions
if compare_versions(arcvers, '9.6.4.1') < 0:
arcvers = None
# PGPRO-3227, PGPRO-3834
if windows_os and version == '10':
if compare_versions(arcvers, '10.11.1'):
arcvers = None
if windows_os and version == '11':
if compare_versions(arcvers, '11.6.1') < 0:
arcvers = None
if arcvers:
arcversions.append(arcvers)
arcversions.sort(key=extend_ver)
if not arcversions:
print("No previous minor versions found. Test skipped.")
return
# Choose first and last versions
testversions = [arcversions[0], arcversions[-1]]
if testversions[0] == testversions[1]:
testversions = [testversions[0]]
# Workaround for unsupported libpq options
fix_extra_libpq_options = False
if edition == 'ent' and version == '10':
if compare_versions(arcversions[0], '10.4.1') < 0:
fix_extra_libpq_options = True
pre12version = version in ['9.6', '10', '11']
if windows_os:
waldir = r'C:\tmp\pgwal'
srcdir = r'C:\tmp'
else:
waldir = os.path.join(tempfile.gettempdir(), 'pgwal')
srcdir = '/var/src'
pgsrcdir = None
for oldversion in testversions:
print("Installing", oldversion)
pgold = PgInstall(product=name, edition=edition,
version=oldversion, milestone='archive',
branch=None, windows=windows_os)
pgold.setup_repo()
if not windows_os:
pgold.install_base()
pgold.initdb_start()
else:
pgold.install_postgres_win()
setup_pgpass('replicator', 'replicator')
server_version = pgold.get_server_version()
client_version = pgold.get_psql_version()
print("Old server version:\n%s\nOld client version:\n%s" %
(server_version, client_version))
pgold.exec_psql('ALTER SYSTEM SET port=15432')
oldpgprefix = pgold.get_pg_prefix()
olddatadir = pgold.get_datadir()
if oldpgprefix == '/usr':
raise Exception("/usr as postgres prefix is not supported.")
if pgold.get_configdir() != pgold.get_datadir():
raise Exception("Separate config dir is not supported.")
pgold.stop_service()
time.sleep(5)
if not windows_os:
subprocess.check_call('cp -a "%s" "%s.old"' %
(oldpgprefix, oldpgprefix), shell=True)
subprocess.check_call('cp -a "%s" "%s.old"' %
(olddatadir, olddatadir), shell=True)
oldpgprefix += ".old"
olddatadir += ".old"
else:
print('xcopy /S /E /O /X /I /Q "%s" "%s.old"' %
(oldpgprefix, oldpgprefix))
subprocess.check_call('xcopy /S /E /O /X /I /Q "%s" "%s.old"' %
(oldpgprefix, oldpgprefix), shell=True)
oldpgprefix += ".old"
olddatadir = os.path.join(oldpgprefix, 'data')
if os.path.exists(os.path.join(olddatadir,
'postgresql.conf.old')):
os.remove(os.path.join(olddatadir, 'postgresql.conf.old'))
pgold.remove_full(remove_data=True)
pgold.pg_prefix = oldpgprefix
pgold.datadir = olddatadir
pgold.configdir = olddatadir
pgold.port = 15432
if not windows_os:
pgold.pg_preexec = 'sudo -u postgres ' \
'LD_LIBRARY_PATH=${LD_LIBRARY_PATH} '
old_env = os.environ.copy()
old_env["LD_LIBRARY_PATH"] = os.path.join(oldpgprefix, 'lib')
pgold.env = old_env
else:
subprocess.check_call(
'sc create postgres-old binpath= '
'"\\"%s\\" runservice -N postgres-old -D \\"%s\\" -w"'
' start= demand obj= "NT Authority\\NetworkService" ' %
(os.path.join(oldpgprefix, 'bin', 'pg_ctl'), olddatadir),
shell=True)
pgold.service_name = 'postgres-old'
pgnew = PgInstall(product=name, edition=edition,
version=version, milestone=milestone,
branch=branch, windows=windows_os)
pgnew.setup_repo()
if not windows_os:
pgnew.install_base()
pgnew.initdb_start()
else:
pgnew.install_postgres_win()
if not pgsrcdir:
pgsrcdir = prepare_pg_regress(pgnew, srcdir)
pgnew.stop_service()
pgnew.remove_data()
# Test replication from old to new
setup_sender(pgold, waldir, pgnew.get_datadir())
start_receiver(pgnew, waldir, pre12version)
run_hs_test(pgold, pgnew, pgsrcdir)
do_server_action(pgnew, "stop")
do_server_action(pgold, "stop")
# Test replication from new to old
pgnew.init_cluster(force_remove=True)
pgold.remove_data()
setup_sender(pgnew, waldir, pgold.get_datadir())
if fix_extra_libpq_options:
workaround_for_extra_libpq_options(pgold)
start_receiver(pgold, waldir, pre12version)
run_hs_test(pgnew, pgold, pgsrcdir)
do_server_action(pgnew, "stop")
do_server_action(pgold, "stop")
pgnew.remove_full(remove_data=True)
shutil.rmtree(olddatadir)
shutil.rmtree(oldpgprefix)
if windows_os:
subprocess.check_call('sc delete postgres-old', shell=True)
print("OK")
class TestScram():
"""
Only Enterprise Edition Feature
"""
dist = ""
if platform.system() == 'Linux':
dist = " ".join(get_distro()[0:2])
elif platform.system() == 'Windows':
dist = 'Windows'
else:
print("Unknown Distro")
@staticmethod
def random_password():
return ''.join(random.choice(ascii_lowercase) for i in range(16))
@staticmethod
def create_hash_password(hash_type, password):
if hash_type == 'md5':
hash = hashlib.md5()
hash.update(password)
return 'md5' + hash.hexdigest()
elif hash_type == 'sha256':
hash = hashlib.sha256()
hash.update(password)
return 'AAAAAAAAAAAAAA==:4096:' + hash.hexdigest()
else:
print("Error. Bad hash type. Use md5 or sha256")
return None
@pytest.mark.test_scram_configuring
def test_scram_configuring(self, request):
"""Check that we can set GUC variables via SET command,
they saved and in pg_authid password saved in right format
Scenario:
1. Check that default password encryption is md5
2. Set password encryption is plain
3. Check that password encryption is plain
4. Create role test_plain_user with password 'test_plain_password'
5. Check from pg_authid that password for user test_plain_user
in plain and equal to 'test_plain_password'
6. Set password encrytpion is md5
7. Create role test_md5_user with password 'test_md5_password'
8. Check from pg_authid that password for user test_md5_user in md5
9. Set password encrytpion is scram
10. Create role test_scram_user with password 'test_scram_password'
11. Check from pg_authid that password for user test_scram_user
in scram
12. Set password encryption is on
13. Create role test_on_user with password 'test_on_password'
14. Check from pg_authid that password for user test_on_user in md5
15. Set password encrytpion is off
16. Create role test_off_user with password 'test_off_password'
17. Check from pg_authid that password for user test_off_user in plain
"""
# Step 1
conn_string = "host='localhost' user='******'"
conn = psycopg2.connect(conn_string)
cursor = conn.cursor()
cursor.execute("SHOW password_encryption")
current_encryption = cursor.fetchall()[0][0]
assert current_encryption == 'md5'
# Step 2
cursor.execute("SET password_encryption = 'plain'")
# Step 3
cursor.execute("SHOW password_encryption")
current_encryption = cursor.fetchall()[0][0]
assert current_encryption == 'plain'
# Step 4
cursor.execute("CREATE ROLE test_plain_user"
" WITH PASSWORD 'test_plain_password' LOGIN")
# Step 5
cursor.execute("SELECT rolpassword FROM pg_authid"
" WHERE rolname = 'test_plain_user'")
assert cursor.fetchall()[0][0] == 'test_plain_password'
# Step 6
cursor.execute("SET password_encryption = 'md5'")
# Step 7
cursor.execute("CREATE ROLE test_md5_user"
" WITH PASSWORD 'test_md5_password' LOGIN")
# Step 8
cursor.execute("SELECT rolpassword FROM pg_authid"
" WHERE rolname = 'test_md5_user'")
test_md5_user_pass = cursor.fetchall()[0][0]
assert test_md5_user_pass[0:3] == 'md5'
# Step 9
cursor.execute("SET password_encryption = 'scram'")
# Step 10
cursor.execute("CREATE ROLE test_scram_user"
" WITH PASSWORD 'test_scram_password' LOGIN")
# Step 11
cursor.execute("SELECT rolpassword FROM pg_authid"
" WHERE rolname = 'test_scram_user'")
test_scram_user_pass = cursor.fetchall()[0][0]
assert test_scram_user_pass[0:21] == 'AAAAAAAAAAAAAA==:4096'
# Step 12
cursor.execute("SET password_encryption = 'on'")
# Step 13
cursor.execute("CREATE ROLE test_on_user"
" WITH PASSWORD 'test_on_password' LOGIN")
# Step 14
cursor.execute("SELECT rolpassword FROM pg_authid"
" WHERE rolname = 'test_on_user'")
test_on_user_pass = cursor.fetchall()[0][0]
assert test_on_user_pass[0:3] == 'md5'
# Step 15
cursor.execute("SET password_encryption = 'off'")
# Step 16
cursor.execute("CREATE ROLE test_off_user"
" WITH PASSWORD 'test_off_password' LOGIN")
# Step 17
cursor.execute("SELECT rolpassword FROM pg_authid"
" WHERE rolname = 'test_off_user'")
assert cursor.fetchall()[0][0] == 'test_off_password'
cursor.close()
conn.close()
@pytest.mark.xfail(reason="Implementation of feature was changed",
strict=True)
@pytest.mark.test_authentication
def test_authentication(self, request, install_postgres):
"""Check that we can authenticate user with different password types
Scenario:
1. Edit pg_hba conf for test and restart postgres
2. Create roles for test
3. Try to connect to db with hashed password
4. Try to connect to db with password that stored in md5 hash
5. Try to connect to db with password in scram format
6. Try to connect with password in scram hash
"""
# Step 1
hba_auth = """
local all test_md5_hash_user md5
local all test_md5_user_auth md5
local all test_scram_hash_user scram
local all test_scram_user_auth scram
local all all peer
host all all 0.0.0.0/0 trust
host all all ::0/0 trust"""
install_postgres.edit_pg_hba_conf(hba_auth)
install_postgres.manage_psql('restart')
# Step 2
conn_string = "host='localhost' user='******' dbname='postgres'"
conn = psycopg2.connect(conn_string)
cursor = conn.cursor()
md5_password = self.create_hash_password('md5', self.random_password())
cursor.execute("CREATE ROLE test_md5_user_auth"
" WITH PASSWORD 'test_md5_password' LOGIN")
cursor.execute("CREATE ROLE test_md5_hash_user"
" WITH PASSWORD '%s' LOGIN" % md5_password)
scram_password = self.create_hash_password('sha256',
self.random_password())
conn.commit()
cursor.close()
conn.close()
install_postgres.set_option('password_encryption', 'scram')
conn = psycopg2.connect(conn_string)
cursor = conn.cursor()
cursor.execute("CREATE ROLE test_scram_user_auth"
" WITH PASSWORD 'test_scram_password' LOGIN")
cursor.execute("CREATE ROLE test_scram_hash_user"
" WITH PASSWORD ('%s' USING \'plain\') LOGIN" %
scram_password)
conn.commit()
cursor.close()
conn.close()
# Step 3
conn_string_md5_user = \
"host='localhost' user='******'" \
" password='******' dbname='postgres'"
conn_md5_user = psycopg2.connect(conn_string_md5_user)
assert conn_md5_user.status == 1
conn_md5_user.close()
# Step 4
conn_string_test_md5_hash_user = \
"host='localhost' user='******'" \
" password='******' dbname='postgres'" % md5_password
conn_md5_hash_user = psycopg2.connect(conn_string_test_md5_hash_user)
assert conn_md5_hash_user.status == 1
cursor = conn_md5_hash_user.cursor()
cursor.execute("SELECT 1")
cursor.close()
conn_md5_hash_user.close()
# Step 5
conn_test_scram_user_auth = \
"host='localhost' user='******'" \
" password='******' dbname='postgres'"
conn_test_scram_user_auth = psycopg2.connect(conn_test_scram_user_auth)
assert conn_test_scram_user_auth.status == 1
cursor = conn_test_scram_user_auth.cursor()
cursor.execute("SELECT 1")
cursor.close()
conn_test_scram_user_auth.close()
# Step 6
conn_scram_hash_user = \
"host='localhost' user='******'" \
" password=%s dbname='postgres'" % scram_password
conn_scram_hash_user = psycopg2.connect(conn_scram_hash_user)
assert conn_scram_hash_user.status == 1
cursor = conn_scram_hash_user.cursor()
cursor.execute("SELECT 1")
cursor.close()
conn_scram_hash_user.close()
import glob
import platform
import pytest
import os
import shutil
import subprocess
from helpers.utils import get_distro
from helpers.os_helpers import delete_data_directory
from helpers.os_helpers import download_file
from tests.settings import TMP_DIR
if platform.system() == 'Linux':
dist = get_distro()
elif platform.system() == 'Windows':
dist = 'Windows'
else:
print("Unknown Distro")
def pytest_addoption(parser):
"""This method needed for running pytest test with options
Example: command "pytest --product_edition=std" will install
postgrespro with standard edition
:param parser pytest default param for command line args:
:return:
"""
parser.addoption(
"--target", action="store", default='linux',
def test_full_install(self, request):
"""
Scenario:
1. Install current version
2. Check that setup successfull
:return:
"""
dist = ""
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
request.cls.pgid = '%s-%s' % (edition, version)
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
# Step 1
pginst = PgInstall(product=name,
edition=edition,
version=version,
milestone=milestone,
branch=branch,
windows=(self.system == 'Windows'))
# This is a workaround for the problem described in PGPRO-3596
if pginst.os.is_altlinux() and pginst.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
request.cls.pginst = pginst
pginst.setup_repo()
print("Running on %s." % target)
all_available_packages = pginst.all_packages_in_repo
print("All available packages in repo %s:\n" % pginst.reponame,
"\n".join(all_available_packages))
if self.system != 'Windows':
pginst.install_full()
pginst.install_package(" ".join(all_available_packages))
check_package_contents(pginst, all_available_packages)
check_executables(pginst, all_available_packages)
pginst.initdb_start()
else:
pginst.install_perl_win()
pginst.install_postgres_win()
server_version = pginst.get_server_version()
client_version = pginst.get_psql_version()
print("Server version:\n%s\nClient version:\n%s" %
(server_version, client_version))
if name == 'postgrespro' and not (edition in ['1c', 'sql']):
ppversion = pginst.exec_psql_select("SELECT pgpro_version()")
# PGPRO-3760
assert ppversion.startswith('PostgresPro ')
# assert ppversion.startswith('PostgresPro ' + version)
ppedition = pginst.exec_psql_select("SELECT pgpro_edition()")
if edition == 'ent':
assert ppedition == 'enterprise'
elif edition == 'ent-cert':
assert ppedition == 'enterprise'
else:
assert ppedition == 'standard'
print('pgpro_source_id:',
pginst.exec_psql_select("SELECT pgpro_build()"))
if version not in ["9.6", "10", "13"]:
pginst.env = {}
for var in os.environ:
pginst.env[var] = str(os.environ[var])
pginst.env["LANG"] = 'C'
# PGPRO-4100 TODO: Use pgpro_controldata
cdout = pginst.exec_server_bin('pg_controldata', '"%s"' %
pginst.get_datadir()).split('\n')
if name == 'postgrespro' and not (edition in ['1c', 'sql']):
assert cdout[0].startswith('pg_control edition:')
cdedition = cdout[0].replace('pg_control edition:', '').strip()
if edition == 'ent':
assert cdedition == 'Postgres Pro Enterprise'
elif edition == 'std':
assert cdedition == 'Postgres Pro Standard'
print("OK")
def test_dev_usage(self, request):
"""
Scenario:
1. Install only -dev package
2. Try to build PGXS extension
:return:
"""
dist = ""
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
# Step 1
pginst = PgInstall(product=name,
edition=edition,
version=version,
milestone=milestone,
branch=branch,
windows=(self.system == 'Windows'))
pginst.setup_repo()
print("Running on %s." % target)
if pginst.os.is_altlinux() and pginst.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
if self.system != 'Windows':
pginst.install_server_dev()
pg_bin_path = pginst.get_default_bin_path()
curpath = os.path.dirname(os.path.abspath(__file__))
test_script = r"""
set -e
if which apt-get; then
apt-get install -y gcc || true
apt-get install -y make
grep -E '(Debian GNU/Linux 9|Debian GNU/Linux 10|'\
'"Ubuntu [0-9]+\.[0-9]+|"OSNova Linux|'\
'"Astra Linux \(Smolensk 1.6\)"|"Astra Linux \(Orel\)")'\
/etc/os-release >/dev/null 2>/dev/null && \
apt install -y libdpkg-perl
elif which zypper; then
zypper install -y gcc make
elif which yum; then
yum install -y gcc make redhat-rpm-config
fi
tar fax ../extras/pg_wait_sampling.tar.gz -C /tmp && \
cd /tmp/pg_wait_sampling*/
export PATH=%s:$PATH
make USE_PGXS=1
make USE_PGXS=1 install
chmod 777 .
""" % (pg_bin_path)
subprocess.check_call(test_script, cwd=curpath, shell=True)
pginst.install_full()
pginst.initdb_start()
pginst.exec_psql('ALTER SYSTEM SET shared_preload_libraries = '
'pg_wait_sampling')
pginst.restart_service()
test_script = r"""
cd /tmp/pg_wait_sampling*/
sudo -u postgres sh -c "export PATH=%s:$PATH; make USE_PGXS=1 installcheck"
""" % (pg_bin_path)
subprocess.check_call(test_script, shell=True)
else:
pginst.install_postgres_win()
pginst.install_perl_win()
print("OK")
def test_upgrade_minor(self, request):
"""
Scenario:
1. Install current version
2. Check that setup successfull (select version)
:return:
"""
global windows_os
distro = get_distro()
if distro[2] == 'x86_64' or self.system == 'Windows':
distro = distro[:-1]
dist = " ".join(distro)
if self.system == 'Linux':
windows_os = False
elif self.system == 'Windows':
windows_os = True
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
if edition not in ['std', 'ent', '1c']:
print("Minor upgrade only for 1c, std and ent")
return
if name != 'postgrespro':
print("Minor upgrade test is only for postgrespro.")
return
small_key = "-".join([name, edition, version])
specified_version = False
if dist in ARCHIVE_VERSIONS \
and small_key in ARCHIVE_VERSIONS[dist]:
specified_version = ARCHIVE_VERSIONS[dist][small_key]
if specified_version is None:
return "%s %s %s does not support archived versions on %s." % \
(name, edition, version, dist)
print("specified version is %s" % specified_version)
print("Running on %s." % target)
pgnew = PgInstall(product=name,
edition=edition,
version=version,
milestone=milestone,
branch=branch,
windows=windows_os)
if pgnew.os.is_altlinux() and pgnew.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
pgnew.setup_repo()
if not windows_os:
pgnew.install_full()
subprocess.check_call('cp -a "%s" "%s"' %
(pgnew.get_pg_prefix(), client_dir),
shell=True)
pgnew.remove_full(False, True)
pgnew.remove_data(True)
# PGPRO-3310
if pgnew.os_name in DEBIAN_BASED:
remove_alternatives()
else:
pgnew.install_postgres_win()
pgnew.stop_service()
subprocess.check_call('xcopy /S /E /O /X /I /Q "%s" "%s"' %
(pgnew.get_pg_prefix(), client_dir),
shell=True)
pgnew.remove_full(True)
pgconfig = subprocess.check_output(
'"%s"' % os.path.join(client_dir, 'bin', 'pg_config'),
shell=True).decode(ConsoleEncoding)
vere = re.search(r'PGPRO\_VERSION\s=\s([0-9.]+)', pgconfig)
if (vere):
current_ver = vere.group(1)
else:
vere = re.search(r'VERSION\s=\s\w+\s([0-9.]+)', pgconfig)
current_ver = vere.group(1)
print("Current version is %s" % current_ver)
test_versions = get_test_versions(edition, version, specified_version,
current_ver)
if test_versions is None:
print("No archive versions found.")
return
print(test_versions)
dump_file_name = download_dump(name, edition, version + '-old',
tempdir)
for oldversion in test_versions:
print("Installing", oldversion)
key = "-".join([name, edition, oldversion])
pgold = PgInstall(product=name,
edition=edition,
version=oldversion,
milestone='archive',
branch=None,
windows=windows_os)
if pgold.os.is_altlinux() and pgold.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
pgold.setup_repo()
if not windows_os:
# PGPRO-3889
if (pgold.os_name.startswith('CentOS') or
pgold.os_name.startswith('Red Hat') or
pgold.os_name.startswith('Oracle Linux')) and \
pgold.os_version.startswith('8'):
for pkg in pgold.all_packages_in_repo[:]:
if ('jit' in pkg):
pgold.all_packages_in_repo.remove(pkg)
if (pgold.os_name.startswith('SLES')
and pgold.os_version.startswith('15')):
for pkg in pgold.all_packages_in_repo[:]:
if 'zstd' in pkg:
pgold.all_packages_in_repo.remove(pkg)
# PGPRO-2954
for pkg in pgold.all_packages_in_repo[:]:
if 'bouncer' in pkg or 'badger' in pkg:
pgold.all_packages_in_repo.remove(pkg)
if pgnew.os_name == 'ROSA Enterprise Linux Server' \
and pgnew.os_version.startswith('7.3') \
and edition == 'std' \
and version == '9.6' \
and compare_versions(oldversion, '9.6.13.1') == 0:
for pkg in pgold.all_packages_in_repo[:]:
if 'probackup' in pkg:
pgold.all_packages_in_repo.remove(pkg)
pgold.install_full()
pgold.initdb_start()
else:
pgold.install_postgres_win()
pgold.load_shared_libraries()
with open(os.path.join(tempdir, 'load-%s.log' % oldversion),
'wb') as out:
pgold.exec_psql_file(
dump_file_name,
'-q%s' %
('' if pgold.os_arch == 'x86' else ' -v ON_ERROR_STOP=1'),
stdout=out)
expected_file_name = os.path.join(tempdir, "%s-expected.sql" % key)
dumpall(pgold, expected_file_name)
pgold.delete_repo()
pgnew = PgInstall(product=name,
edition=edition,
version=version,
milestone=milestone,
branch=None,
windows=windows_os)
if pgnew.os.is_altlinux() and pgnew.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
pgnew.setup_repo()
pgold.stop_service()
if not windows_os:
pgnew.update_all_packages()
pgnew.start_service()
else:
pgnew.install_postgres_win()
result_file_name = os.path.join(tempdir, "%s-result.sql" % key)
dumpall(pgnew, result_file_name)
diff_dbs(expected_file_name, result_file_name,
os.path.join(tempdir, "%s.sql.diff" % key))
pgnew.stop_service()
repo_diff = list(
set(pgold.all_packages_in_repo) -
set(pgnew.all_packages_in_repo))
print("repo diff is %s" % repo_diff)
for package in repo_diff:
try:
pgold.remove_package(package)
except Exception:
pass
pgnew.remove_full(True)
# PGPRO-3310
if pgnew.os_name in DEBIAN_BASED:
remove_alternatives()
if pgold.os_name in DEBIAN_BASED and version == '9.6':
try:
subprocess.check_call("apt-get purge -y 'postgres*'",
shell=True)
except Exception:
pass
# PGPRO-2563
if pgold.os_name == 'Ubuntu' and version == '9.6' and \
edition == 'ent':
time.sleep(20)
def test_upgrade(self, request):
"""
Scenario:
1. Install testible version
2. if route install upgradeable version
3. Create DB with covering dump
4. Upgrade by pg_upgrade
5. Check that upgrade successfull (calculate diff between dump)
:return:
"""
distro = get_distro()
if distro[2] == 'x86_64' or self.system == 'Windows':
distro = distro[:-1]
dist = " ".join(distro)
request.cls.dist = dist
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
request.cls.product_info = product_info
key = "-".join([name, edition, version])
request.cls.key = key
print("Running on %s." % target)
if dist in FIRST_RELEASE and key in FIRST_RELEASE[dist] and \
FIRST_RELEASE[dist][key] is None:
print("Platform not supported")
return
if key not in UPGRADE_ROUTES:
print('No routes for upgrade')
return
upgrade_route = UPGRADE_ROUTES[key]
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
# Install the tested version
branch = request.config.getoption('--branch')
pg = install_server(product=name,
edition=edition,
version=version,
milestone=milestone,
branch=branch,
windows=(self.system == 'Windows'))
request.cls.pg = pg
stop(pg)
if pg.os_name in DEBIAN_BASED and pg.version == '9.6':
print("Two products 9.6 cannot be "
"installed simultaneously on debian-based OS")
return
if self.system == 'Windows':
backup_datadir_win(pg)
for route in upgrade_route['from']:
initdb_params = route['initdb-params'] if \
'initdb-params' in route else ''
init_cluster(pg, True, initdb_params, True, False)
stop(pg)
old_name = route['name']
old_edition = route['edition']
old_version = route['version']
old_key = "-".join([old_name, old_edition, old_version])
if dist in FIRST_RELEASE and old_key in FIRST_RELEASE[dist]:
if FIRST_RELEASE[dist][old_key] is None:
print("Distributive is not supported")
continue
if compare_versions(FIRST_RELEASE[dist][old_key],
get_last_version(old_edition,
old_version)) > 0:
print("Wait for %s" % FIRST_RELEASE[dist][old_key])
continue
print("=====Check upgrade from %s" % old_key)
pgold = install_server(product=old_name,
edition=old_edition,
version=old_version,
milestone=None,
branch=None,
windows=(self.system == 'Windows'),
old=True)
if self.system != 'Windows':
init_cluster(pgold, True, initdb_params, None, True)
generate_db(
pgold,
pg,
on_error_stop=False if pgold.os_arch == 'x86' else True)
dumpall(pgold, os.path.join(tempdir, "%s.sql" % old_key))
stop(pgold)
upgrade(pg, pgold)
start(pg)
after_upgrade(pg, pgold)
dump_and_diff_dbs(old_key, pg, 'upgrade')
stop(pg)
pgold.remove_full(do_not_remove=[
r"^libc.*", r".*icu.*", r".*zstd.*", r"^llvm.*"
])
# PGPRO-2459
if pgold.os_name in DEBIAN_BASED and \
old_name == "postgrespro" and old_version == "9.6" and \
old_edition != '1c':
subprocess.check_call(
"apt-get purge -y postgrespro-common "
"postgrespro-client-common",
shell=True)
class TestPgprobackup():
dist = ""
if platform.system() == 'Linux':
dist = " ".join(get_distro()[0:2])
elif platform.system() == 'Windows':
dist = 'Windows'
else:
print("Unknown Distro")
# TODO add restore from backup for all test cases
DISTRO = get_distro()[0]
PG_HBA_CONFIG = """
local replication all trust
local all all trust
host all all 0.0.0.0/0 trust
host all all ::0/0 trust"""
def parse_pgprobackup_show_command_out(self, out):
""" Parse show command output for pgprobackup feature
:param out:
:return:
"""
return dict(v.split("=") for v in
out.replace('\t', ' ').strip().split('\n')
if v.strip() and "=" in v)
@staticmethod
def create_backup_directory():
"""Create new directory for tablespace
:return: str path to tablespace
"""
backup_path = os.path.join(TMP_DIR, 'backup')
os.mkdir(backup_path)
os.chown(backup_path,
pwd.getpwnam("postgres").pw_uid,
grp.getgrnam("postgres").gr_gid)
os.environ["BACKUP_PATH"] = backup_path
return backup_path
def execute_pg_probackup(self, *options):
"""
:param options: strings with params for pg_probackup
:return: output of command execution
"""
cmd = ["%s/pg_probackup" % pg_bindir()] + list(options)
return subprocess.check_output(cmd)
@pytest.mark.test_install_pgprobackup
def test_install_pgprobackup(self, request):
""" Install pg_probackup utility and configure postgresql
for running pg_probackup.
Scenario:
1. Check version function
2. Check help function
"""
# Step 1
assert self.execute_pg_probackup("--version") is not None
# Step 2
assert self.execute_pg_probackup("--help") is not None
@pytest.mark.xfail(reason="Implementation of feature was changed",
strict=True)
@pytest.mark.test_pgprobackup_compression_continious_backup
def test_pgprobackup_compression_continious_backup(self,
request,
install_postgres):
"""Test pg_probackup with compression feature
and full continous backup
Scenario:
1. Create backup dir
2. Set data dir to environment variables
3. Init backup dir
4. Set options for pg_probackup
5. Create tablespace with compression
6. Create table in compression tablespace
7. Edit pg_hba_config
8. Make full backup
9. Get backup id
10. Check that backup status is OK
11. Check that backup validation is OK
"""
# Step 1
backup_dir = self.create_backup_directory()
# Step 2
os.environ["PGDATA"] = install_postgres.get_option('data_directory')
# Step 3
install_postgres.edit_pg_hba_conf(self.PG_HBA_CONFIG)
install_postgres.manage_psql('restart')
# Step 4
self.execute_pg_probackup("init")
for root, dirs, files in os.walk(backup_dir):
for d in dirs:
os.chown(os.path.join(root, d),
pwd.getpwnam("postgres").pw_uid,
grp.getgrnam("postgres").gr_gid)
for f in files:
os.chown(os.path.join(root, f),
pwd.getpwnam("postgres").pw_uid,
grp.getgrnam("postgres").gr_gid)
# Step 5
install_postgres.set_option("ptrack_enable", "on")
install_postgres.set_option("wal_level", "archive")
install_postgres.set_option("archive_mode", "on")
install_postgres.set_option(
"archive_command",
"test ! -f {0}/wal/%f && cp %p {1}/wal/%f".format(
backup_dir, backup_dir))
install_postgres.set_option("cfs_gc_workers", "0")
# Step 6
tablespace_path = os.path.join(TMP_DIR, 'pgprobackup_compression')
os.mkdir(tablespace_path)
os.chown(tablespace_path,
pwd.getpwnam("postgres").pw_uid,
grp.getgrnam("postgres").gr_gid)
conn = psycopg2.connect(install_postgres.connstring)
execute(
conn,
'CREATE TABLESPACE pgprobackup_compression'
' LOCATION \'%s\' WITH(compression = true);' % tablespace_path)
# Step 7
execute(conn,
'CREATE TABLE tbl TABLESPACE pgprobackup_compression'
' AS SELECT i, md5(random()::text)'
' FROM generate_series(0,1e05) AS i;')
# Step 8
self.execute_pg_probackup("backup", "-b", "full",
"-d", "postgres", "-U", "postgres")
# Step 9
# Get last backup id and get out for show command with this backup
pgprobackup_show = subprocess.Popen(
["%s/pg_probackup" % pg_bindir(), "show", "-U", "postgres"],
stdout=subprocess.PIPE)
awk_backup_id = subprocess.Popen(
["awk", "FNR == 4 {print $1}"], stdin=pgprobackup_show.stdout,
stdout=subprocess.PIPE)
backup_id = awk_backup_id.communicate()[0].strip()
pgprobackup_show.stdout.close()
# Step 10
backup_info = self.parse_pgprobackup_show_command_out(
self.execute_pg_probackup("show", backup_id))
assert backup_info['STATUS'] == 'OK'
# Step 11
self.execute_pg_probackup("validate", backup_id)
@pytest.mark.xfail(reason="Implementation of feature was changed",
strict=True)
@pytest.mark.test_pgprobackup_retention_policy_options
def test_pgprobackup_retention_policy_options(self, request):
"""Scenario
1. Create 3 backups
2. Set option redundancy to 1 backup and run purge
3. Check that only one backup must saved after purge
4. Up system date for 3 days
5. Create 2 backups
6. Set option window to 1 and run purge
7. Check that only one backup was saved
8. Create 2 backups
9. Up time for 2 days
10. Run purge with redundancy=2 and window=1
11. Check that saved 2 backups
"""
# Step 1
for i in range(3):
self.execute_pg_probackup("backup", "-b", "full",
"-d", "postgres", "-U", "postgres")
# Step 2
self.execute_pg_probackup("retention", "--redundancy=1", "purge")
# Step 3
pgprobackup_show = subprocess.Popen(
["%s/pg_probackup" % pg_bindir(), "show", "-U", "postgres"],
stdout=subprocess.PIPE)
awk_backup_ids = subprocess.Popen(
["awk", "NR > 3 {print $1}"], stdin=pgprobackup_show.stdout,
stdout=subprocess.PIPE)
assert len(awk_backup_ids.communicate()[0].strip().split()) == 1
# Step 4
for i in range(2):
self.execute_pg_probackup("backup", "-b", "full",
"-d", "postgres", "-U", "postgres")
# Step 5
new_time = datetime.datetime.now() + datetime.timedelta(days=3)
subprocess.check_output(["sudo", "date", "-s", str(new_time)])
# Step 6
self.execute_pg_probackup("retention", "--window=1", "purge")
# Step 7
pgprobackup_show = subprocess.Popen(
["%s/pg_probackup" % pg_bindir(), "show", "-U", "postgres"],
stdout=subprocess.PIPE)
awk_backup_ids = subprocess.Popen(
["awk", "NR > 3 {print $1}"], stdin=pgprobackup_show.stdout,
stdout=subprocess.PIPE)
assert len(awk_backup_ids.communicate()[0].strip().split()) == 1
# Step 8
for i in range(2):
self.execute_pg_probackup("backup", "-b", "full",
"-d", "postgres", "-U", "postgres")
# Step 9
new_time = datetime.datetime.now() + datetime.timedelta(days=3)
subprocess.check_output(["sudo", "date", "-s", str(new_time)])
# Step 10
self.execute_pg_probackup("retention", "--window=1",
"--redundancy=2", "purge")
# Step 11
pgprobackup_show = subprocess.Popen(
["%s/pg_probackup" % pg_bindir(), "show", "-U", "postgres"],
stdout=subprocess.PIPE)
awk_backup_ids = subprocess.Popen(
["awk", "NR > 3 {print $1}"], stdin=pgprobackup_show.stdout,
stdout=subprocess.PIPE)
assert len(awk_backup_ids.communicate()[0].strip().split()) == 2
def test_multimaster_install(self, request):
"""
Scenario:
1. Install current version
2. Check that setup successfull (select version)
:return:
"""
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
if version.startswith('9.') or version == '10' \
or version == '13' \
or not edition.startswith('ent'):
print('Version %s %s is not supported' % (edition, version))
return
elif version == '11' and dist == 'Windows':
print('Windows is not supported under version %s' % version)
return
# Step 1
pginst = PgInstall(product=name,
edition=edition,
version=version,
milestone=milestone,
branch=branch,
windows=(self.system == 'Windows'))
request.cls.pginst = pginst
if pginst.os.is_altlinux() and pginst.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
pginst.setup_repo()
print("Running on %s." % target)
if self.system != 'Windows':
pginst.install_full_topless()
else:
pginst.install_postgres_win()
pginst.stop_service()
mm = Multimaster(size=2,
pginst=pginst,
rootdir=os.path.abspath(
os.path.join(pginst.get_datadir(), os.pardir)))
mm.start()
for i, cl_state in mm.get_cluster_state_all().items():
if int(cl_state[0]) != i:
raise Exception('Wrong ID on node %i! %i != %i' %
(i, i, cl_state[0]))
if cl_state[1].lower() != 'online':
print(cl_state)
raise Exception('Node %i is not online!' % i)
print(mm.get_nodes_state_all())
for i, ns_all in mm.get_nodes_state_all().items():
for node_state in ns_all:
if node_state[1].rstrip() != 't':
raise Exception('Node %s is not enabled! (on node %i)' %
(node_state[0], i))
pgbench = {}
for i, node in mm.nodes.items():
pgbench[i] = mm.pgbench(i,
type='simple-update',
duration=60,
max_tries=100)
pgbench[1].init()
for i in range(1, mm.size + 1):
if not mm.nodes[i].referee:
print('Running pgbench on node%i' % i)
pgbench[i].start()
mm.wait_for_txid(600)
print('Cluster state:')
print(mm.get_cluster_state_all())
print(mm.get_nodes_state_all())
print('Current TXID (after pgbench): %s' %
mm.psql('SELECT txid_current()', '-Aqt'))
print('Isolating node 2...')
mm.isolate(2)
if mm.has_referee:
mm.wait_for_referee(1, 60)
pgbench[1].wait()
pgbench[1].start()
mm.wait_for_txid(2700)
print('Cluster state:')
print(mm.get_cluster_state_all(allow_fail=True))
print(mm.get_nodes_state_all(allow_fail=True))
print('De-isolating node 2...')
mm.deisolate(2)
print('Current TXID (after isolation): %s' %
mm.psql('SELECT txid_current()', '-Aqt'))
if not mm.check(2):
print('Try to recover node 2')
mm.wait(2, 600)
print('Cluster state:')
print(mm.get_cluster_state_all(allow_fail=True))
print(mm.get_nodes_state_all(allow_fail=True))
mm.wait_for_txid(3500)
for i in range(1, mm.size):
if not mm.nodes[i].referee:
print('Pgbench %i terminated rc=%i' % (i, pgbench[i].stop()))
mm.sure_pgbench_is_dead(i)
pgbench_tables = ('pgbench_branches', 'pgbench_tellers',
'pgbench_accounts', 'pgbench_history')
print('Current TXID (after pgbench termination): %s' %
mm.psql('SELECT txid_current()', '-Aqt'))
for table in pgbench_tables:
result1 = mm.pg_dump(1, [table])
for i in range(2, mm.size + 1):
if not mm.nodes[i].referee:
result = mm.pg_dump(i, [table])
diff_dbs(result1, result, '%s_1_%i.diff' % (table, i))
class TestCompression():
dist = ""
if platform.system() == 'Linux':
dist = " ".join(get_distro()[0:2])
elif platform.system() == 'Windows':
dist = 'Windows'
else:
print("Unknown Distro")
PGBENCH_SCHEMA_UNLOGGED = """
CREATE UNLOGGED TABLE pgbench_branches_unlogged(
bid SERIAL PRIMARY KEY,
bbalance INTEGER NOT NULL,
filler CHAR(88) NOT NULL
);
CREATE UNLOGGED TABLE pgbench_tellers_unlogged(
tid SERIAL PRIMARY KEY,
bid INTEGER NOT NULL REFERENCES pgbench_branches_unlogged,
tbalance INTEGER NOT NULL,
filler CHAR(84) NOT NULL
);
CREATE UNLOGGED TABLE pgbench_accounts_unlogged(
aid BIGSERIAL PRIMARY KEY,
bid INTEGER NOT NULL REFERENCES pgbench_branches_unlogged,
abalance INTEGER NOT NULL,
filler CHAR(84) NOT NULL
);
CREATE UNLOGGED TABLE pgbench_history_unlogged(
tid INTEGER NOT NULL REFERENCES pgbench_tellers_unlogged,
bid INTEGER NOT NULL REFERENCES pgbench_branches_unlogged,
aid BIGINT NOT NULL REFERENCES pgbench_accounts_unlogged,
delta INTEGER NOT NULL,
mtime TIMESTAMP NOT NULL,
filler CHAR(22)
-- UNIQUE (tid, bid, aid, mtime)
);
"""
@staticmethod
def get_directory_size(start_path):
""" Get directory size recursively
:param start_path: directory for start
:return: total size of directory in bytes
"""
total_size = 0
for dirpath, dirnames, filenames in os.walk(start_path):
for f in filenames:
fp = os.path.join(dirpath, f)
total_size += os.path.getsize(fp)
# TODO return in Mb
return total_size
@staticmethod
def get_filenames(directory):
"""
This function will return the file names in a directory
tree by walking the tree either top-down or bottom-up. For each
directory in the tree rooted at directory top (including top itself),
it yields a 3-tuple (dirpath, dirnames, filenames).
"""
for root, directories, files in os.walk(directory):
for filename in files:
fn, ext = os.path.splitext(filename)
yield ext
@staticmethod
def create_tablespace_directory():
"""Create new directory for tablespace
:return: str path to tablespace
"""
tablespace_catalog = 'tablespace-' + str(random.randint(0, 100))
tablespace_path = os.path.join(TMP_DIR, tablespace_catalog)
os.mkdir(tablespace_path)
os.chown(tablespace_path,
pwd.getpwnam("postgres").pw_uid,
grp.getgrnam("postgres").gr_gid)
return tablespace_path
def create_tablespace(self, tablespace_name, compression=False):
""" Create tablespace
:return:
"""
tablespace_location = self.create_tablespace_directory()
conn_string = "host='localhost' user='******' "
conn = psycopg2.connect(conn_string)
conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
cursor = conn.cursor()
# TODO add check that PGPRO edition is enterprise
if compression:
cursor.execute('CREATE TABLESPACE {0} LOCATION \'{1}\''
' WITH (compression=true);'.format(
tablespace_name, tablespace_location))
else:
cursor.execute('CREATE TABLESPACE {0} LOCATION \'{1}\';'.format(
tablespace_name, tablespace_location))
return tablespace_location
@pytest.mark.test_compression_standalone_positive
def test_compression_standalone_positive(self, request, install_postgres):
""" Test for compression feature.
Scenario:
1. Create test tables
1. Create tablespace with compression
2. Save size of created tables by fixture in file system
(count and size of files)
3. Run data generator for tablespace with compression
4. Save size of table with compression
5. Check that files for table in tablespace without compression >
that files in tablespace with compression
6. Check tablespace folder for files with *.cfm extension
7. Check that tables has some data
"""
# Step 1
create_test_table('20', 'pgbench')
# Step 2
compression_files_directory = self.create_tablespace('compression',
compression=True)
# Step 3
data_size_without_compression = self.get_directory_size(
install_postgres.get_option('data_directory'))
# Step 4
print(data_size_without_compression)
install_postgres.set_option('default_tablespace', 'compression')
create_test_table(size='20', schema='pgbench')
# Step 5
data_size_with_compression = self.get_directory_size(
install_postgres.get_option('data_directory'))
print(data_size_with_compression)
print(compression_files_directory)
# Step 6
assert data_size_with_compression < data_size_without_compression
# Step 7
compression_files = self.get_filenames(compression_files_directory)
assert '.cfm' in compression_files
# Step 8
conn_string = "host='localhost' user='******' "
conn = psycopg2.connect(conn_string)
cursor = conn.cursor()
cursor.execute("SELECT * FROM pgbench_tellers LIMIT 10;")
result = cursor.fetchall()
print(result)
assert result != 0
conn.close()
drop_test_table(conn_string)
@pytest.mark.test_compression_unlogged_tables
def test_compression_unlogged_tables(self, request, install_postgres):
""" Test for compression feature.
Scenario:
1. Create tablespace with compression
2. Run pgbench (unlogged schema) for tablespace with compression
3. Check tablespace folder for files with *.cfm extension
"""
# Step 1
compression_files_directory = self.create_tablespace(
'compression_unlogged_tables', compression=True)
print(compression_files_directory)
# Step 2
install_postgres.set_option('default_tablespace',
'compression_unlogged_tables')
create_test_table(size='20', schema=self.PGBENCH_SCHEMA_UNLOGGED)
# Step 3
compression_files = self.get_filenames(compression_files_directory)
assert '.cfm' in compression_files
conn_string = "host='localhost' user='******' "
conn = psycopg2.connect(conn_string)
cursor = conn.cursor()
cursor.execute("SELECT * FROM pgbench_tellers_unlogged LIMIT 10;")
result = cursor.fetchall()
print(result)
assert result != 0
conn.close()
@pytest.mark.test_compression_negative
def test_compression_negative(self, request, install_postgres):
""" Test for compression feature.
Scenario:
1. Create tablespace with compression
2. Run pgbench for tablespace with compression and
kill postgres process
3. Run postgres process and check that postgres is running
4. Check that cfm files created
5. Check that pgbench tables don't created
"""
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
product_info = " ".join([self.dist, name, edition, version])
# pylint: disable=no-member
tag_mark = pytest.allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
# Step 1
compression_files_directory = self.create_tablespace(
'compression_negative', compression=True)
print(compression_files_directory)
install_postgres.set_option('default_tablespace',
'compression_negative')
postgres_pid = install_postgres.get_postmaster_pid()
print(postgres_pid)
# Step 2
create_table_process = Process(target=create_test_table,
args=(
'20',
'pgbench',
))
create_table_process.start()
time.sleep(10)
# Step 3
if create_table_process.is_alive():
process = psutil.Process(postgres_pid)
process.kill()
compression_files = self.get_filenames(compression_files_directory)
# Step 4
# Files for tables created but no any table was created
# because transaction was corruped
assert '.cfm' in compression_files
install_postgres.manage_psql('start')
# Step 5
conn_string = "host='localhost' user='******' "
conn = psycopg2.connect(conn_string)
cursor = conn.cursor()
cursor.execute("SELECT * FROM pg_tables"
" WHERE tablespace=\'compression_negative\';")
assert len(cursor.fetchall()) == 0
drop_test_table(conn_string)
@pytest.mark.test_compression_unlogged_tables_negative
def test_compression_unlogged_tables_negative(self, request,
install_postgres):
""" Test for compression feature.
Scenario:
1. Create tablespace with compression for test
2. Run pgbench (unlogged schema) for tablespace with compression
3. Kill postgres process
4. Start postgres process
5. Check that tables was created and readable
6. Check tablespace folder for files with *.cfm extension
"""
# Step 1
compression_files_directory = self.create_tablespace(
'compression_unlogged_tables_negative', compression=True)
# Step 2
install_postgres.set_option('default_tablespace',
'compression_unlogged_tables_negative')
create_test_table(size='20', schema=self.PGBENCH_SCHEMA_UNLOGGED)
# Step 3
conn_string = "host='localhost' user='******' "
conn = psycopg2.connect(conn_string)
cursor = conn.cursor()
cursor.execute("select * from pgbench_tellers_unlogged limit 10;")
conn.close()
process = psutil.Process(install_postgres.get_postmaster_pid())
process.kill()
# Step 4
install_postgres.manage_psql('start')
# Step 5
conn_string = "host='localhost' user='******' "
conn = psycopg2.connect(conn_string)
cursor = conn.cursor()
cursor.execute(
"SELECT * FROM pg_tables"
" WHERE tablespace=\'compression_unlogged_tables_negative\';")
assert len(cursor.fetchall()) != 0
conn_string = "host='localhost' user='******' "
conn = psycopg2.connect(conn_string)
cursor = conn.cursor()
cursor.execute("SELECT * FROM pgbench_tellers_unlogged LIMIT 10;")
result = cursor.fetchall()
assert result != 0
conn.close()
# Step 6
compression_files = self.get_filenames(compression_files_directory)
assert '.cfm' in compression_files
@pytest.mark.usefixtures('install_postgres')
@pytest.mark.test_compression_alter_tablepsace_to_compression
def test_compression_alter_tablepsace_to_compression(self, request):
"""Scenario:
1. Create tablespace for test
2. Create test data with pgbench
3. Save rows count from pgbench_history
4. Move table to branch with compression
5. Check that rows count before and after moving the same
"""
# Step 1
self.create_tablespace('compression_alter_tablepsace',
compression=True)
# Step 2
conn_string = "host=localhost user=postgres"
load_pgbench(conn_string, ["-i", "-s", "10"])
load_pgbench(conn_string, ["-s", "10", "-t", "1000", "-c", "10"])
# Step 3
conn = psycopg2.connect(conn_string)
cursor = conn.cursor()
cursor.execute("select count(*) from pgbench_history")
rows_before_move = cursor.fetchall()[0][0]
conn.commit()
conn.close()
# Step 4
conn_string = "host='localhost' user='******' "
conn = psycopg2.connect(conn_string)
cursor = conn.cursor()
cursor.execute("ALTER TABLE pgbench_history"
" SET TABLESPACE compression_alter_tablepsace")
cursor.execute("select count(*) from pgbench_history")
rows_after_move = cursor.fetchall()[0][0]
conn.commit()
conn.close()
# Step 5
assert rows_before_move == rows_after_move
@pytest.mark.usefixtures('install_postgres')
@pytest.mark.test_compression_alter_tablepsace_from_compression
def test_compression_alter_tablepsace_from_compression(self, request):
"""Scenario:
1. Create tablespace for test
2. Create test data with pgbench
3. Save rows count from pgbench_history
4. Move table to branch without compression
5. Check that rows count before and after moving the same
"""
# Step 1
self.create_tablespace('compression_alter_from_tablepsace',
compression=True)
# Step 2
conn_string = "host=localhost user=postgres"
load_pgbench(conn_string, ["-i", "-s", "10"])
load_pgbench(conn_string, ["-s", "10", "-t", "1000", "-c", "10"])
# Step 3
conn = psycopg2.connect(conn_string)
cursor = conn.cursor()
cursor.execute("select count(*) from pgbench_history")
rows_before_move = cursor.fetchall()[0][0]
conn.commit()
conn.close()
# Step 4
conn_string = "host='localhost' user='******' "
conn = psycopg2.connect(conn_string)
cursor = conn.cursor()
cursor.execute("ALTER TABLE pgbench_history"
" SET TABLESPACE compression_alter_from_tablepsace")
cursor.execute("select count(*) from pgbench_history")
rows_after_move = cursor.fetchall()[0][0]
conn.commit()
conn.close()
# Step 5
assert rows_before_move == rows_after_move
def test_make_check(self, request):
"""
Scenario:
1. Install current version
2. Check that setup successfull
We need to perform the test on Windows in two stages:
First we setup the server and prepare environment,
then we exclude current user from the Administrators group.
Second we execute `make installcheck` without admin rights.
:return:
"""
dist = ""
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
pgid = '%s-%s' % (edition, version)
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
# Step 1
pginst = PgInstall(product=name,
edition=edition,
version=version,
milestone=milestone,
branch=branch,
windows=(self.system == 'Windows'))
request.cls.pginst = pginst
pginst.make_check_passed = False
curpath = os.path.dirname(os.path.abspath(__file__))
if pginst.os.is_altlinux() and pginst.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
if self.system == 'Windows':
if os.path.exists(pginst.get_default_bin_path()):
# Refresh environment to get correct PYTHONHOME
refresh_env_win()
# The instance is already installed and
# installcheck environment is presumably prepared,
# so just run make_installcheck (once more)
subprocess.check_call(
'"%s" "%s" "%s"' %
(os.path.join(curpath, 'make_installcheck.cmd'),
get_pg_prefix(pginst), pginst.service_name),
shell=True)
pginst.make_check_passed = True
return
pginst.setup_repo()
print("Running on %s." % target)
tarball = pginst.download_source()
tar = tarfile.open(tarball, 'r:bz2')
tar.extractall()
tar.close()
for comp in [
'orafce', 'plv8', 'pgpro-stats', 'pgpro-pwr',
'pgpro-controldata', 'pg-filedump', 'pg-portal-modify',
'pg-repack'
]:
pkgname = '%s-%s-%s' % (comp, edition, version)
if pkgname not in pginst.get_packages_in_repo():
pkgname = comp
if pkgname not in pginst.get_packages_in_repo():
continue
pginst.download_source(comp.replace('-', '_'),
pginst.get_package_version(pkgname),
'tar.gz')
if self.system != 'Windows':
pginst.install_full()
pginst.initdb_start()
else:
pginst.install_perl_win()
pginst.install_postgres_win(port=55432)
try:
pginst.exec_psql('CREATE EXTENSION plpython3u')
except Exception:
subprocess.check_call('SETX PYTHONHOME C:\\Python27 -m',
shell=True)
if version != "9.6" or self.system == 'Windows' or \
(edition == '1c' and pginst.os_name not in DEBIAN_BASED):
buildinfo = os.path.join(pginst.get_pg_prefix(), 'doc',
'buildinfo.txt')
else:
buildinfo = subprocess.check_output(
'ls /usr/share/doc/postgres*pro*/buildinfo.txt',
shell=True).decode(ConsoleEncoding).strip()
with open(buildinfo, 'r') as bi:
bitxt = bi.read()
assert (re.search(r'^Documentation translation', bitxt,
re.MULTILINE))
assert (re.search(r'^Source', bitxt, re.MULTILINE))
assert (re.search(r'^SPEC', bitxt, re.MULTILINE))
print("The binary package buildinfo:\n%s\n" % bitxt)
pginst.install_default_config()
pginst.exec_psql("ALTER SYSTEM SET max_worker_processes = 16")
pginst.exec_psql("ALTER SYSTEM SET lc_messages = 'C'")
# Prepare pg_hba.conf for src/interfaces/ecpg/test/connect/test5
with open(os.path.join(pginst.get_configdir(), 'pg_hba.conf'),
'r+') as conf:
hba = re.sub(r'^(local\s+all\s+all\s+peer)$',
"local all regress_ecpg_user1 md5\n"
"local all regress_ecpg_user2 md5\n"
"local all regress_hacker trust\n"
"local all regress_superuser trust\n"
"local all nosuper trust\n"
r'\1',
conf.read(),
flags=re.MULTILINE)
conf.seek(0)
conf.write(hba)
pginst.load_shared_libraries(restart_service=False)
pginst.restart_service()
if self.system != 'Windows':
subprocess.check_call(
'"%s" "%s" "%s"' %
(os.path.join(curpath, 'make_installcheck.sh'),
get_pg_prefix(pginst), pginst.service_name),
shell=True)
pginst.make_check_passed = True
else:
# First run is performed to setup the environment
subprocess.check_call(
'"%s" "%s" "%s"' %
(os.path.join(curpath, 'make_installcheck.cmd'),
get_pg_prefix(pginst), pginst.service_name),
shell=True)
request.session.customexitstatus = 222
import platform
import distro
import pytest
from allure_commons.types import LabelType
from helpers.utils import create_env_info_from_config, get_distro
from helpers.environment_manager import Environment
if platform.system() == 'Linux':
dist = " ".join(get_distro()[0:2])
elif platform.system() == 'Windows':
dist = 'Windows'
else:
print("Unknown Distro")
def pytest_addoption(parser):
"""Option for pytest run, list of images for test
:param parser pytest default param for command line args:
:return:
"""
parser.addoption(
"--target", dest="target",
help='system(s) under test (image(s))')
parser.addoption(
"--product_version", action="store", default='9.6',
help="Specify product version. Available values: 9.5, 9.6")
parser.addoption(
"--product_name", action="store", default='postgrespro',
def test_install(self, request):
""" Install pg_probackup utility and configure postgresql
for running pg_probackup.
Scenario:
1. Install postgrespro version
2. Check version function
3. Check help function
"""
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
request.cls.skip = False
if name != 'postgrespro' or edition == '1c' or version == '13':
# TODO: Enable this test for v13
print("PgProBackup test is only for postgrespro std and ent.")
request.cls.skip = True
return
# Step 1
self.pginst = PgInstall(product=name, edition=edition, version=version,
milestone=milestone, branch=branch,
windows=(self.system == 'Windows'))
self.pginst.setup_repo()
if self.pginst.os.is_altlinux()and self.pginst.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
if self.system != 'Windows':
self.pginst.install_full()
self.pginst.initdb_start()
else:
self.pginst.install_postgres_win()
self.fix_permissions(tempdir)
self.bindir = self.pginst.get_bin_path() + os.sep
print('Bindir is %s' % self.bindir)
self.pginst.load_shared_libraries(restart_service=True)
request.cls.pginst = self.pginst
# Step 2
assert self.execute_pg_probackup("--help")
ver = self.execute_pg_probackup("--version")
print('Version is %s' % ver)
vere = re.search(r'[^(]+\s([0-9.]+)\s.*', ver)
self.version = vere.group(1)
assert self.version
request.cls.version = self.version
