def __init__(self, remote=False, host=None): self.remote = remote self.host = host self.dist_info = get_distro(remote, host) self.os_name = self.dist_info[0] self.os_version = self.dist_info[1] self.os_arch = self.dist_info[2]
def getDistribution(): if sys.platform == "win32": return platform.win32_ver() elif sys.platform == "linux" or sys.platform == "linux2": distname, version, id = get_distro() return "%s %s" % (distname, version) elif sys.platform == "darwin": release, versioninfo, machine = platform.mac_ver() return "%s %s" % (release, versioninfo) else: return "Undefined"
def test_clean_install(self, request): """ Scenario: 1. Install current version 2. Check that setup successfull (select version) :return: """ if self.system == 'Linux': dist = " ".join(get_distro()[0:2]) elif self.system == 'Windows': dist = 'Windows' else: raise Exception("OS %s is not supported." % self.system) version = request.config.getoption('--product_version') name = request.config.getoption('--product_name') edition = request.config.getoption('--product_edition') milestone = request.config.getoption('--product_milestone') target = request.config.getoption('--target') product_info = " ".join([dist, name, edition, version]) tag_mark = allure.label(LabelType.TAG, product_info) request.node.add_marker(tag_mark) branch = request.config.getoption('--branch') # Step 1 pginst = PgInstall(product=name, edition=edition, version=version, milestone=milestone, branch=branch, windows=(self.system == 'Windows')) if pginst.os.is_altlinux() and pginst.os.os_arch == 'aarch64': os.environ['LANG'] = 'en_US.UTF-8' request.cls.pginst = pginst pginst.setup_repo() print("Running on %s." % target) print("Minor product version is: %s\n" % pginst.get_product_minor_version()) if self.system != 'Windows': pginst.install_base() pginst.initdb_start() else: pginst.install_postgres_win() server_version = pginst.get_server_version() client_version = pginst.get_psql_version() print("Server version:\n%s\nClient version:\n%s" % (server_version, client_version)) print("OK")
def test_extensions_prepare(self, request): """ Scenario: 1. Install current version :return: """ if self.system == 'Linux': dist = " ".join(get_distro()[0:2]) elif self.system == 'Windows': dist = 'Windows' else: raise Exception("OS %s is not supported." % self.system) version = request.config.getoption('--product_version') name = request.config.getoption('--product_name') edition = request.config.getoption('--product_edition') milestone = request.config.getoption('--product_milestone') target = request.config.getoption('--target') product_info = " ".join([dist, name, edition, version]) tag_mark = allure.label(LabelType.TAG, product_info) request.node.add_marker(tag_mark) branch = request.config.getoption('--branch') # Step 1 pginst = PgInstall(product=name, edition=edition, version=version, milestone=milestone, branch=branch, windows=(self.system == 'Windows')) request.cls.pginst = pginst pginst.setup_repo() print("Running on %s." % target) if pginst.os.is_altlinux() and pginst.os.os_arch == 'aarch64': os.environ['LANG'] = 'en_US.UTF-8' if self.system != 'Windows': pginst.install_full() pginst.initdb_start() else: pginst.install_postgres_win()
def test_hotstandby_compat(self, request): """ Scenario: 1. Install current version 2. Check that setup successfull (select version) :return: """ global windows_os if self.system == 'Linux': dist = " ".join(get_distro()[0:2]) elif self.system == 'Windows': dist = 'Windows' windows_os = True else: raise Exception("OS %s is not supported." % self.system) version = request.config.getoption('--product_version') name = request.config.getoption('--product_name') edition = request.config.getoption('--product_edition') milestone = request.config.getoption('--product_milestone') target = request.config.getoption('--target') product_info = " ".join([dist, name, edition, version]) tag_mark = allure.label(LabelType.TAG, product_info) request.node.add_marker(tag_mark) branch = request.config.getoption('--branch') if name != 'postgrespro': print("Hot Standby compatibility test is only for postgrespro.") return if edition == "ent": archive_url = PGPRO_ARCHIVE_ENTERPRISE elif edition == "std": archive_url = PGPRO_ARCHIVE_STANDARD else: raise Exception("Unsupported postgrespro edition (%s)." % edition) print("Running on %s." % target) # Choose two versions -- newest and oldest supported soup = get_soup(archive_url) arcversions = [] startswith = 'pgproee-' if edition == 'ent' else \ ('pgpro-' if edition == 'std' else 'pg1c-') for link in soup.findAll('a'): href = link.get('href') if href.startswith(startswith) and href.endswith('/'): vere = re.search(r'\w+-([0-9.]+)/', href) if vere: if vere.group(1).startswith(version): arcvers = vere.group(1) if version == '9.6': # Due to CATALOG_VERSION_NO change # we don't support lower 9.6 versions if compare_versions(arcvers, '9.6.4.1') < 0: arcvers = None # PGPRO-3227, PGPRO-3834 if windows_os and version == '10': if compare_versions(arcvers, '10.11.1'): arcvers = None if windows_os and version == '11': if compare_versions(arcvers, '11.6.1') < 0: arcvers = None if arcvers: arcversions.append(arcvers) arcversions.sort(key=extend_ver) if not arcversions: print("No previous minor versions found. Test skipped.") return # Choose first and last versions testversions = [arcversions[0], arcversions[-1]] if testversions[0] == testversions[1]: testversions = [testversions[0]] # Workaround for unsupported libpq options fix_extra_libpq_options = False if edition == 'ent' and version == '10': if compare_versions(arcversions[0], '10.4.1') < 0: fix_extra_libpq_options = True pre12version = version in ['9.6', '10', '11'] if windows_os: waldir = r'C:\tmp\pgwal' srcdir = r'C:\tmp' else: waldir = os.path.join(tempfile.gettempdir(), 'pgwal') srcdir = '/var/src' pgsrcdir = None for oldversion in testversions: print("Installing", oldversion) pgold = PgInstall(product=name, edition=edition, version=oldversion, milestone='archive', branch=None, windows=windows_os) pgold.setup_repo() if not windows_os: pgold.install_base() pgold.initdb_start() else: pgold.install_postgres_win() setup_pgpass('replicator', 'replicator') server_version = pgold.get_server_version() client_version = pgold.get_psql_version() print("Old server version:\n%s\nOld client version:\n%s" % (server_version, client_version)) pgold.exec_psql('ALTER SYSTEM SET port=15432') oldpgprefix = pgold.get_pg_prefix() olddatadir = pgold.get_datadir() if oldpgprefix == '/usr': raise Exception("/usr as postgres prefix is not supported.") if pgold.get_configdir() != pgold.get_datadir(): raise Exception("Separate config dir is not supported.") pgold.stop_service() time.sleep(5) if not windows_os: subprocess.check_call('cp -a "%s" "%s.old"' % (oldpgprefix, oldpgprefix), shell=True) subprocess.check_call('cp -a "%s" "%s.old"' % (olddatadir, olddatadir), shell=True) oldpgprefix += ".old" olddatadir += ".old" else: print('xcopy /S /E /O /X /I /Q "%s" "%s.old"' % (oldpgprefix, oldpgprefix)) subprocess.check_call('xcopy /S /E /O /X /I /Q "%s" "%s.old"' % (oldpgprefix, oldpgprefix), shell=True) oldpgprefix += ".old" olddatadir = os.path.join(oldpgprefix, 'data') if os.path.exists(os.path.join(olddatadir, 'postgresql.conf.old')): os.remove(os.path.join(olddatadir, 'postgresql.conf.old')) pgold.remove_full(remove_data=True) pgold.pg_prefix = oldpgprefix pgold.datadir = olddatadir pgold.configdir = olddatadir pgold.port = 15432 if not windows_os: pgold.pg_preexec = 'sudo -u postgres ' \ 'LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ' old_env = os.environ.copy() old_env["LD_LIBRARY_PATH"] = os.path.join(oldpgprefix, 'lib') pgold.env = old_env else: subprocess.check_call( 'sc create postgres-old binpath= ' '"\\"%s\\" runservice -N postgres-old -D \\"%s\\" -w"' ' start= demand obj= "NT Authority\\NetworkService" ' % (os.path.join(oldpgprefix, 'bin', 'pg_ctl'), olddatadir), shell=True) pgold.service_name = 'postgres-old' pgnew = PgInstall(product=name, edition=edition, version=version, milestone=milestone, branch=branch, windows=windows_os) pgnew.setup_repo() if not windows_os: pgnew.install_base() pgnew.initdb_start() else: pgnew.install_postgres_win() if not pgsrcdir: pgsrcdir = prepare_pg_regress(pgnew, srcdir) pgnew.stop_service() pgnew.remove_data() # Test replication from old to new setup_sender(pgold, waldir, pgnew.get_datadir()) start_receiver(pgnew, waldir, pre12version) run_hs_test(pgold, pgnew, pgsrcdir) do_server_action(pgnew, "stop") do_server_action(pgold, "stop") # Test replication from new to old pgnew.init_cluster(force_remove=True) pgold.remove_data() setup_sender(pgnew, waldir, pgold.get_datadir()) if fix_extra_libpq_options: workaround_for_extra_libpq_options(pgold) start_receiver(pgold, waldir, pre12version) run_hs_test(pgnew, pgold, pgsrcdir) do_server_action(pgnew, "stop") do_server_action(pgold, "stop") pgnew.remove_full(remove_data=True) shutil.rmtree(olddatadir) shutil.rmtree(oldpgprefix) if windows_os: subprocess.check_call('sc delete postgres-old', shell=True) print("OK")
class TestScram(): """ Only Enterprise Edition Feature """ dist = "" if platform.system() == 'Linux': dist = " ".join(get_distro()[0:2]) elif platform.system() == 'Windows': dist = 'Windows' else: print("Unknown Distro") @staticmethod def random_password(): return ''.join(random.choice(ascii_lowercase) for i in range(16)) @staticmethod def create_hash_password(hash_type, password): if hash_type == 'md5': hash = hashlib.md5() hash.update(password) return 'md5' + hash.hexdigest() elif hash_type == 'sha256': hash = hashlib.sha256() hash.update(password) return 'AAAAAAAAAAAAAA==:4096:' + hash.hexdigest() else: print("Error. Bad hash type. Use md5 or sha256") return None @pytest.mark.test_scram_configuring def test_scram_configuring(self, request): """Check that we can set GUC variables via SET command, they saved and in pg_authid password saved in right format Scenario: 1. Check that default password encryption is md5 2. Set password encryption is plain 3. Check that password encryption is plain 4. Create role test_plain_user with password 'test_plain_password' 5. Check from pg_authid that password for user test_plain_user in plain and equal to 'test_plain_password' 6. Set password encrytpion is md5 7. Create role test_md5_user with password 'test_md5_password' 8. Check from pg_authid that password for user test_md5_user in md5 9. Set password encrytpion is scram 10. Create role test_scram_user with password 'test_scram_password' 11. Check from pg_authid that password for user test_scram_user in scram 12. Set password encryption is on 13. Create role test_on_user with password 'test_on_password' 14. Check from pg_authid that password for user test_on_user in md5 15. Set password encrytpion is off 16. Create role test_off_user with password 'test_off_password' 17. Check from pg_authid that password for user test_off_user in plain """ # Step 1 conn_string = "host='localhost' user='******'" conn = psycopg2.connect(conn_string) cursor = conn.cursor() cursor.execute("SHOW password_encryption") current_encryption = cursor.fetchall()[0][0] assert current_encryption == 'md5' # Step 2 cursor.execute("SET password_encryption = 'plain'") # Step 3 cursor.execute("SHOW password_encryption") current_encryption = cursor.fetchall()[0][0] assert current_encryption == 'plain' # Step 4 cursor.execute("CREATE ROLE test_plain_user" " WITH PASSWORD 'test_plain_password' LOGIN") # Step 5 cursor.execute("SELECT rolpassword FROM pg_authid" " WHERE rolname = 'test_plain_user'") assert cursor.fetchall()[0][0] == 'test_plain_password' # Step 6 cursor.execute("SET password_encryption = 'md5'") # Step 7 cursor.execute("CREATE ROLE test_md5_user" " WITH PASSWORD 'test_md5_password' LOGIN") # Step 8 cursor.execute("SELECT rolpassword FROM pg_authid" " WHERE rolname = 'test_md5_user'") test_md5_user_pass = cursor.fetchall()[0][0] assert test_md5_user_pass[0:3] == 'md5' # Step 9 cursor.execute("SET password_encryption = 'scram'") # Step 10 cursor.execute("CREATE ROLE test_scram_user" " WITH PASSWORD 'test_scram_password' LOGIN") # Step 11 cursor.execute("SELECT rolpassword FROM pg_authid" " WHERE rolname = 'test_scram_user'") test_scram_user_pass = cursor.fetchall()[0][0] assert test_scram_user_pass[0:21] == 'AAAAAAAAAAAAAA==:4096' # Step 12 cursor.execute("SET password_encryption = 'on'") # Step 13 cursor.execute("CREATE ROLE test_on_user" " WITH PASSWORD 'test_on_password' LOGIN") # Step 14 cursor.execute("SELECT rolpassword FROM pg_authid" " WHERE rolname = 'test_on_user'") test_on_user_pass = cursor.fetchall()[0][0] assert test_on_user_pass[0:3] == 'md5' # Step 15 cursor.execute("SET password_encryption = 'off'") # Step 16 cursor.execute("CREATE ROLE test_off_user" " WITH PASSWORD 'test_off_password' LOGIN") # Step 17 cursor.execute("SELECT rolpassword FROM pg_authid" " WHERE rolname = 'test_off_user'") assert cursor.fetchall()[0][0] == 'test_off_password' cursor.close() conn.close() @pytest.mark.xfail(reason="Implementation of feature was changed", strict=True) @pytest.mark.test_authentication def test_authentication(self, request, install_postgres): """Check that we can authenticate user with different password types Scenario: 1. Edit pg_hba conf for test and restart postgres 2. Create roles for test 3. Try to connect to db with hashed password 4. Try to connect to db with password that stored in md5 hash 5. Try to connect to db with password in scram format 6. Try to connect with password in scram hash """ # Step 1 hba_auth = """ local all test_md5_hash_user md5 local all test_md5_user_auth md5 local all test_scram_hash_user scram local all test_scram_user_auth scram local all all peer host all all 0.0.0.0/0 trust host all all ::0/0 trust""" install_postgres.edit_pg_hba_conf(hba_auth) install_postgres.manage_psql('restart') # Step 2 conn_string = "host='localhost' user='******' dbname='postgres'" conn = psycopg2.connect(conn_string) cursor = conn.cursor() md5_password = self.create_hash_password('md5', self.random_password()) cursor.execute("CREATE ROLE test_md5_user_auth" " WITH PASSWORD 'test_md5_password' LOGIN") cursor.execute("CREATE ROLE test_md5_hash_user" " WITH PASSWORD '%s' LOGIN" % md5_password) scram_password = self.create_hash_password('sha256', self.random_password()) conn.commit() cursor.close() conn.close() install_postgres.set_option('password_encryption', 'scram') conn = psycopg2.connect(conn_string) cursor = conn.cursor() cursor.execute("CREATE ROLE test_scram_user_auth" " WITH PASSWORD 'test_scram_password' LOGIN") cursor.execute("CREATE ROLE test_scram_hash_user" " WITH PASSWORD ('%s' USING \'plain\') LOGIN" % scram_password) conn.commit() cursor.close() conn.close() # Step 3 conn_string_md5_user = \ "host='localhost' user='******'" \ " password='******' dbname='postgres'" conn_md5_user = psycopg2.connect(conn_string_md5_user) assert conn_md5_user.status == 1 conn_md5_user.close() # Step 4 conn_string_test_md5_hash_user = \ "host='localhost' user='******'" \ " password='******' dbname='postgres'" % md5_password conn_md5_hash_user = psycopg2.connect(conn_string_test_md5_hash_user) assert conn_md5_hash_user.status == 1 cursor = conn_md5_hash_user.cursor() cursor.execute("SELECT 1") cursor.close() conn_md5_hash_user.close() # Step 5 conn_test_scram_user_auth = \ "host='localhost' user='******'" \ " password='******' dbname='postgres'" conn_test_scram_user_auth = psycopg2.connect(conn_test_scram_user_auth) assert conn_test_scram_user_auth.status == 1 cursor = conn_test_scram_user_auth.cursor() cursor.execute("SELECT 1") cursor.close() conn_test_scram_user_auth.close() # Step 6 conn_scram_hash_user = \ "host='localhost' user='******'" \ " password=%s dbname='postgres'" % scram_password conn_scram_hash_user = psycopg2.connect(conn_scram_hash_user) assert conn_scram_hash_user.status == 1 cursor = conn_scram_hash_user.cursor() cursor.execute("SELECT 1") cursor.close() conn_scram_hash_user.close()
import glob import platform import pytest import os import shutil import subprocess from helpers.utils import get_distro from helpers.os_helpers import delete_data_directory from helpers.os_helpers import download_file from tests.settings import TMP_DIR if platform.system() == 'Linux': dist = get_distro() elif platform.system() == 'Windows': dist = 'Windows' else: print("Unknown Distro") def pytest_addoption(parser): """This method needed for running pytest test with options Example: command "pytest --product_edition=std" will install postgrespro with standard edition :param parser pytest default param for command line args: :return: """ parser.addoption( "--target", action="store", default='linux',
def test_full_install(self, request): """ Scenario: 1. Install current version 2. Check that setup successfull :return: """ dist = "" if self.system == 'Linux': dist = " ".join(get_distro()[0:2]) elif self.system == 'Windows': dist = 'Windows' else: raise Exception("OS %s is not supported." % self.system) version = request.config.getoption('--product_version') name = request.config.getoption('--product_name') edition = request.config.getoption('--product_edition') milestone = request.config.getoption('--product_milestone') request.cls.pgid = '%s-%s' % (edition, version) target = request.config.getoption('--target') product_info = " ".join([dist, name, edition, version]) tag_mark = allure.label(LabelType.TAG, product_info) request.node.add_marker(tag_mark) branch = request.config.getoption('--branch') # Step 1 pginst = PgInstall(product=name, edition=edition, version=version, milestone=milestone, branch=branch, windows=(self.system == 'Windows')) # This is a workaround for the problem described in PGPRO-3596 if pginst.os.is_altlinux() and pginst.os.os_arch == 'aarch64': os.environ['LANG'] = 'en_US.UTF-8' request.cls.pginst = pginst pginst.setup_repo() print("Running on %s." % target) all_available_packages = pginst.all_packages_in_repo print("All available packages in repo %s:\n" % pginst.reponame, "\n".join(all_available_packages)) if self.system != 'Windows': pginst.install_full() pginst.install_package(" ".join(all_available_packages)) check_package_contents(pginst, all_available_packages) check_executables(pginst, all_available_packages) pginst.initdb_start() else: pginst.install_perl_win() pginst.install_postgres_win() server_version = pginst.get_server_version() client_version = pginst.get_psql_version() print("Server version:\n%s\nClient version:\n%s" % (server_version, client_version)) if name == 'postgrespro' and not (edition in ['1c', 'sql']): ppversion = pginst.exec_psql_select("SELECT pgpro_version()") # PGPRO-3760 assert ppversion.startswith('PostgresPro ') # assert ppversion.startswith('PostgresPro ' + version) ppedition = pginst.exec_psql_select("SELECT pgpro_edition()") if edition == 'ent': assert ppedition == 'enterprise' elif edition == 'ent-cert': assert ppedition == 'enterprise' else: assert ppedition == 'standard' print('pgpro_source_id:', pginst.exec_psql_select("SELECT pgpro_build()")) if version not in ["9.6", "10", "13"]: pginst.env = {} for var in os.environ: pginst.env[var] = str(os.environ[var]) pginst.env["LANG"] = 'C' # PGPRO-4100 TODO: Use pgpro_controldata cdout = pginst.exec_server_bin('pg_controldata', '"%s"' % pginst.get_datadir()).split('\n') if name == 'postgrespro' and not (edition in ['1c', 'sql']): assert cdout[0].startswith('pg_control edition:') cdedition = cdout[0].replace('pg_control edition:', '').strip() if edition == 'ent': assert cdedition == 'Postgres Pro Enterprise' elif edition == 'std': assert cdedition == 'Postgres Pro Standard' print("OK")
def test_dev_usage(self, request): """ Scenario: 1. Install only -dev package 2. Try to build PGXS extension :return: """ dist = "" if self.system == 'Linux': dist = " ".join(get_distro()[0:2]) elif self.system == 'Windows': dist = 'Windows' else: raise Exception("OS %s is not supported." % self.system) version = request.config.getoption('--product_version') name = request.config.getoption('--product_name') edition = request.config.getoption('--product_edition') milestone = request.config.getoption('--product_milestone') target = request.config.getoption('--target') product_info = " ".join([dist, name, edition, version]) tag_mark = allure.label(LabelType.TAG, product_info) request.node.add_marker(tag_mark) branch = request.config.getoption('--branch') # Step 1 pginst = PgInstall(product=name, edition=edition, version=version, milestone=milestone, branch=branch, windows=(self.system == 'Windows')) pginst.setup_repo() print("Running on %s." % target) if pginst.os.is_altlinux() and pginst.os.os_arch == 'aarch64': os.environ['LANG'] = 'en_US.UTF-8' if self.system != 'Windows': pginst.install_server_dev() pg_bin_path = pginst.get_default_bin_path() curpath = os.path.dirname(os.path.abspath(__file__)) test_script = r""" set -e if which apt-get; then apt-get install -y gcc || true apt-get install -y make grep -E '(Debian GNU/Linux 9|Debian GNU/Linux 10|'\ '"Ubuntu [0-9]+\.[0-9]+|"OSNova Linux|'\ '"Astra Linux \(Smolensk 1.6\)"|"Astra Linux \(Orel\)")'\ /etc/os-release >/dev/null 2>/dev/null && \ apt install -y libdpkg-perl elif which zypper; then zypper install -y gcc make elif which yum; then yum install -y gcc make redhat-rpm-config fi tar fax ../extras/pg_wait_sampling.tar.gz -C /tmp && \ cd /tmp/pg_wait_sampling*/ export PATH=%s:$PATH make USE_PGXS=1 make USE_PGXS=1 install chmod 777 . """ % (pg_bin_path) subprocess.check_call(test_script, cwd=curpath, shell=True) pginst.install_full() pginst.initdb_start() pginst.exec_psql('ALTER SYSTEM SET shared_preload_libraries = ' 'pg_wait_sampling') pginst.restart_service() test_script = r""" cd /tmp/pg_wait_sampling*/ sudo -u postgres sh -c "export PATH=%s:$PATH; make USE_PGXS=1 installcheck" """ % (pg_bin_path) subprocess.check_call(test_script, shell=True) else: pginst.install_postgres_win() pginst.install_perl_win() print("OK")
def test_upgrade_minor(self, request): """ Scenario: 1. Install current version 2. Check that setup successfull (select version) :return: """ global windows_os distro = get_distro() if distro[2] == 'x86_64' or self.system == 'Windows': distro = distro[:-1] dist = " ".join(distro) if self.system == 'Linux': windows_os = False elif self.system == 'Windows': windows_os = True else: raise Exception("OS %s is not supported." % self.system) version = request.config.getoption('--product_version') name = request.config.getoption('--product_name') edition = request.config.getoption('--product_edition') milestone = request.config.getoption('--product_milestone') target = request.config.getoption('--target') product_info = " ".join([dist, name, edition, version]) tag_mark = allure.label(LabelType.TAG, product_info) request.node.add_marker(tag_mark) branch = request.config.getoption('--branch') if edition not in ['std', 'ent', '1c']: print("Minor upgrade only for 1c, std and ent") return if name != 'postgrespro': print("Minor upgrade test is only for postgrespro.") return small_key = "-".join([name, edition, version]) specified_version = False if dist in ARCHIVE_VERSIONS \ and small_key in ARCHIVE_VERSIONS[dist]: specified_version = ARCHIVE_VERSIONS[dist][small_key] if specified_version is None: return "%s %s %s does not support archived versions on %s." % \ (name, edition, version, dist) print("specified version is %s" % specified_version) print("Running on %s." % target) pgnew = PgInstall(product=name, edition=edition, version=version, milestone=milestone, branch=branch, windows=windows_os) if pgnew.os.is_altlinux() and pgnew.os.os_arch == 'aarch64': os.environ['LANG'] = 'en_US.UTF-8' pgnew.setup_repo() if not windows_os: pgnew.install_full() subprocess.check_call('cp -a "%s" "%s"' % (pgnew.get_pg_prefix(), client_dir), shell=True) pgnew.remove_full(False, True) pgnew.remove_data(True) # PGPRO-3310 if pgnew.os_name in DEBIAN_BASED: remove_alternatives() else: pgnew.install_postgres_win() pgnew.stop_service() subprocess.check_call('xcopy /S /E /O /X /I /Q "%s" "%s"' % (pgnew.get_pg_prefix(), client_dir), shell=True) pgnew.remove_full(True) pgconfig = subprocess.check_output( '"%s"' % os.path.join(client_dir, 'bin', 'pg_config'), shell=True).decode(ConsoleEncoding) vere = re.search(r'PGPRO\_VERSION\s=\s([0-9.]+)', pgconfig) if (vere): current_ver = vere.group(1) else: vere = re.search(r'VERSION\s=\s\w+\s([0-9.]+)', pgconfig) current_ver = vere.group(1) print("Current version is %s" % current_ver) test_versions = get_test_versions(edition, version, specified_version, current_ver) if test_versions is None: print("No archive versions found.") return print(test_versions) dump_file_name = download_dump(name, edition, version + '-old', tempdir) for oldversion in test_versions: print("Installing", oldversion) key = "-".join([name, edition, oldversion]) pgold = PgInstall(product=name, edition=edition, version=oldversion, milestone='archive', branch=None, windows=windows_os) if pgold.os.is_altlinux() and pgold.os.os_arch == 'aarch64': os.environ['LANG'] = 'en_US.UTF-8' pgold.setup_repo() if not windows_os: # PGPRO-3889 if (pgold.os_name.startswith('CentOS') or pgold.os_name.startswith('Red Hat') or pgold.os_name.startswith('Oracle Linux')) and \ pgold.os_version.startswith('8'): for pkg in pgold.all_packages_in_repo[:]: if ('jit' in pkg): pgold.all_packages_in_repo.remove(pkg) if (pgold.os_name.startswith('SLES') and pgold.os_version.startswith('15')): for pkg in pgold.all_packages_in_repo[:]: if 'zstd' in pkg: pgold.all_packages_in_repo.remove(pkg) # PGPRO-2954 for pkg in pgold.all_packages_in_repo[:]: if 'bouncer' in pkg or 'badger' in pkg: pgold.all_packages_in_repo.remove(pkg) if pgnew.os_name == 'ROSA Enterprise Linux Server' \ and pgnew.os_version.startswith('7.3') \ and edition == 'std' \ and version == '9.6' \ and compare_versions(oldversion, '9.6.13.1') == 0: for pkg in pgold.all_packages_in_repo[:]: if 'probackup' in pkg: pgold.all_packages_in_repo.remove(pkg) pgold.install_full() pgold.initdb_start() else: pgold.install_postgres_win() pgold.load_shared_libraries() with open(os.path.join(tempdir, 'load-%s.log' % oldversion), 'wb') as out: pgold.exec_psql_file( dump_file_name, '-q%s' % ('' if pgold.os_arch == 'x86' else ' -v ON_ERROR_STOP=1'), stdout=out) expected_file_name = os.path.join(tempdir, "%s-expected.sql" % key) dumpall(pgold, expected_file_name) pgold.delete_repo() pgnew = PgInstall(product=name, edition=edition, version=version, milestone=milestone, branch=None, windows=windows_os) if pgnew.os.is_altlinux() and pgnew.os.os_arch == 'aarch64': os.environ['LANG'] = 'en_US.UTF-8' pgnew.setup_repo() pgold.stop_service() if not windows_os: pgnew.update_all_packages() pgnew.start_service() else: pgnew.install_postgres_win() result_file_name = os.path.join(tempdir, "%s-result.sql" % key) dumpall(pgnew, result_file_name) diff_dbs(expected_file_name, result_file_name, os.path.join(tempdir, "%s.sql.diff" % key)) pgnew.stop_service() repo_diff = list( set(pgold.all_packages_in_repo) - set(pgnew.all_packages_in_repo)) print("repo diff is %s" % repo_diff) for package in repo_diff: try: pgold.remove_package(package) except Exception: pass pgnew.remove_full(True) # PGPRO-3310 if pgnew.os_name in DEBIAN_BASED: remove_alternatives() if pgold.os_name in DEBIAN_BASED and version == '9.6': try: subprocess.check_call("apt-get purge -y 'postgres*'", shell=True) except Exception: pass # PGPRO-2563 if pgold.os_name == 'Ubuntu' and version == '9.6' and \ edition == 'ent': time.sleep(20)
def test_upgrade(self, request): """ Scenario: 1. Install testible version 2. if route install upgradeable version 3. Create DB with covering dump 4. Upgrade by pg_upgrade 5. Check that upgrade successfull (calculate diff between dump) :return: """ distro = get_distro() if distro[2] == 'x86_64' or self.system == 'Windows': distro = distro[:-1] dist = " ".join(distro) request.cls.dist = dist version = request.config.getoption('--product_version') name = request.config.getoption('--product_name') edition = request.config.getoption('--product_edition') milestone = request.config.getoption('--product_milestone') target = request.config.getoption('--target') product_info = " ".join([dist, name, edition, version]) request.cls.product_info = product_info key = "-".join([name, edition, version]) request.cls.key = key print("Running on %s." % target) if dist in FIRST_RELEASE and key in FIRST_RELEASE[dist] and \ FIRST_RELEASE[dist][key] is None: print("Platform not supported") return if key not in UPGRADE_ROUTES: print('No routes for upgrade') return upgrade_route = UPGRADE_ROUTES[key] tag_mark = allure.label(LabelType.TAG, product_info) request.node.add_marker(tag_mark) # Install the tested version branch = request.config.getoption('--branch') pg = install_server(product=name, edition=edition, version=version, milestone=milestone, branch=branch, windows=(self.system == 'Windows')) request.cls.pg = pg stop(pg) if pg.os_name in DEBIAN_BASED and pg.version == '9.6': print("Two products 9.6 cannot be " "installed simultaneously on debian-based OS") return if self.system == 'Windows': backup_datadir_win(pg) for route in upgrade_route['from']: initdb_params = route['initdb-params'] if \ 'initdb-params' in route else '' init_cluster(pg, True, initdb_params, True, False) stop(pg) old_name = route['name'] old_edition = route['edition'] old_version = route['version'] old_key = "-".join([old_name, old_edition, old_version]) if dist in FIRST_RELEASE and old_key in FIRST_RELEASE[dist]: if FIRST_RELEASE[dist][old_key] is None: print("Distributive is not supported") continue if compare_versions(FIRST_RELEASE[dist][old_key], get_last_version(old_edition, old_version)) > 0: print("Wait for %s" % FIRST_RELEASE[dist][old_key]) continue print("=====Check upgrade from %s" % old_key) pgold = install_server(product=old_name, edition=old_edition, version=old_version, milestone=None, branch=None, windows=(self.system == 'Windows'), old=True) if self.system != 'Windows': init_cluster(pgold, True, initdb_params, None, True) generate_db( pgold, pg, on_error_stop=False if pgold.os_arch == 'x86' else True) dumpall(pgold, os.path.join(tempdir, "%s.sql" % old_key)) stop(pgold) upgrade(pg, pgold) start(pg) after_upgrade(pg, pgold) dump_and_diff_dbs(old_key, pg, 'upgrade') stop(pg) pgold.remove_full(do_not_remove=[ r"^libc.*", r".*icu.*", r".*zstd.*", r"^llvm.*" ]) # PGPRO-2459 if pgold.os_name in DEBIAN_BASED and \ old_name == "postgrespro" and old_version == "9.6" and \ old_edition != '1c': subprocess.check_call( "apt-get purge -y postgrespro-common " "postgrespro-client-common", shell=True)
class TestPgprobackup(): dist = "" if platform.system() == 'Linux': dist = " ".join(get_distro()[0:2]) elif platform.system() == 'Windows': dist = 'Windows' else: print("Unknown Distro") # TODO add restore from backup for all test cases DISTRO = get_distro()[0] PG_HBA_CONFIG = """ local replication all trust local all all trust host all all 0.0.0.0/0 trust host all all ::0/0 trust""" def parse_pgprobackup_show_command_out(self, out): """ Parse show command output for pgprobackup feature :param out: :return: """ return dict(v.split("=") for v in out.replace('\t', ' ').strip().split('\n') if v.strip() and "=" in v) @staticmethod def create_backup_directory(): """Create new directory for tablespace :return: str path to tablespace """ backup_path = os.path.join(TMP_DIR, 'backup') os.mkdir(backup_path) os.chown(backup_path, pwd.getpwnam("postgres").pw_uid, grp.getgrnam("postgres").gr_gid) os.environ["BACKUP_PATH"] = backup_path return backup_path def execute_pg_probackup(self, *options): """ :param options: strings with params for pg_probackup :return: output of command execution """ cmd = ["%s/pg_probackup" % pg_bindir()] + list(options) return subprocess.check_output(cmd) @pytest.mark.test_install_pgprobackup def test_install_pgprobackup(self, request): """ Install pg_probackup utility and configure postgresql for running pg_probackup. Scenario: 1. Check version function 2. Check help function """ # Step 1 assert self.execute_pg_probackup("--version") is not None # Step 2 assert self.execute_pg_probackup("--help") is not None @pytest.mark.xfail(reason="Implementation of feature was changed", strict=True) @pytest.mark.test_pgprobackup_compression_continious_backup def test_pgprobackup_compression_continious_backup(self, request, install_postgres): """Test pg_probackup with compression feature and full continous backup Scenario: 1. Create backup dir 2. Set data dir to environment variables 3. Init backup dir 4. Set options for pg_probackup 5. Create tablespace with compression 6. Create table in compression tablespace 7. Edit pg_hba_config 8. Make full backup 9. Get backup id 10. Check that backup status is OK 11. Check that backup validation is OK """ # Step 1 backup_dir = self.create_backup_directory() # Step 2 os.environ["PGDATA"] = install_postgres.get_option('data_directory') # Step 3 install_postgres.edit_pg_hba_conf(self.PG_HBA_CONFIG) install_postgres.manage_psql('restart') # Step 4 self.execute_pg_probackup("init") for root, dirs, files in os.walk(backup_dir): for d in dirs: os.chown(os.path.join(root, d), pwd.getpwnam("postgres").pw_uid, grp.getgrnam("postgres").gr_gid) for f in files: os.chown(os.path.join(root, f), pwd.getpwnam("postgres").pw_uid, grp.getgrnam("postgres").gr_gid) # Step 5 install_postgres.set_option("ptrack_enable", "on") install_postgres.set_option("wal_level", "archive") install_postgres.set_option("archive_mode", "on") install_postgres.set_option( "archive_command", "test ! -f {0}/wal/%f && cp %p {1}/wal/%f".format( backup_dir, backup_dir)) install_postgres.set_option("cfs_gc_workers", "0") # Step 6 tablespace_path = os.path.join(TMP_DIR, 'pgprobackup_compression') os.mkdir(tablespace_path) os.chown(tablespace_path, pwd.getpwnam("postgres").pw_uid, grp.getgrnam("postgres").gr_gid) conn = psycopg2.connect(install_postgres.connstring) execute( conn, 'CREATE TABLESPACE pgprobackup_compression' ' LOCATION \'%s\' WITH(compression = true);' % tablespace_path) # Step 7 execute(conn, 'CREATE TABLE tbl TABLESPACE pgprobackup_compression' ' AS SELECT i, md5(random()::text)' ' FROM generate_series(0,1e05) AS i;') # Step 8 self.execute_pg_probackup("backup", "-b", "full", "-d", "postgres", "-U", "postgres") # Step 9 # Get last backup id and get out for show command with this backup pgprobackup_show = subprocess.Popen( ["%s/pg_probackup" % pg_bindir(), "show", "-U", "postgres"], stdout=subprocess.PIPE) awk_backup_id = subprocess.Popen( ["awk", "FNR == 4 {print $1}"], stdin=pgprobackup_show.stdout, stdout=subprocess.PIPE) backup_id = awk_backup_id.communicate()[0].strip() pgprobackup_show.stdout.close() # Step 10 backup_info = self.parse_pgprobackup_show_command_out( self.execute_pg_probackup("show", backup_id)) assert backup_info['STATUS'] == 'OK' # Step 11 self.execute_pg_probackup("validate", backup_id) @pytest.mark.xfail(reason="Implementation of feature was changed", strict=True) @pytest.mark.test_pgprobackup_retention_policy_options def test_pgprobackup_retention_policy_options(self, request): """Scenario 1. Create 3 backups 2. Set option redundancy to 1 backup and run purge 3. Check that only one backup must saved after purge 4. Up system date for 3 days 5. Create 2 backups 6. Set option window to 1 and run purge 7. Check that only one backup was saved 8. Create 2 backups 9. Up time for 2 days 10. Run purge with redundancy=2 and window=1 11. Check that saved 2 backups """ # Step 1 for i in range(3): self.execute_pg_probackup("backup", "-b", "full", "-d", "postgres", "-U", "postgres") # Step 2 self.execute_pg_probackup("retention", "--redundancy=1", "purge") # Step 3 pgprobackup_show = subprocess.Popen( ["%s/pg_probackup" % pg_bindir(), "show", "-U", "postgres"], stdout=subprocess.PIPE) awk_backup_ids = subprocess.Popen( ["awk", "NR > 3 {print $1}"], stdin=pgprobackup_show.stdout, stdout=subprocess.PIPE) assert len(awk_backup_ids.communicate()[0].strip().split()) == 1 # Step 4 for i in range(2): self.execute_pg_probackup("backup", "-b", "full", "-d", "postgres", "-U", "postgres") # Step 5 new_time = datetime.datetime.now() + datetime.timedelta(days=3) subprocess.check_output(["sudo", "date", "-s", str(new_time)]) # Step 6 self.execute_pg_probackup("retention", "--window=1", "purge") # Step 7 pgprobackup_show = subprocess.Popen( ["%s/pg_probackup" % pg_bindir(), "show", "-U", "postgres"], stdout=subprocess.PIPE) awk_backup_ids = subprocess.Popen( ["awk", "NR > 3 {print $1}"], stdin=pgprobackup_show.stdout, stdout=subprocess.PIPE) assert len(awk_backup_ids.communicate()[0].strip().split()) == 1 # Step 8 for i in range(2): self.execute_pg_probackup("backup", "-b", "full", "-d", "postgres", "-U", "postgres") # Step 9 new_time = datetime.datetime.now() + datetime.timedelta(days=3) subprocess.check_output(["sudo", "date", "-s", str(new_time)]) # Step 10 self.execute_pg_probackup("retention", "--window=1", "--redundancy=2", "purge") # Step 11 pgprobackup_show = subprocess.Popen( ["%s/pg_probackup" % pg_bindir(), "show", "-U", "postgres"], stdout=subprocess.PIPE) awk_backup_ids = subprocess.Popen( ["awk", "NR > 3 {print $1}"], stdin=pgprobackup_show.stdout, stdout=subprocess.PIPE) assert len(awk_backup_ids.communicate()[0].strip().split()) == 2
def test_multimaster_install(self, request): """ Scenario: 1. Install current version 2. Check that setup successfull (select version) :return: """ if self.system == 'Linux': dist = " ".join(get_distro()[0:2]) elif self.system == 'Windows': dist = 'Windows' else: raise Exception("OS %s is not supported." % self.system) version = request.config.getoption('--product_version') name = request.config.getoption('--product_name') edition = request.config.getoption('--product_edition') milestone = request.config.getoption('--product_milestone') target = request.config.getoption('--target') product_info = " ".join([dist, name, edition, version]) tag_mark = allure.label(LabelType.TAG, product_info) request.node.add_marker(tag_mark) branch = request.config.getoption('--branch') if version.startswith('9.') or version == '10' \ or version == '13' \ or not edition.startswith('ent'): print('Version %s %s is not supported' % (edition, version)) return elif version == '11' and dist == 'Windows': print('Windows is not supported under version %s' % version) return # Step 1 pginst = PgInstall(product=name, edition=edition, version=version, milestone=milestone, branch=branch, windows=(self.system == 'Windows')) request.cls.pginst = pginst if pginst.os.is_altlinux() and pginst.os.os_arch == 'aarch64': os.environ['LANG'] = 'en_US.UTF-8' pginst.setup_repo() print("Running on %s." % target) if self.system != 'Windows': pginst.install_full_topless() else: pginst.install_postgres_win() pginst.stop_service() mm = Multimaster(size=2, pginst=pginst, rootdir=os.path.abspath( os.path.join(pginst.get_datadir(), os.pardir))) mm.start() for i, cl_state in mm.get_cluster_state_all().items(): if int(cl_state[0]) != i: raise Exception('Wrong ID on node %i! %i != %i' % (i, i, cl_state[0])) if cl_state[1].lower() != 'online': print(cl_state) raise Exception('Node %i is not online!' % i) print(mm.get_nodes_state_all()) for i, ns_all in mm.get_nodes_state_all().items(): for node_state in ns_all: if node_state[1].rstrip() != 't': raise Exception('Node %s is not enabled! (on node %i)' % (node_state[0], i)) pgbench = {} for i, node in mm.nodes.items(): pgbench[i] = mm.pgbench(i, type='simple-update', duration=60, max_tries=100) pgbench[1].init() for i in range(1, mm.size + 1): if not mm.nodes[i].referee: print('Running pgbench on node%i' % i) pgbench[i].start() mm.wait_for_txid(600) print('Cluster state:') print(mm.get_cluster_state_all()) print(mm.get_nodes_state_all()) print('Current TXID (after pgbench): %s' % mm.psql('SELECT txid_current()', '-Aqt')) print('Isolating node 2...') mm.isolate(2) if mm.has_referee: mm.wait_for_referee(1, 60) pgbench[1].wait() pgbench[1].start() mm.wait_for_txid(2700) print('Cluster state:') print(mm.get_cluster_state_all(allow_fail=True)) print(mm.get_nodes_state_all(allow_fail=True)) print('De-isolating node 2...') mm.deisolate(2) print('Current TXID (after isolation): %s' % mm.psql('SELECT txid_current()', '-Aqt')) if not mm.check(2): print('Try to recover node 2') mm.wait(2, 600) print('Cluster state:') print(mm.get_cluster_state_all(allow_fail=True)) print(mm.get_nodes_state_all(allow_fail=True)) mm.wait_for_txid(3500) for i in range(1, mm.size): if not mm.nodes[i].referee: print('Pgbench %i terminated rc=%i' % (i, pgbench[i].stop())) mm.sure_pgbench_is_dead(i) pgbench_tables = ('pgbench_branches', 'pgbench_tellers', 'pgbench_accounts', 'pgbench_history') print('Current TXID (after pgbench termination): %s' % mm.psql('SELECT txid_current()', '-Aqt')) for table in pgbench_tables: result1 = mm.pg_dump(1, [table]) for i in range(2, mm.size + 1): if not mm.nodes[i].referee: result = mm.pg_dump(i, [table]) diff_dbs(result1, result, '%s_1_%i.diff' % (table, i))
class TestCompression(): dist = "" if platform.system() == 'Linux': dist = " ".join(get_distro()[0:2]) elif platform.system() == 'Windows': dist = 'Windows' else: print("Unknown Distro") PGBENCH_SCHEMA_UNLOGGED = """ CREATE UNLOGGED TABLE pgbench_branches_unlogged( bid SERIAL PRIMARY KEY, bbalance INTEGER NOT NULL, filler CHAR(88) NOT NULL ); CREATE UNLOGGED TABLE pgbench_tellers_unlogged( tid SERIAL PRIMARY KEY, bid INTEGER NOT NULL REFERENCES pgbench_branches_unlogged, tbalance INTEGER NOT NULL, filler CHAR(84) NOT NULL ); CREATE UNLOGGED TABLE pgbench_accounts_unlogged( aid BIGSERIAL PRIMARY KEY, bid INTEGER NOT NULL REFERENCES pgbench_branches_unlogged, abalance INTEGER NOT NULL, filler CHAR(84) NOT NULL ); CREATE UNLOGGED TABLE pgbench_history_unlogged( tid INTEGER NOT NULL REFERENCES pgbench_tellers_unlogged, bid INTEGER NOT NULL REFERENCES pgbench_branches_unlogged, aid BIGINT NOT NULL REFERENCES pgbench_accounts_unlogged, delta INTEGER NOT NULL, mtime TIMESTAMP NOT NULL, filler CHAR(22) -- UNIQUE (tid, bid, aid, mtime) ); """ @staticmethod def get_directory_size(start_path): """ Get directory size recursively :param start_path: directory for start :return: total size of directory in bytes """ total_size = 0 for dirpath, dirnames, filenames in os.walk(start_path): for f in filenames: fp = os.path.join(dirpath, f) total_size += os.path.getsize(fp) # TODO return in Mb return total_size @staticmethod def get_filenames(directory): """ This function will return the file names in a directory tree by walking the tree either top-down or bottom-up. For each directory in the tree rooted at directory top (including top itself), it yields a 3-tuple (dirpath, dirnames, filenames). """ for root, directories, files in os.walk(directory): for filename in files: fn, ext = os.path.splitext(filename) yield ext @staticmethod def create_tablespace_directory(): """Create new directory for tablespace :return: str path to tablespace """ tablespace_catalog = 'tablespace-' + str(random.randint(0, 100)) tablespace_path = os.path.join(TMP_DIR, tablespace_catalog) os.mkdir(tablespace_path) os.chown(tablespace_path, pwd.getpwnam("postgres").pw_uid, grp.getgrnam("postgres").gr_gid) return tablespace_path def create_tablespace(self, tablespace_name, compression=False): """ Create tablespace :return: """ tablespace_location = self.create_tablespace_directory() conn_string = "host='localhost' user='******' " conn = psycopg2.connect(conn_string) conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) cursor = conn.cursor() # TODO add check that PGPRO edition is enterprise if compression: cursor.execute('CREATE TABLESPACE {0} LOCATION \'{1}\'' ' WITH (compression=true);'.format( tablespace_name, tablespace_location)) else: cursor.execute('CREATE TABLESPACE {0} LOCATION \'{1}\';'.format( tablespace_name, tablespace_location)) return tablespace_location @pytest.mark.test_compression_standalone_positive def test_compression_standalone_positive(self, request, install_postgres): """ Test for compression feature. Scenario: 1. Create test tables 1. Create tablespace with compression 2. Save size of created tables by fixture in file system (count and size of files) 3. Run data generator for tablespace with compression 4. Save size of table with compression 5. Check that files for table in tablespace without compression > that files in tablespace with compression 6. Check tablespace folder for files with *.cfm extension 7. Check that tables has some data """ # Step 1 create_test_table('20', 'pgbench') # Step 2 compression_files_directory = self.create_tablespace('compression', compression=True) # Step 3 data_size_without_compression = self.get_directory_size( install_postgres.get_option('data_directory')) # Step 4 print(data_size_without_compression) install_postgres.set_option('default_tablespace', 'compression') create_test_table(size='20', schema='pgbench') # Step 5 data_size_with_compression = self.get_directory_size( install_postgres.get_option('data_directory')) print(data_size_with_compression) print(compression_files_directory) # Step 6 assert data_size_with_compression < data_size_without_compression # Step 7 compression_files = self.get_filenames(compression_files_directory) assert '.cfm' in compression_files # Step 8 conn_string = "host='localhost' user='******' " conn = psycopg2.connect(conn_string) cursor = conn.cursor() cursor.execute("SELECT * FROM pgbench_tellers LIMIT 10;") result = cursor.fetchall() print(result) assert result != 0 conn.close() drop_test_table(conn_string) @pytest.mark.test_compression_unlogged_tables def test_compression_unlogged_tables(self, request, install_postgres): """ Test for compression feature. Scenario: 1. Create tablespace with compression 2. Run pgbench (unlogged schema) for tablespace with compression 3. Check tablespace folder for files with *.cfm extension """ # Step 1 compression_files_directory = self.create_tablespace( 'compression_unlogged_tables', compression=True) print(compression_files_directory) # Step 2 install_postgres.set_option('default_tablespace', 'compression_unlogged_tables') create_test_table(size='20', schema=self.PGBENCH_SCHEMA_UNLOGGED) # Step 3 compression_files = self.get_filenames(compression_files_directory) assert '.cfm' in compression_files conn_string = "host='localhost' user='******' " conn = psycopg2.connect(conn_string) cursor = conn.cursor() cursor.execute("SELECT * FROM pgbench_tellers_unlogged LIMIT 10;") result = cursor.fetchall() print(result) assert result != 0 conn.close() @pytest.mark.test_compression_negative def test_compression_negative(self, request, install_postgres): """ Test for compression feature. Scenario: 1. Create tablespace with compression 2. Run pgbench for tablespace with compression and kill postgres process 3. Run postgres process and check that postgres is running 4. Check that cfm files created 5. Check that pgbench tables don't created """ version = request.config.getoption('--product_version') name = request.config.getoption('--product_name') edition = request.config.getoption('--product_edition') product_info = " ".join([self.dist, name, edition, version]) # pylint: disable=no-member tag_mark = pytest.allure.label(LabelType.TAG, product_info) request.node.add_marker(tag_mark) # Step 1 compression_files_directory = self.create_tablespace( 'compression_negative', compression=True) print(compression_files_directory) install_postgres.set_option('default_tablespace', 'compression_negative') postgres_pid = install_postgres.get_postmaster_pid() print(postgres_pid) # Step 2 create_table_process = Process(target=create_test_table, args=( '20', 'pgbench', )) create_table_process.start() time.sleep(10) # Step 3 if create_table_process.is_alive(): process = psutil.Process(postgres_pid) process.kill() compression_files = self.get_filenames(compression_files_directory) # Step 4 # Files for tables created but no any table was created # because transaction was corruped assert '.cfm' in compression_files install_postgres.manage_psql('start') # Step 5 conn_string = "host='localhost' user='******' " conn = psycopg2.connect(conn_string) cursor = conn.cursor() cursor.execute("SELECT * FROM pg_tables" " WHERE tablespace=\'compression_negative\';") assert len(cursor.fetchall()) == 0 drop_test_table(conn_string) @pytest.mark.test_compression_unlogged_tables_negative def test_compression_unlogged_tables_negative(self, request, install_postgres): """ Test for compression feature. Scenario: 1. Create tablespace with compression for test 2. Run pgbench (unlogged schema) for tablespace with compression 3. Kill postgres process 4. Start postgres process 5. Check that tables was created and readable 6. Check tablespace folder for files with *.cfm extension """ # Step 1 compression_files_directory = self.create_tablespace( 'compression_unlogged_tables_negative', compression=True) # Step 2 install_postgres.set_option('default_tablespace', 'compression_unlogged_tables_negative') create_test_table(size='20', schema=self.PGBENCH_SCHEMA_UNLOGGED) # Step 3 conn_string = "host='localhost' user='******' " conn = psycopg2.connect(conn_string) cursor = conn.cursor() cursor.execute("select * from pgbench_tellers_unlogged limit 10;") conn.close() process = psutil.Process(install_postgres.get_postmaster_pid()) process.kill() # Step 4 install_postgres.manage_psql('start') # Step 5 conn_string = "host='localhost' user='******' " conn = psycopg2.connect(conn_string) cursor = conn.cursor() cursor.execute( "SELECT * FROM pg_tables" " WHERE tablespace=\'compression_unlogged_tables_negative\';") assert len(cursor.fetchall()) != 0 conn_string = "host='localhost' user='******' " conn = psycopg2.connect(conn_string) cursor = conn.cursor() cursor.execute("SELECT * FROM pgbench_tellers_unlogged LIMIT 10;") result = cursor.fetchall() assert result != 0 conn.close() # Step 6 compression_files = self.get_filenames(compression_files_directory) assert '.cfm' in compression_files @pytest.mark.usefixtures('install_postgres') @pytest.mark.test_compression_alter_tablepsace_to_compression def test_compression_alter_tablepsace_to_compression(self, request): """Scenario: 1. Create tablespace for test 2. Create test data with pgbench 3. Save rows count from pgbench_history 4. Move table to branch with compression 5. Check that rows count before and after moving the same """ # Step 1 self.create_tablespace('compression_alter_tablepsace', compression=True) # Step 2 conn_string = "host=localhost user=postgres" load_pgbench(conn_string, ["-i", "-s", "10"]) load_pgbench(conn_string, ["-s", "10", "-t", "1000", "-c", "10"]) # Step 3 conn = psycopg2.connect(conn_string) cursor = conn.cursor() cursor.execute("select count(*) from pgbench_history") rows_before_move = cursor.fetchall()[0][0] conn.commit() conn.close() # Step 4 conn_string = "host='localhost' user='******' " conn = psycopg2.connect(conn_string) cursor = conn.cursor() cursor.execute("ALTER TABLE pgbench_history" " SET TABLESPACE compression_alter_tablepsace") cursor.execute("select count(*) from pgbench_history") rows_after_move = cursor.fetchall()[0][0] conn.commit() conn.close() # Step 5 assert rows_before_move == rows_after_move @pytest.mark.usefixtures('install_postgres') @pytest.mark.test_compression_alter_tablepsace_from_compression def test_compression_alter_tablepsace_from_compression(self, request): """Scenario: 1. Create tablespace for test 2. Create test data with pgbench 3. Save rows count from pgbench_history 4. Move table to branch without compression 5. Check that rows count before and after moving the same """ # Step 1 self.create_tablespace('compression_alter_from_tablepsace', compression=True) # Step 2 conn_string = "host=localhost user=postgres" load_pgbench(conn_string, ["-i", "-s", "10"]) load_pgbench(conn_string, ["-s", "10", "-t", "1000", "-c", "10"]) # Step 3 conn = psycopg2.connect(conn_string) cursor = conn.cursor() cursor.execute("select count(*) from pgbench_history") rows_before_move = cursor.fetchall()[0][0] conn.commit() conn.close() # Step 4 conn_string = "host='localhost' user='******' " conn = psycopg2.connect(conn_string) cursor = conn.cursor() cursor.execute("ALTER TABLE pgbench_history" " SET TABLESPACE compression_alter_from_tablepsace") cursor.execute("select count(*) from pgbench_history") rows_after_move = cursor.fetchall()[0][0] conn.commit() conn.close() # Step 5 assert rows_before_move == rows_after_move
def test_make_check(self, request): """ Scenario: 1. Install current version 2. Check that setup successfull We need to perform the test on Windows in two stages: First we setup the server and prepare environment, then we exclude current user from the Administrators group. Second we execute `make installcheck` without admin rights. :return: """ dist = "" if self.system == 'Linux': dist = " ".join(get_distro()[0:2]) elif self.system == 'Windows': dist = 'Windows' else: raise Exception("OS %s is not supported." % self.system) version = request.config.getoption('--product_version') name = request.config.getoption('--product_name') edition = request.config.getoption('--product_edition') milestone = request.config.getoption('--product_milestone') target = request.config.getoption('--target') product_info = " ".join([dist, name, edition, version]) pgid = '%s-%s' % (edition, version) tag_mark = allure.label(LabelType.TAG, product_info) request.node.add_marker(tag_mark) branch = request.config.getoption('--branch') # Step 1 pginst = PgInstall(product=name, edition=edition, version=version, milestone=milestone, branch=branch, windows=(self.system == 'Windows')) request.cls.pginst = pginst pginst.make_check_passed = False curpath = os.path.dirname(os.path.abspath(__file__)) if pginst.os.is_altlinux() and pginst.os.os_arch == 'aarch64': os.environ['LANG'] = 'en_US.UTF-8' if self.system == 'Windows': if os.path.exists(pginst.get_default_bin_path()): # Refresh environment to get correct PYTHONHOME refresh_env_win() # The instance is already installed and # installcheck environment is presumably prepared, # so just run make_installcheck (once more) subprocess.check_call( '"%s" "%s" "%s"' % (os.path.join(curpath, 'make_installcheck.cmd'), get_pg_prefix(pginst), pginst.service_name), shell=True) pginst.make_check_passed = True return pginst.setup_repo() print("Running on %s." % target) tarball = pginst.download_source() tar = tarfile.open(tarball, 'r:bz2') tar.extractall() tar.close() for comp in [ 'orafce', 'plv8', 'pgpro-stats', 'pgpro-pwr', 'pgpro-controldata', 'pg-filedump', 'pg-portal-modify', 'pg-repack' ]: pkgname = '%s-%s-%s' % (comp, edition, version) if pkgname not in pginst.get_packages_in_repo(): pkgname = comp if pkgname not in pginst.get_packages_in_repo(): continue pginst.download_source(comp.replace('-', '_'), pginst.get_package_version(pkgname), 'tar.gz') if self.system != 'Windows': pginst.install_full() pginst.initdb_start() else: pginst.install_perl_win() pginst.install_postgres_win(port=55432) try: pginst.exec_psql('CREATE EXTENSION plpython3u') except Exception: subprocess.check_call('SETX PYTHONHOME C:\\Python27 -m', shell=True) if version != "9.6" or self.system == 'Windows' or \ (edition == '1c' and pginst.os_name not in DEBIAN_BASED): buildinfo = os.path.join(pginst.get_pg_prefix(), 'doc', 'buildinfo.txt') else: buildinfo = subprocess.check_output( 'ls /usr/share/doc/postgres*pro*/buildinfo.txt', shell=True).decode(ConsoleEncoding).strip() with open(buildinfo, 'r') as bi: bitxt = bi.read() assert (re.search(r'^Documentation translation', bitxt, re.MULTILINE)) assert (re.search(r'^Source', bitxt, re.MULTILINE)) assert (re.search(r'^SPEC', bitxt, re.MULTILINE)) print("The binary package buildinfo:\n%s\n" % bitxt) pginst.install_default_config() pginst.exec_psql("ALTER SYSTEM SET max_worker_processes = 16") pginst.exec_psql("ALTER SYSTEM SET lc_messages = 'C'") # Prepare pg_hba.conf for src/interfaces/ecpg/test/connect/test5 with open(os.path.join(pginst.get_configdir(), 'pg_hba.conf'), 'r+') as conf: hba = re.sub(r'^(local\s+all\s+all\s+peer)$', "local all regress_ecpg_user1 md5\n" "local all regress_ecpg_user2 md5\n" "local all regress_hacker trust\n" "local all regress_superuser trust\n" "local all nosuper trust\n" r'\1', conf.read(), flags=re.MULTILINE) conf.seek(0) conf.write(hba) pginst.load_shared_libraries(restart_service=False) pginst.restart_service() if self.system != 'Windows': subprocess.check_call( '"%s" "%s" "%s"' % (os.path.join(curpath, 'make_installcheck.sh'), get_pg_prefix(pginst), pginst.service_name), shell=True) pginst.make_check_passed = True else: # First run is performed to setup the environment subprocess.check_call( '"%s" "%s" "%s"' % (os.path.join(curpath, 'make_installcheck.cmd'), get_pg_prefix(pginst), pginst.service_name), shell=True) request.session.customexitstatus = 222
import platform import distro import pytest from allure_commons.types import LabelType from helpers.utils import create_env_info_from_config, get_distro from helpers.environment_manager import Environment if platform.system() == 'Linux': dist = " ".join(get_distro()[0:2]) elif platform.system() == 'Windows': dist = 'Windows' else: print("Unknown Distro") def pytest_addoption(parser): """Option for pytest run, list of images for test :param parser pytest default param for command line args: :return: """ parser.addoption( "--target", dest="target", help='system(s) under test (image(s))') parser.addoption( "--product_version", action="store", default='9.6', help="Specify product version. Available values: 9.5, 9.6") parser.addoption( "--product_name", action="store", default='postgrespro',
def test_install(self, request): """ Install pg_probackup utility and configure postgresql for running pg_probackup. Scenario: 1. Install postgrespro version 2. Check version function 3. Check help function """ if self.system == 'Linux': dist = " ".join(get_distro()[0:2]) elif self.system == 'Windows': dist = 'Windows' else: raise Exception("OS %s is not supported." % self.system) version = request.config.getoption('--product_version') name = request.config.getoption('--product_name') edition = request.config.getoption('--product_edition') milestone = request.config.getoption('--product_milestone') target = request.config.getoption('--target') product_info = " ".join([dist, name, edition, version]) tag_mark = allure.label(LabelType.TAG, product_info) request.node.add_marker(tag_mark) branch = request.config.getoption('--branch') request.cls.skip = False if name != 'postgrespro' or edition == '1c' or version == '13': # TODO: Enable this test for v13 print("PgProBackup test is only for postgrespro std and ent.") request.cls.skip = True return # Step 1 self.pginst = PgInstall(product=name, edition=edition, version=version, milestone=milestone, branch=branch, windows=(self.system == 'Windows')) self.pginst.setup_repo() if self.pginst.os.is_altlinux()and self.pginst.os.os_arch == 'aarch64': os.environ['LANG'] = 'en_US.UTF-8' if self.system != 'Windows': self.pginst.install_full() self.pginst.initdb_start() else: self.pginst.install_postgres_win() self.fix_permissions(tempdir) self.bindir = self.pginst.get_bin_path() + os.sep print('Bindir is %s' % self.bindir) self.pginst.load_shared_libraries(restart_service=True) request.cls.pginst = self.pginst # Step 2 assert self.execute_pg_probackup("--help") ver = self.execute_pg_probackup("--version") print('Version is %s' % ver) vere = re.search(r'[^(]+\s([0-9.]+)\s.*', ver) self.version = vere.group(1) assert self.version request.cls.version = self.version