def test_upgrade_minor(self, request):
"""
Scenario:
1. Install current version
2. Check that setup successfull (select version)
:return:
"""
global windows_os
distro = get_distro()
if distro[2] == 'x86_64' or self.system == 'Windows':
distro = distro[:-1]
dist = " ".join(distro)
if self.system == 'Linux':
windows_os = False
elif self.system == 'Windows':
windows_os = True
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
if edition not in ['std', 'ent', '1c']:
print("Minor upgrade only for 1c, std and ent")
return
if name != 'postgrespro':
print("Minor upgrade test is only for postgrespro.")
return
small_key = "-".join([name, edition, version])
specified_version = False
if dist in ARCHIVE_VERSIONS \
and small_key in ARCHIVE_VERSIONS[dist]:
specified_version = ARCHIVE_VERSIONS[dist][small_key]
if specified_version is None:
return "%s %s %s does not support archived versions on %s." % \
(name, edition, version, dist)
print("specified version is %s" % specified_version)
print("Running on %s." % target)
pgnew = PgInstall(product=name,
edition=edition,
version=version,
milestone=milestone,
branch=branch,
windows=windows_os)
if pgnew.os.is_altlinux() and pgnew.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
pgnew.setup_repo()
if not windows_os:
pgnew.install_full()
subprocess.check_call('cp -a "%s" "%s"' %
(pgnew.get_pg_prefix(), client_dir),
shell=True)
pgnew.remove_full(False, True)
pgnew.remove_data(True)
# PGPRO-3310
if pgnew.os_name in DEBIAN_BASED:
remove_alternatives()
else:
pgnew.install_postgres_win()
pgnew.stop_service()
subprocess.check_call('xcopy /S /E /O /X /I /Q "%s" "%s"' %
(pgnew.get_pg_prefix(), client_dir),
shell=True)
pgnew.remove_full(True)
pgconfig = subprocess.check_output(
'"%s"' % os.path.join(client_dir, 'bin', 'pg_config'),
shell=True).decode(ConsoleEncoding)
vere = re.search(r'PGPRO\_VERSION\s=\s([0-9.]+)', pgconfig)
if (vere):
current_ver = vere.group(1)
else:
vere = re.search(r'VERSION\s=\s\w+\s([0-9.]+)', pgconfig)
current_ver = vere.group(1)
print("Current version is %s" % current_ver)
test_versions = get_test_versions(edition, version, specified_version,
current_ver)
if test_versions is None:
print("No archive versions found.")
return
print(test_versions)
dump_file_name = download_dump(name, edition, version + '-old',
tempdir)
for oldversion in test_versions:
print("Installing", oldversion)
key = "-".join([name, edition, oldversion])
pgold = PgInstall(product=name,
edition=edition,
version=oldversion,
milestone='archive',
branch=None,
windows=windows_os)
if pgold.os.is_altlinux() and pgold.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
pgold.setup_repo()
if not windows_os:
# PGPRO-3889
if (pgold.os_name.startswith('CentOS') or
pgold.os_name.startswith('Red Hat') or
pgold.os_name.startswith('Oracle Linux')) and \
pgold.os_version.startswith('8'):
for pkg in pgold.all_packages_in_repo[:]:
if ('jit' in pkg):
pgold.all_packages_in_repo.remove(pkg)
if (pgold.os_name.startswith('SLES')
and pgold.os_version.startswith('15')):
for pkg in pgold.all_packages_in_repo[:]:
if 'zstd' in pkg:
pgold.all_packages_in_repo.remove(pkg)
# PGPRO-2954
for pkg in pgold.all_packages_in_repo[:]:
if 'bouncer' in pkg or 'badger' in pkg:
pgold.all_packages_in_repo.remove(pkg)
if pgnew.os_name == 'ROSA Enterprise Linux Server' \
and pgnew.os_version.startswith('7.3') \
and edition == 'std' \
and version == '9.6' \
and compare_versions(oldversion, '9.6.13.1') == 0:
for pkg in pgold.all_packages_in_repo[:]:
if 'probackup' in pkg:
pgold.all_packages_in_repo.remove(pkg)
pgold.install_full()
pgold.initdb_start()
else:
pgold.install_postgres_win()
pgold.load_shared_libraries()
with open(os.path.join(tempdir, 'load-%s.log' % oldversion),
'wb') as out:
pgold.exec_psql_file(
dump_file_name,
'-q%s' %
('' if pgold.os_arch == 'x86' else ' -v ON_ERROR_STOP=1'),
stdout=out)
expected_file_name = os.path.join(tempdir, "%s-expected.sql" % key)
dumpall(pgold, expected_file_name)
pgold.delete_repo()
pgnew = PgInstall(product=name,
edition=edition,
version=version,
milestone=milestone,
branch=None,
windows=windows_os)
if pgnew.os.is_altlinux() and pgnew.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
pgnew.setup_repo()
pgold.stop_service()
if not windows_os:
pgnew.update_all_packages()
pgnew.start_service()
else:
pgnew.install_postgres_win()
result_file_name = os.path.join(tempdir, "%s-result.sql" % key)
dumpall(pgnew, result_file_name)
diff_dbs(expected_file_name, result_file_name,
os.path.join(tempdir, "%s.sql.diff" % key))
pgnew.stop_service()
repo_diff = list(
set(pgold.all_packages_in_repo) -
set(pgnew.all_packages_in_repo))
print("repo diff is %s" % repo_diff)
for package in repo_diff:
try:
pgold.remove_package(package)
except Exception:
pass
pgnew.remove_full(True)
# PGPRO-3310
if pgnew.os_name in DEBIAN_BASED:
remove_alternatives()
if pgold.os_name in DEBIAN_BASED and version == '9.6':
try:
subprocess.check_call("apt-get purge -y 'postgres*'",
shell=True)
except Exception:
pass
# PGPRO-2563
if pgold.os_name == 'Ubuntu' and version == '9.6' and \
edition == 'ent':
time.sleep(20)
class TestPgprobackup():
system = platform.system()
bindir = ''
def fix_permissions(self, dir):
if self.system != 'Windows':
import pwd
import grp
os.chown(dir,
pwd.getpwnam("postgres").pw_uid,
grp.getgrnam("postgres").gr_gid)
for root, dirs, files in os.walk(dir):
for d in dirs:
os.chown(os.path.join(root, d),
pwd.getpwnam("postgres").pw_uid,
grp.getgrnam("postgres").gr_gid)
for f in files:
os.chown(os.path.join(root, f),
pwd.getpwnam("postgres").pw_uid,
grp.getgrnam("postgres").gr_gid)
else:
# Grant Full Access to "Network Service" and Users
subprocess.check_call(
'icacls "%s" /grant *S-1-5-32-545:(OI)(CI)F /T' % dir,
shell=True)
subprocess.check_call(
'icacls "%s" /grant *S-1-5-20:(OI)(CI)F /T' % dir,
shell=True)
subprocess.check_call(
r'icacls "%s\*" /grant *S-1-5-32-545:(OI)(CI)F /T' % dir,
shell=True)
subprocess.check_call(
r'icacls "%s\*" /grant *S-1-5-20:(OI)(CI)F /T' % dir,
shell=True)
def execute_pg_probackup(self, *options):
"""
:param options: strings with params for pg_probackup
:return: output of command execution
"""
cmd = '%s"%spg_probackup" %s' % (
self.pginst.pg_sudo_cmd, self.bindir, " ".join(list(options)))
ret = subprocess.check_output(cmd, shell=True).decode()
return ret
def download_source(self):
if self.pginst.milestone == 'alpha':
baseurl = PGPRO_DEV_SOURCES_BASE
elif self.pginst.milestone == 'beta':
baseurl = PGPRO_STABLE_SOURCES_BASE
else:
baseurl = PGPRO_ARCHIVE_SOURCES_BASE
tar_href = 'pg_probackup-%s.tar.gz' % self.version
# tar_file = os.path.join(tempdir, tar_href)
urlretrieve(baseurl + '/' + tar_href, tar_href)
return tar_href
def test_install(self, request):
""" Install pg_probackup utility and configure postgresql
for running pg_probackup.
Scenario:
1. Install postgrespro version
2. Check version function
3. Check help function
"""
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
request.cls.skip = False
if name != 'postgrespro' or edition == '1c' or version == '13':
# TODO: Enable this test for v13
print("PgProBackup test is only for postgrespro std and ent.")
request.cls.skip = True
return
# Step 1
self.pginst = PgInstall(product=name, edition=edition, version=version,
milestone=milestone, branch=branch,
windows=(self.system == 'Windows'))
self.pginst.setup_repo()
if self.pginst.os.is_altlinux()and self.pginst.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
if self.system != 'Windows':
self.pginst.install_full()
self.pginst.initdb_start()
else:
self.pginst.install_postgres_win()
self.fix_permissions(tempdir)
self.bindir = self.pginst.get_bin_path() + os.sep
print('Bindir is %s' % self.bindir)
self.pginst.load_shared_libraries(restart_service=True)
request.cls.pginst = self.pginst
# Step 2
assert self.execute_pg_probackup("--help")
ver = self.execute_pg_probackup("--version")
print('Version is %s' % ver)
vere = re.search(r'[^(]+\s([0-9.]+)\s.*', ver)
self.version = vere.group(1)
assert self.version
request.cls.version = self.version
def test_pgprobackup_internal(self, request):
if request.cls.skip:
print("PgProBackup test is only for postgrespro std and ent.")
return
self.pginst = request.cls.pginst
if sys.version_info > (3, 0) and \
compare_versions(self.version, '2.4.5') < 0:
print("Only 2 python temporary supported")
return
if self.system == 'Windows':
print("Only linuxes temporary supported")
return
self.version = request.cls.version
self.bindir = self.pginst.get_bin_path() + os.sep
os.environ['PATH'] = os.environ['PATH'] + ':' + self.bindir
tar_file = self.download_source()
print(tar_file)
tar = tarfile.open(tar_file, "r:gz")
tar.extractall()
tar.close()
dir = '.'.join(tar_file.split('.')[:-2])
print(dir)
# Patch tests for 2.4.2
if self.version == '2.4.2':
urlretrieve('https://github.com/postgrespro/pg_probackup/raw/8147'
'001/tests/incr_restore.py',
os.path.join(dir, 'tests', 'incr_restore.py'))
self.fix_permissions(dir)
subprocess.check_call('pip%s install testgres%s' %
(sys.version_info[0],
'==1.8.2' if sys.version_info[0] == 2 else ''),
shell=True)
# PGPRO-4108 wait ptrack2.0 in 10
cmd = "%s sh -c 'PG_CONFIG=\"%s/pg_config\"" \
" LANG=C PG_PROBACKUP_PTRACK=%s " \
" PG_PROBACKUP_TEST_BASIC=ON python%s -m unittest -v tests'" \
% (self.pginst.pg_sudo_cmd, self.pginst.get_bin_path(),
'ON' if compare_versions(self.pginst.version, '10') > 0
else 'OFF',
'2.7' if self.pginst.os_name in REDHAT_BASED and
self.pginst.os_version.startswith('6') else
sys.version_info[0])
print(subprocess.check_output(cmd, cwd=dir, shell=True).decode())
print("OK")
def test_pgprobackup_simple_backup_restore(self, request):
"""Test pg_probackup with compression feature
and full continous backup
Scenario:
1. Create backup dir
3. Init backup dir
4. Set options for pg_probackup
5. Create tablespace with compression
6. Create table in compression tablespace
7. Edit pg_hba_config
8. Make full backup
9. Get backup id
10. Check that backup status is OK
11. Check that backup validation is OK
"""
if request.cls.skip:
print("PgProBackup test is only for postgrespro std and ent.")
return
self.pginst = request.cls.pginst
self.bindir = self.pginst.get_bin_path() + os.sep
# Step 1
backup_dir = os.path.join(tempdir, 'backup')
os.mkdir(backup_dir)
self.fix_permissions(backup_dir)
# Step 3
self.execute_pg_probackup("init", "-B", '"%s"' % backup_dir)
self.execute_pg_probackup("add-instance", "-B", '"%s"' % backup_dir,
"-D", '"%s"' %
self.pginst.get_default_datadir(),
"--instance", "main")
# Step 5
if self.pginst.edition == 'ent':
self.pginst.exec_psql("ALTER SYSTEM SET cfs_gc_workers TO '0'")
if self.pginst.version == '9.6':
with open(os.path.join(self.pginst.get_default_configdir(),
"pg_hba.conf"), "a") as hba:
hba.write("host replication all all trust") \
if self.system == 'Windows' else \
hba.write("local replication all trust")
self.pginst.exec_psql("ALTER SYSTEM SET wal_level TO 'replica'")
self.pginst.exec_psql("ALTER SYSTEM SET max_wal_senders TO '1'")
self.pginst.restart_service()
dump_file_name = download_dump(self.pginst.product,
self.pginst.edition,
self.pginst.version, tempdir)
with open(os.path.join(tempdir, 'load-dump.log'), 'wb') as out:
self.pginst.exec_psql_file(dump_file_name, '-q',
stdout=out)
# Step 6
tablespace_path = os.path.join(tempdir, 'pgprobackup')
os.mkdir(tablespace_path)
self.fix_permissions(tablespace_path)
create_command = 'CREATE TABLESPACE pgprobackup ' \
'LOCATION \'%s\'%s;' % \
(tablespace_path,
' WITH(compression = true)' if
self.pginst.edition == 'ent' else '')
self.pginst.exec_psql(create_command)
# Step 7
self.pginst.exec_psql(
"CREATE TABLE tbl TABLESPACE pgprobackup"
" AS SELECT i, rpad('',30,'a')"
" FROM generate_series(0,100000) AS i;")
before_backup_file = os.path.join(tempdir, 'before.sql')
self.pginst.do_in_all_dbs(truncate_unlogged_sql)
self.pginst.exec_client_bin('pg_dumpall', '-f "%s"' %
before_backup_file)
# Step 8
self.execute_pg_probackup("backup", "-b", "full", "-B",
'"%s"' % backup_dir, "-d", "postgres",
"-U", "postgres", "--instance",
"main", "--stream", "" if
self.pginst.get_default_datadir() ==
self.pginst.get_default_configdir()
else "--external-dirs=%s" %
self.pginst.get_default_configdir()
)
# Step 9
# Get last backup id and get out for show command with this backup
pgprobackup_show = json.loads(
self.execute_pg_probackup("show", "-B", '"%s"' % backup_dir, "-U",
"postgres", "--format=json"))
backup_id = pgprobackup_show[0]['backups'][0]['id']
# Step 10
assert pgprobackup_show[0]['backups'][0]['status'] == 'OK'
# Step 11
print("Validating backup %s" % backup_id)
self.execute_pg_probackup("validate", "-i", backup_id, "-B",
'"%s"' % backup_dir, '--instance', 'main')
# Step 12
print("Drop data and restore backup")
self.pginst.stop_service()
self.pginst.remove_data()
shutil.rmtree(tablespace_path)
self.execute_pg_probackup("restore", "-i", backup_id, "-B",
'"%s"' % backup_dir, '--instance', 'main')
# PBCKP-91
if self.system == 'Windows':
self.fix_permissions(self.pginst.get_default_datadir())
self.fix_permissions(tablespace_path)
self.pginst.start_service()
after_backup_file = os.path.join(tempdir, 'after.sql')
self.pginst.exec_client_bin('pg_dumpall', '-f "%s"' %
after_backup_file)
diff_dbs(before_backup_file, after_backup_file,
os.path.join(tempdir, 'diff.sql'))