def test_validator_processing_message_new(self, ServiceObjKAFKAavro):
allure.description('Testing validator processing message new')
allure.label('validator operations')
startTime = '2018-06-20T09:08:28+03:00'
endTime = '2018-07-29T09:08:28+03:00'
dateFrom = '2018-07-02T09:19:32+03:00'
dateTo = '2018-07-02T09:19:32+03:00'
log.debug('[test_get_history] dateFrom: ' + str(dateFrom))
device_v_term_id = random.randint(1, 100000)
filter = 'null'
limit = 1
sort = 'asc'
expected = []
for time in range(2):
device_activity_info = random.randint(1, 100000)
device_time_device = 1530523171000 + 1000 * time
device_time_device_utc = datetime.fromtimestamp(
int(device_time_device / 1000)).strftime('%Y-%m-%dT%H:%M:%SZ')
ServiceObjKAFKAavro.send_valid(termId=device_v_term_id,
time_platform=device_time_device,
time_device=device_time_device,
activity_info=device_activity_info)
def test_send_raw_to_validator(self, KAFKAjson, KAFKAjson2):
allure.description('Testing send raw json to validator topic')
allure.label('kafka operations')
d1 = nistest.KafkaDriver(topic=config.get('kafka_topic_raw'),
server=config.get('kafka_server'),
schema_registry=config.get('kafka_schema'))
offset1 = d1.get_last_offset()
rawData = {
"producer": "test",
"sessionKey": "123e4567-e89b-12d3-a456-426655440002",
"termId": 123,
"protocolId": 1,
"deviceTypeId": 100,
"timePlatformFrom1970": 1527601137000,
"timeDeviceFrom1970": 1527601137000,
"activityInfo": 1
}
rawData = json.dumps(rawData, sort_keys=True).encode('utf-8')
last_offset = d1.get_last_offset()
d1.send(msg=rawData, lang='json', schema=None)
res = d1.read_from_offset(offset1)
result = d1.read_from_offset(offset=last_offset,
lang='json',
schema=None)
log.debug('result: ' + str(result))
assert rawData == result
res_delete = d1.delete_topic()
assert res_delete == 0
def test_create_kafka(self):
allure.description('Testing create topic')
allure.label('kafka operations')
log.debug('start test_kafka')
d1 = nistest.KafkaDriver(topic=config.get('kafka_topic_raw'))
res_create = d1.create_topic('testtopic2')
assert res_create == None
def test_send_raw_to_validator_new(self, KAFKAjson, KAFKAjson2):
allure.description('Testing send raw json to validator topic')
allure.label('kafka operations')
device_time_device = 1530523171000
device_time_device_utc = datetime.fromtimestamp(
int(device_time_device / 1000)).strftime('%Y-%m-%dT%H:%M:%SZ')
KAFKAjson.send_raw(termId=100,
protocolId=200,
deviceTypeId=100,
timePlatformFrom1970=device_time_device,
timeDeviceFrom1970=device_time_device,
activityInfo=300)
def test_clean_install(self, request):
"""
Scenario:
1. Install current version
2. Check that setup successfull (select version)
:return:
"""
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
# Step 1
pginst = PgInstall(product=name,
edition=edition,
version=version,
milestone=milestone,
branch=branch,
windows=(self.system == 'Windows'))
if pginst.os.is_altlinux() and pginst.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
request.cls.pginst = pginst
pginst.setup_repo()
print("Running on %s." % target)
print("Minor product version is: %s\n" %
pginst.get_product_minor_version())
if self.system != 'Windows':
pginst.install_base()
pginst.initdb_start()
else:
pginst.install_postgres_win()
server_version = pginst.get_server_version()
client_version = pginst.get_psql_version()
print("Server version:\n%s\nClient version:\n%s" %
(server_version, client_version))
print("OK")
def test_extensions_prepare(self, request):
"""
Scenario:
1. Install current version
:return:
"""
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
# Step 1
pginst = PgInstall(product=name, edition=edition,
version=version, milestone=milestone,
branch=branch, windows=(self.system == 'Windows'))
request.cls.pginst = pginst
pginst.setup_repo()
print("Running on %s." % target)
if pginst.os.is_altlinux() and pginst.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
if self.system != 'Windows':
pginst.install_full()
pginst.initdb_start()
else:
pginst.install_postgres_win()
def test_full_install(self, request):
"""
Scenario:
1. Install current version
2. Check that setup successfull
:return:
"""
dist = ""
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
request.cls.pgid = '%s-%s' % (edition, version)
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
# Step 1
pginst = PgInstall(product=name,
edition=edition,
version=version,
milestone=milestone,
branch=branch,
windows=(self.system == 'Windows'))
# This is a workaround for the problem described in PGPRO-3596
if pginst.os.is_altlinux() and pginst.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
request.cls.pginst = pginst
pginst.setup_repo()
print("Running on %s." % target)
all_available_packages = pginst.all_packages_in_repo
print("All available packages in repo %s:\n" % pginst.reponame,
"\n".join(all_available_packages))
if self.system != 'Windows':
pginst.install_full()
pginst.install_package(" ".join(all_available_packages))
check_package_contents(pginst, all_available_packages)
check_executables(pginst, all_available_packages)
pginst.initdb_start()
else:
pginst.install_perl_win()
pginst.install_postgres_win()
server_version = pginst.get_server_version()
client_version = pginst.get_psql_version()
print("Server version:\n%s\nClient version:\n%s" %
(server_version, client_version))
if name == 'postgrespro' and not (edition in ['1c', 'sql']):
ppversion = pginst.exec_psql_select("SELECT pgpro_version()")
# PGPRO-3760
assert ppversion.startswith('PostgresPro ')
# assert ppversion.startswith('PostgresPro ' + version)
ppedition = pginst.exec_psql_select("SELECT pgpro_edition()")
if edition == 'ent':
assert ppedition == 'enterprise'
elif edition == 'ent-cert':
assert ppedition == 'enterprise'
else:
assert ppedition == 'standard'
print('pgpro_source_id:',
pginst.exec_psql_select("SELECT pgpro_build()"))
if version not in ["9.6", "10", "13"]:
pginst.env = {}
for var in os.environ:
pginst.env[var] = str(os.environ[var])
pginst.env["LANG"] = 'C'
# PGPRO-4100 TODO: Use pgpro_controldata
cdout = pginst.exec_server_bin('pg_controldata', '"%s"' %
pginst.get_datadir()).split('\n')
if name == 'postgrespro' and not (edition in ['1c', 'sql']):
assert cdout[0].startswith('pg_control edition:')
cdedition = cdout[0].replace('pg_control edition:', '').strip()
if edition == 'ent':
assert cdedition == 'Postgres Pro Enterprise'
elif edition == 'std':
assert cdedition == 'Postgres Pro Standard'
print("OK")
def test_kafka_send_avro(self, KAFKAjson, KAFKAjson2):
allure.description('Testing send raw json to validator topic')
allure.label('kafka operations')
d2 = nistest.KafkaDriver(topic=config.get('kafka_topic_valid'),
server=config.get('kafka_server'),
schema_registry=config.get('kafka_schema'))
res_delete = d2.delete_topic()
offset2 = d2.get_last_offset()
schemaSF = '''{
"type":"record",
"name":"Farming",
"namespace":"nis.dev.validator.avro",
"fields":[
{
"name": "term_id",
"type": "long"
},
{
"name": "time_platform",
"type": {
"type": "long",
"logicalType": "timestamp-millis"
}
},
{
"name": "time_device",
"type": {
"type": "long",
"logicalType": "timestamp-millis"
}
},
{
"name": "activity_info",
"type": "int"
}
]
}'''
rawData2 = {
"term_id": 100,
"time_platform": 1527685363266,
"time_device": 1527685363266,
"activity_info": 300
}
expected = json.dumps(rawData2, indent=4, sort_keys=True, default=str)
last_offset = d2.get_last_offset()
d2.send(msg=rawData2, lang='avro', schema=schemaSF)
res = d2.read_from_offset(offset2, 'avro', schemaSF)
result = d2.read_from_offset(offset=offset2,
lang='avro',
schema=schemaSF)
log.debug('result: ' + str(result))
assert expected == result
res_delete = d2.delete_topic()
assert res_delete == 0
def test_multimaster_install(self, request):
"""
Scenario:
1. Install current version
2. Check that setup successfull (select version)
:return:
"""
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
if version.startswith('9.') or version == '10' \
or version == '13' \
or not edition.startswith('ent'):
print('Version %s %s is not supported' % (edition, version))
return
elif version == '11' and dist == 'Windows':
print('Windows is not supported under version %s' % version)
return
# Step 1
pginst = PgInstall(product=name,
edition=edition,
version=version,
milestone=milestone,
branch=branch,
windows=(self.system == 'Windows'))
request.cls.pginst = pginst
if pginst.os.is_altlinux() and pginst.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
pginst.setup_repo()
print("Running on %s." % target)
if self.system != 'Windows':
pginst.install_full_topless()
else:
pginst.install_postgres_win()
pginst.stop_service()
mm = Multimaster(size=2,
pginst=pginst,
rootdir=os.path.abspath(
os.path.join(pginst.get_datadir(), os.pardir)))
mm.start()
for i, cl_state in mm.get_cluster_state_all().items():
if int(cl_state[0]) != i:
raise Exception('Wrong ID on node %i! %i != %i' %
(i, i, cl_state[0]))
if cl_state[1].lower() != 'online':
print(cl_state)
raise Exception('Node %i is not online!' % i)
print(mm.get_nodes_state_all())
for i, ns_all in mm.get_nodes_state_all().items():
for node_state in ns_all:
if node_state[1].rstrip() != 't':
raise Exception('Node %s is not enabled! (on node %i)' %
(node_state[0], i))
pgbench = {}
for i, node in mm.nodes.items():
pgbench[i] = mm.pgbench(i,
type='simple-update',
duration=60,
max_tries=100)
pgbench[1].init()
for i in range(1, mm.size + 1):
if not mm.nodes[i].referee:
print('Running pgbench on node%i' % i)
pgbench[i].start()
mm.wait_for_txid(600)
print('Cluster state:')
print(mm.get_cluster_state_all())
print(mm.get_nodes_state_all())
print('Current TXID (after pgbench): %s' %
mm.psql('SELECT txid_current()', '-Aqt'))
print('Isolating node 2...')
mm.isolate(2)
if mm.has_referee:
mm.wait_for_referee(1, 60)
pgbench[1].wait()
pgbench[1].start()
mm.wait_for_txid(2700)
print('Cluster state:')
print(mm.get_cluster_state_all(allow_fail=True))
print(mm.get_nodes_state_all(allow_fail=True))
print('De-isolating node 2...')
mm.deisolate(2)
print('Current TXID (after isolation): %s' %
mm.psql('SELECT txid_current()', '-Aqt'))
if not mm.check(2):
print('Try to recover node 2')
mm.wait(2, 600)
print('Cluster state:')
print(mm.get_cluster_state_all(allow_fail=True))
print(mm.get_nodes_state_all(allow_fail=True))
mm.wait_for_txid(3500)
for i in range(1, mm.size):
if not mm.nodes[i].referee:
print('Pgbench %i terminated rc=%i' % (i, pgbench[i].stop()))
mm.sure_pgbench_is_dead(i)
pgbench_tables = ('pgbench_branches', 'pgbench_tellers',
'pgbench_accounts', 'pgbench_history')
print('Current TXID (after pgbench termination): %s' %
mm.psql('SELECT txid_current()', '-Aqt'))
for table in pgbench_tables:
result1 = mm.pg_dump(1, [table])
for i in range(2, mm.size + 1):
if not mm.nodes[i].referee:
result = mm.pg_dump(i, [table])
diff_dbs(result1, result, '%s_1_%i.diff' % (table, i))
def test_dev_usage(self, request):
"""
Scenario:
1. Install only -dev package
2. Try to build PGXS extension
:return:
"""
dist = ""
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
# Step 1
pginst = PgInstall(product=name,
edition=edition,
version=version,
milestone=milestone,
branch=branch,
windows=(self.system == 'Windows'))
pginst.setup_repo()
print("Running on %s." % target)
if pginst.os.is_altlinux() and pginst.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
if self.system != 'Windows':
pginst.install_server_dev()
pg_bin_path = pginst.get_default_bin_path()
curpath = os.path.dirname(os.path.abspath(__file__))
test_script = r"""
set -e
if which apt-get; then
apt-get install -y gcc || true
apt-get install -y make
grep -E '(Debian GNU/Linux 9|Debian GNU/Linux 10|'\
'"Ubuntu [0-9]+\.[0-9]+|"OSNova Linux|'\
'"Astra Linux \(Smolensk 1.6\)"|"Astra Linux \(Orel\)")'\
/etc/os-release >/dev/null 2>/dev/null && \
apt install -y libdpkg-perl
elif which zypper; then
zypper install -y gcc make
elif which yum; then
yum install -y gcc make redhat-rpm-config
fi
tar fax ../extras/pg_wait_sampling.tar.gz -C /tmp && \
cd /tmp/pg_wait_sampling*/
export PATH=%s:$PATH
make USE_PGXS=1
make USE_PGXS=1 install
chmod 777 .
""" % (pg_bin_path)
subprocess.check_call(test_script, cwd=curpath, shell=True)
pginst.install_full()
pginst.initdb_start()
pginst.exec_psql('ALTER SYSTEM SET shared_preload_libraries = '
'pg_wait_sampling')
pginst.restart_service()
test_script = r"""
cd /tmp/pg_wait_sampling*/
sudo -u postgres sh -c "export PATH=%s:$PATH; make USE_PGXS=1 installcheck"
""" % (pg_bin_path)
subprocess.check_call(test_script, shell=True)
else:
pginst.install_postgres_win()
pginst.install_perl_win()
print("OK")
def test_validator_processing_message(self):
allure.description('Testing validator processing message')
allure.label('validator operations')
d1 = nistest.KafkaDriver(topic=config.get('kafka_topic_raw'),
server=config.get('kafka_server'),
schema_registry=config.get('kafka_schema'))
d2 = nistest.KafkaDriver(topic=config.get('kafka_topic_valid'),
server=config.get('kafka_server'),
schema_registry=config.get('kafka_schema'))
res1_delete = d2.delete_topic()
#assert res1_delete == 0
res2_delete = d1.delete_topic()
#assert res2_delete == 0
offset1 = d1.get_last_offset()
offset2 = d2.get_last_offset()
rawData = {
"producer": "test",
"sessionKey": "123e4567-e89b-12d3-a456-426655440002",
"termId": 123,
"protocolId": 1,
"deviceTypeId": 100,
"timePlatformFrom1970": 1527601137000,
"timeDeviceFrom1970": 1527601137001,
"activityInfo": 1
}
schemaSF = '''{
"type":"record",
"name":"Farming",
"namespace":"nis.dev.validator.avro",
"fields":[
{
"name": "term_id",
"type": "long"
},
{
"name": "time_platform",
"type": {
"type": "long",
"logicalType": "timestamp-millis"
}
},
{
"name": "time_device",
"type": {
"type": "long",
"logicalType": "timestamp-millis"
}
},
{
"name": "activity_info",
"type": "int"
}
]
}'''
rawData = json.dumps(rawData, sort_keys=True).encode('utf-8')
d1.send(msg=rawData, lang='json', schema=None)
rawData2 = {
"term_id": 123,
"time_platform": 1527601137000,
"time_device": 1527601137001,
"activity_info": 1
}
rawDataJson2 = json.dumps(rawData2,
indent=4,
sort_keys=True,
default=str)
log.debug('offset1: ' + str(offset1))
log.debug('offset2: ' + str(offset2))
res1 = d1.read_from_offset(offset1)
res2 = d2.read_from_offset(offset2, 'avro', schemaSF)
#res2 = json.dumps(res2, sort_keys=True)
log.debug('result1 json: ' + str(res1))
log.debug('result2 avro: ' + str(res2))
assert rawData == res1
assert rawDataJson2 == res2
def test_upgrade_minor(self, request):
"""
Scenario:
1. Install current version
2. Check that setup successfull (select version)
:return:
"""
global windows_os
distro = get_distro()
if distro[2] == 'x86_64' or self.system == 'Windows':
distro = distro[:-1]
dist = " ".join(distro)
if self.system == 'Linux':
windows_os = False
elif self.system == 'Windows':
windows_os = True
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
if edition not in ['std', 'ent', '1c']:
print("Minor upgrade only for 1c, std and ent")
return
if name != 'postgrespro':
print("Minor upgrade test is only for postgrespro.")
return
small_key = "-".join([name, edition, version])
specified_version = False
if dist in ARCHIVE_VERSIONS \
and small_key in ARCHIVE_VERSIONS[dist]:
specified_version = ARCHIVE_VERSIONS[dist][small_key]
if specified_version is None:
return "%s %s %s does not support archived versions on %s." % \
(name, edition, version, dist)
print("specified version is %s" % specified_version)
print("Running on %s." % target)
pgnew = PgInstall(product=name,
edition=edition,
version=version,
milestone=milestone,
branch=branch,
windows=windows_os)
if pgnew.os.is_altlinux() and pgnew.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
pgnew.setup_repo()
if not windows_os:
pgnew.install_full()
subprocess.check_call('cp -a "%s" "%s"' %
(pgnew.get_pg_prefix(), client_dir),
shell=True)
pgnew.remove_full(False, True)
pgnew.remove_data(True)
# PGPRO-3310
if pgnew.os_name in DEBIAN_BASED:
remove_alternatives()
else:
pgnew.install_postgres_win()
pgnew.stop_service()
subprocess.check_call('xcopy /S /E /O /X /I /Q "%s" "%s"' %
(pgnew.get_pg_prefix(), client_dir),
shell=True)
pgnew.remove_full(True)
pgconfig = subprocess.check_output(
'"%s"' % os.path.join(client_dir, 'bin', 'pg_config'),
shell=True).decode(ConsoleEncoding)
vere = re.search(r'PGPRO\_VERSION\s=\s([0-9.]+)', pgconfig)
if (vere):
current_ver = vere.group(1)
else:
vere = re.search(r'VERSION\s=\s\w+\s([0-9.]+)', pgconfig)
current_ver = vere.group(1)
print("Current version is %s" % current_ver)
test_versions = get_test_versions(edition, version, specified_version,
current_ver)
if test_versions is None:
print("No archive versions found.")
return
print(test_versions)
dump_file_name = download_dump(name, edition, version + '-old',
tempdir)
for oldversion in test_versions:
print("Installing", oldversion)
key = "-".join([name, edition, oldversion])
pgold = PgInstall(product=name,
edition=edition,
version=oldversion,
milestone='archive',
branch=None,
windows=windows_os)
if pgold.os.is_altlinux() and pgold.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
pgold.setup_repo()
if not windows_os:
# PGPRO-3889
if (pgold.os_name.startswith('CentOS') or
pgold.os_name.startswith('Red Hat') or
pgold.os_name.startswith('Oracle Linux')) and \
pgold.os_version.startswith('8'):
for pkg in pgold.all_packages_in_repo[:]:
if ('jit' in pkg):
pgold.all_packages_in_repo.remove(pkg)
if (pgold.os_name.startswith('SLES')
and pgold.os_version.startswith('15')):
for pkg in pgold.all_packages_in_repo[:]:
if 'zstd' in pkg:
pgold.all_packages_in_repo.remove(pkg)
# PGPRO-2954
for pkg in pgold.all_packages_in_repo[:]:
if 'bouncer' in pkg or 'badger' in pkg:
pgold.all_packages_in_repo.remove(pkg)
if pgnew.os_name == 'ROSA Enterprise Linux Server' \
and pgnew.os_version.startswith('7.3') \
and edition == 'std' \
and version == '9.6' \
and compare_versions(oldversion, '9.6.13.1') == 0:
for pkg in pgold.all_packages_in_repo[:]:
if 'probackup' in pkg:
pgold.all_packages_in_repo.remove(pkg)
pgold.install_full()
pgold.initdb_start()
else:
pgold.install_postgres_win()
pgold.load_shared_libraries()
with open(os.path.join(tempdir, 'load-%s.log' % oldversion),
'wb') as out:
pgold.exec_psql_file(
dump_file_name,
'-q%s' %
('' if pgold.os_arch == 'x86' else ' -v ON_ERROR_STOP=1'),
stdout=out)
expected_file_name = os.path.join(tempdir, "%s-expected.sql" % key)
dumpall(pgold, expected_file_name)
pgold.delete_repo()
pgnew = PgInstall(product=name,
edition=edition,
version=version,
milestone=milestone,
branch=None,
windows=windows_os)
if pgnew.os.is_altlinux() and pgnew.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
pgnew.setup_repo()
pgold.stop_service()
if not windows_os:
pgnew.update_all_packages()
pgnew.start_service()
else:
pgnew.install_postgres_win()
result_file_name = os.path.join(tempdir, "%s-result.sql" % key)
dumpall(pgnew, result_file_name)
diff_dbs(expected_file_name, result_file_name,
os.path.join(tempdir, "%s.sql.diff" % key))
pgnew.stop_service()
repo_diff = list(
set(pgold.all_packages_in_repo) -
set(pgnew.all_packages_in_repo))
print("repo diff is %s" % repo_diff)
for package in repo_diff:
try:
pgold.remove_package(package)
except Exception:
pass
pgnew.remove_full(True)
# PGPRO-3310
if pgnew.os_name in DEBIAN_BASED:
remove_alternatives()
if pgold.os_name in DEBIAN_BASED and version == '9.6':
try:
subprocess.check_call("apt-get purge -y 'postgres*'",
shell=True)
except Exception:
pass
# PGPRO-2563
if pgold.os_name == 'Ubuntu' and version == '9.6' and \
edition == 'ent':
time.sleep(20)
def test_dump_restore(self, request):
"""
Scenario:
3. if route install upgradeble version
7. Check that upgrade successfull (select from table)
:return:
"""
product_info = request.cls.product_info
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
key = request.cls.key
dist = request.cls.dist
print("Test dump-restore %s" % product_info)
if dist in FIRST_RELEASE and key in FIRST_RELEASE[dist] and \
FIRST_RELEASE[dist][key] is None:
print("Platform not supported")
return
if key not in DUMP_RESTORE_ROUTES:
print('No routes for dump-restore')
return
dump_restore_route = DUMP_RESTORE_ROUTES[key]
pg = request.cls.pg
if pg.os_name in DEBIAN_BASED and pg.version == '9.6':
print("Two products 9.6 cannot be "
"installed simultaneously on debian-based")
return
for route in dump_restore_route['from']:
initdb_params = route['initdb-params'] if \
'initdb-params' in route else ''
init_cluster(pg, True, initdb_params, True, False)
stop(pg)
old_name = route['name']
old_edition = route['edition']
old_version = route['version']
old_key = "-".join([old_name, old_edition, old_version])
if dist in FIRST_RELEASE and old_key in FIRST_RELEASE[dist]:
if FIRST_RELEASE[dist][old_key] is None:
print("Distributive is not supported")
continue
if compare_versions(FIRST_RELEASE[dist][old_key],
get_last_version(old_edition,
old_version)) > 0:
print("Wait for %s" % FIRST_RELEASE[dist][old_key])
continue
print("=====Check dump-restore from %s" % old_key)
file_name = os.path.join(tempdir, "%s.sql" % old_key)
if (os.path.isfile(file_name)):
start(pg)
with open(os.path.join(tempdir, 'load-dr-%s.log' % old_key),
'wb') as out:
pg.exec_psql_file(file_name, '-q', stdout=out)
dump_and_diff_dbs(old_key, pg, 'dump-restore')
stop(pg)
else:
pgold = install_server(product=old_name,
edition=old_edition,
version=old_version,
milestone=None,
branch=None,
windows=(self.system == 'Windows'),
old=True)
if self.system != 'Windows':
init_cluster(pgold, True, initdb_params, None, True)
generate_db(
pgold,
pg,
on_error_stop=False if pgold.os_arch == 'x86' else True)
dumpall(pgold, file_name)
stop(pgold)
start(pg)
with open(os.path.join(tempdir, 'load-dr-%s.log' % old_key),
'wb') as out:
pg.exec_psql_file(file_name, '-q', stdout=out)
dump_and_diff_dbs(old_key, pg, 'dump-restore')
stop(pg)
pgold.remove_full(True,
do_not_remove=[
r"^libc.*", r".*icu.*", r".*zstd.*",
r"^llvm.*"
])
# PGPRO-2459
if pgold.os_name in DEBIAN_BASED and \
old_name == "postgrespro" and old_version == "9.6":
subprocess.check_call(
"apt-get purge -y "
"postgrespro-common "
"postgrespro-client-common",
shell=True)
def test_upgrade(self, request):
"""
Scenario:
1. Install testible version
2. if route install upgradeable version
3. Create DB with covering dump
4. Upgrade by pg_upgrade
5. Check that upgrade successfull (calculate diff between dump)
:return:
"""
distro = get_distro()
if distro[2] == 'x86_64' or self.system == 'Windows':
distro = distro[:-1]
dist = " ".join(distro)
request.cls.dist = dist
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
request.cls.product_info = product_info
key = "-".join([name, edition, version])
request.cls.key = key
print("Running on %s." % target)
if dist in FIRST_RELEASE and key in FIRST_RELEASE[dist] and \
FIRST_RELEASE[dist][key] is None:
print("Platform not supported")
return
if key not in UPGRADE_ROUTES:
print('No routes for upgrade')
return
upgrade_route = UPGRADE_ROUTES[key]
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
# Install the tested version
branch = request.config.getoption('--branch')
pg = install_server(product=name,
edition=edition,
version=version,
milestone=milestone,
branch=branch,
windows=(self.system == 'Windows'))
request.cls.pg = pg
stop(pg)
if pg.os_name in DEBIAN_BASED and pg.version == '9.6':
print("Two products 9.6 cannot be "
"installed simultaneously on debian-based OS")
return
if self.system == 'Windows':
backup_datadir_win(pg)
for route in upgrade_route['from']:
initdb_params = route['initdb-params'] if \
'initdb-params' in route else ''
init_cluster(pg, True, initdb_params, True, False)
stop(pg)
old_name = route['name']
old_edition = route['edition']
old_version = route['version']
old_key = "-".join([old_name, old_edition, old_version])
if dist in FIRST_RELEASE and old_key in FIRST_RELEASE[dist]:
if FIRST_RELEASE[dist][old_key] is None:
print("Distributive is not supported")
continue
if compare_versions(FIRST_RELEASE[dist][old_key],
get_last_version(old_edition,
old_version)) > 0:
print("Wait for %s" % FIRST_RELEASE[dist][old_key])
continue
print("=====Check upgrade from %s" % old_key)
pgold = install_server(product=old_name,
edition=old_edition,
version=old_version,
milestone=None,
branch=None,
windows=(self.system == 'Windows'),
old=True)
if self.system != 'Windows':
init_cluster(pgold, True, initdb_params, None, True)
generate_db(
pgold,
pg,
on_error_stop=False if pgold.os_arch == 'x86' else True)
dumpall(pgold, os.path.join(tempdir, "%s.sql" % old_key))
stop(pgold)
upgrade(pg, pgold)
start(pg)
after_upgrade(pg, pgold)
dump_and_diff_dbs(old_key, pg, 'upgrade')
stop(pg)
pgold.remove_full(do_not_remove=[
r"^libc.*", r".*icu.*", r".*zstd.*", r"^llvm.*"
])
# PGPRO-2459
if pgold.os_name in DEBIAN_BASED and \
old_name == "postgrespro" and old_version == "9.6" and \
old_edition != '1c':
subprocess.check_call(
"apt-get purge -y postgrespro-common "
"postgrespro-client-common",
shell=True)
def test_hotstandby_compat(self, request):
"""
Scenario:
1. Install current version
2. Check that setup successfull (select version)
:return:
"""
global windows_os
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
windows_os = True
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
if name != 'postgrespro':
print("Hot Standby compatibility test is only for postgrespro.")
return
if edition == "ent":
archive_url = PGPRO_ARCHIVE_ENTERPRISE
elif edition == "std":
archive_url = PGPRO_ARCHIVE_STANDARD
else:
raise Exception("Unsupported postgrespro edition (%s)." % edition)
print("Running on %s." % target)
# Choose two versions -- newest and oldest supported
soup = get_soup(archive_url)
arcversions = []
startswith = 'pgproee-' if edition == 'ent' else \
('pgpro-' if edition == 'std' else 'pg1c-')
for link in soup.findAll('a'):
href = link.get('href')
if href.startswith(startswith) and href.endswith('/'):
vere = re.search(r'\w+-([0-9.]+)/', href)
if vere:
if vere.group(1).startswith(version):
arcvers = vere.group(1)
if version == '9.6':
# Due to CATALOG_VERSION_NO change
# we don't support lower 9.6 versions
if compare_versions(arcvers, '9.6.4.1') < 0:
arcvers = None
# PGPRO-3227, PGPRO-3834
if windows_os and version == '10':
if compare_versions(arcvers, '10.11.1'):
arcvers = None
if windows_os and version == '11':
if compare_versions(arcvers, '11.6.1') < 0:
arcvers = None
if arcvers:
arcversions.append(arcvers)
arcversions.sort(key=extend_ver)
if not arcversions:
print("No previous minor versions found. Test skipped.")
return
# Choose first and last versions
testversions = [arcversions[0], arcversions[-1]]
if testversions[0] == testversions[1]:
testversions = [testversions[0]]
# Workaround for unsupported libpq options
fix_extra_libpq_options = False
if edition == 'ent' and version == '10':
if compare_versions(arcversions[0], '10.4.1') < 0:
fix_extra_libpq_options = True
pre12version = version in ['9.6', '10', '11']
if windows_os:
waldir = r'C:\tmp\pgwal'
srcdir = r'C:\tmp'
else:
waldir = os.path.join(tempfile.gettempdir(), 'pgwal')
srcdir = '/var/src'
pgsrcdir = None
for oldversion in testversions:
print("Installing", oldversion)
pgold = PgInstall(product=name, edition=edition,
version=oldversion, milestone='archive',
branch=None, windows=windows_os)
pgold.setup_repo()
if not windows_os:
pgold.install_base()
pgold.initdb_start()
else:
pgold.install_postgres_win()
setup_pgpass('replicator', 'replicator')
server_version = pgold.get_server_version()
client_version = pgold.get_psql_version()
print("Old server version:\n%s\nOld client version:\n%s" %
(server_version, client_version))
pgold.exec_psql('ALTER SYSTEM SET port=15432')
oldpgprefix = pgold.get_pg_prefix()
olddatadir = pgold.get_datadir()
if oldpgprefix == '/usr':
raise Exception("/usr as postgres prefix is not supported.")
if pgold.get_configdir() != pgold.get_datadir():
raise Exception("Separate config dir is not supported.")
pgold.stop_service()
time.sleep(5)
if not windows_os:
subprocess.check_call('cp -a "%s" "%s.old"' %
(oldpgprefix, oldpgprefix), shell=True)
subprocess.check_call('cp -a "%s" "%s.old"' %
(olddatadir, olddatadir), shell=True)
oldpgprefix += ".old"
olddatadir += ".old"
else:
print('xcopy /S /E /O /X /I /Q "%s" "%s.old"' %
(oldpgprefix, oldpgprefix))
subprocess.check_call('xcopy /S /E /O /X /I /Q "%s" "%s.old"' %
(oldpgprefix, oldpgprefix), shell=True)
oldpgprefix += ".old"
olddatadir = os.path.join(oldpgprefix, 'data')
if os.path.exists(os.path.join(olddatadir,
'postgresql.conf.old')):
os.remove(os.path.join(olddatadir, 'postgresql.conf.old'))
pgold.remove_full(remove_data=True)
pgold.pg_prefix = oldpgprefix
pgold.datadir = olddatadir
pgold.configdir = olddatadir
pgold.port = 15432
if not windows_os:
pgold.pg_preexec = 'sudo -u postgres ' \
'LD_LIBRARY_PATH=${LD_LIBRARY_PATH} '
old_env = os.environ.copy()
old_env["LD_LIBRARY_PATH"] = os.path.join(oldpgprefix, 'lib')
pgold.env = old_env
else:
subprocess.check_call(
'sc create postgres-old binpath= '
'"\\"%s\\" runservice -N postgres-old -D \\"%s\\" -w"'
' start= demand obj= "NT Authority\\NetworkService" ' %
(os.path.join(oldpgprefix, 'bin', 'pg_ctl'), olddatadir),
shell=True)
pgold.service_name = 'postgres-old'
pgnew = PgInstall(product=name, edition=edition,
version=version, milestone=milestone,
branch=branch, windows=windows_os)
pgnew.setup_repo()
if not windows_os:
pgnew.install_base()
pgnew.initdb_start()
else:
pgnew.install_postgres_win()
if not pgsrcdir:
pgsrcdir = prepare_pg_regress(pgnew, srcdir)
pgnew.stop_service()
pgnew.remove_data()
# Test replication from old to new
setup_sender(pgold, waldir, pgnew.get_datadir())
start_receiver(pgnew, waldir, pre12version)
run_hs_test(pgold, pgnew, pgsrcdir)
do_server_action(pgnew, "stop")
do_server_action(pgold, "stop")
# Test replication from new to old
pgnew.init_cluster(force_remove=True)
pgold.remove_data()
setup_sender(pgnew, waldir, pgold.get_datadir())
if fix_extra_libpq_options:
workaround_for_extra_libpq_options(pgold)
start_receiver(pgold, waldir, pre12version)
run_hs_test(pgnew, pgold, pgsrcdir)
do_server_action(pgnew, "stop")
do_server_action(pgold, "stop")
pgnew.remove_full(remove_data=True)
shutil.rmtree(olddatadir)
shutil.rmtree(oldpgprefix)
if windows_os:
subprocess.check_call('sc delete postgres-old', shell=True)
print("OK")
def test_make_check(self, request):
"""
Scenario:
1. Install current version
2. Check that setup successfull
We need to perform the test on Windows in two stages:
First we setup the server and prepare environment,
then we exclude current user from the Administrators group.
Second we execute `make installcheck` without admin rights.
:return:
"""
dist = ""
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
pgid = '%s-%s' % (edition, version)
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
# Step 1
pginst = PgInstall(product=name,
edition=edition,
version=version,
milestone=milestone,
branch=branch,
windows=(self.system == 'Windows'))
request.cls.pginst = pginst
pginst.make_check_passed = False
curpath = os.path.dirname(os.path.abspath(__file__))
if pginst.os.is_altlinux() and pginst.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
if self.system == 'Windows':
if os.path.exists(pginst.get_default_bin_path()):
# Refresh environment to get correct PYTHONHOME
refresh_env_win()
# The instance is already installed and
# installcheck environment is presumably prepared,
# so just run make_installcheck (once more)
subprocess.check_call(
'"%s" "%s" "%s"' %
(os.path.join(curpath, 'make_installcheck.cmd'),
get_pg_prefix(pginst), pginst.service_name),
shell=True)
pginst.make_check_passed = True
return
pginst.setup_repo()
print("Running on %s." % target)
tarball = pginst.download_source()
tar = tarfile.open(tarball, 'r:bz2')
tar.extractall()
tar.close()
for comp in [
'orafce', 'plv8', 'pgpro-stats', 'pgpro-pwr',
'pgpro-controldata', 'pg-filedump', 'pg-portal-modify',
'pg-repack'
]:
pkgname = '%s-%s-%s' % (comp, edition, version)
if pkgname not in pginst.get_packages_in_repo():
pkgname = comp
if pkgname not in pginst.get_packages_in_repo():
continue
pginst.download_source(comp.replace('-', '_'),
pginst.get_package_version(pkgname),
'tar.gz')
if self.system != 'Windows':
pginst.install_full()
pginst.initdb_start()
else:
pginst.install_perl_win()
pginst.install_postgres_win(port=55432)
try:
pginst.exec_psql('CREATE EXTENSION plpython3u')
except Exception:
subprocess.check_call('SETX PYTHONHOME C:\\Python27 -m',
shell=True)
if version != "9.6" or self.system == 'Windows' or \
(edition == '1c' and pginst.os_name not in DEBIAN_BASED):
buildinfo = os.path.join(pginst.get_pg_prefix(), 'doc',
'buildinfo.txt')
else:
buildinfo = subprocess.check_output(
'ls /usr/share/doc/postgres*pro*/buildinfo.txt',
shell=True).decode(ConsoleEncoding).strip()
with open(buildinfo, 'r') as bi:
bitxt = bi.read()
assert (re.search(r'^Documentation translation', bitxt,
re.MULTILINE))
assert (re.search(r'^Source', bitxt, re.MULTILINE))
assert (re.search(r'^SPEC', bitxt, re.MULTILINE))
print("The binary package buildinfo:\n%s\n" % bitxt)
pginst.install_default_config()
pginst.exec_psql("ALTER SYSTEM SET max_worker_processes = 16")
pginst.exec_psql("ALTER SYSTEM SET lc_messages = 'C'")
# Prepare pg_hba.conf for src/interfaces/ecpg/test/connect/test5
with open(os.path.join(pginst.get_configdir(), 'pg_hba.conf'),
'r+') as conf:
hba = re.sub(r'^(local\s+all\s+all\s+peer)$',
"local all regress_ecpg_user1 md5\n"
"local all regress_ecpg_user2 md5\n"
"local all regress_hacker trust\n"
"local all regress_superuser trust\n"
"local all nosuper trust\n"
r'\1',
conf.read(),
flags=re.MULTILINE)
conf.seek(0)
conf.write(hba)
pginst.load_shared_libraries(restart_service=False)
pginst.restart_service()
if self.system != 'Windows':
subprocess.check_call(
'"%s" "%s" "%s"' %
(os.path.join(curpath, 'make_installcheck.sh'),
get_pg_prefix(pginst), pginst.service_name),
shell=True)
pginst.make_check_passed = True
else:
# First run is performed to setup the environment
subprocess.check_call(
'"%s" "%s" "%s"' %
(os.path.join(curpath, 'make_installcheck.cmd'),
get_pg_prefix(pginst), pginst.service_name),
shell=True)
request.session.customexitstatus = 222
def test_install(self, request):
""" Install pg_probackup utility and configure postgresql
for running pg_probackup.
Scenario:
1. Install postgrespro version
2. Check version function
3. Check help function
"""
if self.system == 'Linux':
dist = " ".join(get_distro()[0:2])
elif self.system == 'Windows':
dist = 'Windows'
else:
raise Exception("OS %s is not supported." % self.system)
version = request.config.getoption('--product_version')
name = request.config.getoption('--product_name')
edition = request.config.getoption('--product_edition')
milestone = request.config.getoption('--product_milestone')
target = request.config.getoption('--target')
product_info = " ".join([dist, name, edition, version])
tag_mark = allure.label(LabelType.TAG, product_info)
request.node.add_marker(tag_mark)
branch = request.config.getoption('--branch')
request.cls.skip = False
if name != 'postgrespro' or edition == '1c' or version == '13':
# TODO: Enable this test for v13
print("PgProBackup test is only for postgrespro std and ent.")
request.cls.skip = True
return
# Step 1
self.pginst = PgInstall(product=name, edition=edition, version=version,
milestone=milestone, branch=branch,
windows=(self.system == 'Windows'))
self.pginst.setup_repo()
if self.pginst.os.is_altlinux()and self.pginst.os.os_arch == 'aarch64':
os.environ['LANG'] = 'en_US.UTF-8'
if self.system != 'Windows':
self.pginst.install_full()
self.pginst.initdb_start()
else:
self.pginst.install_postgres_win()
self.fix_permissions(tempdir)
self.bindir = self.pginst.get_bin_path() + os.sep
print('Bindir is %s' % self.bindir)
self.pginst.load_shared_libraries(restart_service=True)
request.cls.pginst = self.pginst
# Step 2
assert self.execute_pg_probackup("--help")
ver = self.execute_pg_probackup("--version")
print('Version is %s' % ver)
vere = re.search(r'[^(]+\s([0-9.]+)\s.*', ver)
self.version = vere.group(1)
assert self.version
request.cls.version = self.version
