def backp_postgresqlconf(self, node, data_dir):
"""Copy old postgresql.conf file to postgresql.conf.bckp and
edit with new config and edit postggresql.conf
:param node: node ip address
:param conf_options: list with postgresql.conf options
:return:
"""
postgresql_conf_file = '/'.join([data_dir, 'postgresql.conf'])
cmd = "cp -n %s %s/postgresql.conf_bckp" % (postgresql_conf_file,
data_dir)
command_executor(cmd,
remote=True,
host=node,
login='******',
password='******')
def exec_cmd_retry(self, cmd, retry_cnt=5, stdout=False):
timeout = 0
attempt = 1
while attempt < retry_cnt:
try:
return command_executor(cmd, self.remote, self.host,
REMOTE_ROOT, REMOTE_ROOT_PASSWORD,
stdout)
except Exception as ex:
print('Exception occured while running command "%s":' % cmd)
print(ex)
print('========')
attempt += 1
timeout += 5
print('Retrying (attempt %d with delay for %d seconds)...' %
(attempt, timeout))
time.sleep(timeout)
if retry_cnt > 1:
print('Last attempt to execute the command...\n')
return command_executor(cmd, self.remote, self.host, REMOTE_ROOT,
REMOTE_ROOT_PASSWORD, stdout)
def remove_package(self, pkg_name, purge=False):
"""
:param pkg_name
:return:
"""
if self.is_pm_yum():
# todo fix this
cmd = "yum remove -y%s %s" % \
(' --noautoremove' if self.os_version.startswith(
'8') else '', pkg_name)
command_executor(cmd, self.remote, self.host, REMOTE_ROOT,
REMOTE_ROOT_PASSWORD)
elif self.is_pm_apt():
cmd = "apt-get %s -y %s" % (
'purge' if purge and self.is_debian_based() else 'remove',
pkg_name.replace('*', '.*'))
command_executor(cmd, self.remote, self.host, REMOTE_ROOT,
REMOTE_ROOT_PASSWORD)
elif self.is_pm_zypper():
cmd = "rpm -qa %s" % pkg_name
installed_pkgs = subprocess.check_output(cmd, shell=True). \
decode(ConsoleEncoding).splitlines()
if (len(installed_pkgs)):
pkg_name = ' '.join(installed_pkgs)
cmd = "zypper remove -y %s" % pkg_name
command_executor(cmd, self.remote, self.host, REMOTE_ROOT,
REMOTE_ROOT_PASSWORD)
elif self.is_windows():
raise Exception("Not implemented on Windows.")
else:
raise Exception("Unsupported system: %s." % self.os_name)
def get_files_in_package(self, pkgname):
if self.is_debian_based():
cmd = "sh -c \"LANG=C dpkg --listfiles %s\"" % pkgname
else:
cmd = "sh -c \"LANG=C rpm -q --list %s 2>&1\"" % pkgname
result = command_executor(cmd,
self.remote,
self.host,
REMOTE_ROOT,
REMOTE_ROOT_PASSWORD,
stdout=True).strip().split('\n')
if result and result[0] == '(contains no files)':
return []
return result
def get_package_deps(self, pkgname):
"""
:param pkgname
:return:
"""
if self.is_debian_based():
cmd = "sh -c \"LANG=C dpkg -s %s\"" % pkgname
else:
cmd = "sh -c \"LANG=C rpm -q --requires %s\"" % pkgname
result = command_executor(cmd,
self.remote,
self.host,
REMOTE_ROOT,
REMOTE_ROOT_PASSWORD,
stdout=True).strip()
if self.is_debian_based():
for line in result.split('\n'):
depprefix = 'Depends: '
if line.startswith(depprefix):
return line[len(depprefix):]
else:
return ', '.join(result.split('\n'))
def get_package_version(self, package_name):
cmd = ''
if self.is_pm_yum():
cmd = "sh -c \"LANG=C yum info %s\"" % package_name
elif self.is_pm_apt():
cmd = "sh -c \"LANG=C apt-cache show %s\"" % package_name
elif self.is_pm_zypper():
cmd = "sh -c \"LANG=C zypper info %s\"" % package_name
if cmd:
out = command_executor(cmd,
self.remote,
self.host,
REMOTE_ROOT,
REMOTE_ROOT_PASSWORD,
stdout=True).split('\n')
for line in out:
# Find "Version : 9.6.15.2-alt1" or "Version: 2:9.6.15.2-alt1"
# or "Version: 10.10.2"
# or "Version: 13beta1"
vere = re.search(r'Version\s*:.*[:\s]([0-9.a-z]+)', line)
if (vere):
return vere.group(1)
return None
def test_create_multimaster_cluster(self, install_postgres):
"""Create cluster with simple configuration and 3 nodes
Scenario:
1. Edit replication settings
2. Add multimaster settings
3. Create extension multimaster
4. Restart all nodes in cluster
5. Check that cluster in online state
6. Check that nodes in online state
"""
# Step 1
multimaster_conn_string = ""
for node in install_postgres:
multimaster_conn_string += \
"dbname=postgres user=postgres host=%s, " % node.node_ip
multimaster_conn_string = multimaster_conn_string.rstrip(", ")
multimaster_conn_string = "'%s'" % multimaster_conn_string
node_id = 1
for node in install_postgres:
node.connstring = "host=%s user=postgres" % node.node_ip
self.backp_postgresqlconf(node.node_ip,
node.get_option('data_directory'))
postgresql_conf_file = '/'.join(
[node.get_option('data_directory'), 'postgresql.conf'])
cmd = "echo \'shared_preload_libraries = \'multimaster\'\' " \
" >> %s " % postgresql_conf_file
command_executor(cmd,
remote=True,
host=node.node_ip,
login='******',
password='******')
cmd = "echo \'multimaster.conn_strings = \'\\\'%s\\\'\'\' " \
">> %s" % (multimaster_conn_string, postgresql_conf_file)
command_executor(cmd,
remote=True,
host=node.node_ip,
login='******',
password='******')
cmd = "echo \'wal_level = logical\' >> " + postgresql_conf_file
command_executor(cmd,
remote=True,
host=node.node_ip,
login='******',
password='******')
cmd = "echo \'max_prepared_transactions = 300\' >> " + \
postgresql_conf_file
command_executor(cmd,
remote=True,
host=node.node_ip,
login='******',
password='******')
cmd = "echo \'max_wal_senders = 10\' >> " + postgresql_conf_file
command_executor(cmd,
remote=True,
host=node.node_ip,
login='******',
password='******')
cmd = "echo \'max_replication_slots = 10\' >> " + \
postgresql_conf_file
command_executor(cmd,
remote=True,
host=node.node_ip,
login='******',
password='******')
cmd = "echo \'max_worker_processes = 250\' >> " + \
postgresql_conf_file
command_executor(cmd,
remote=True,
host=node.node_ip,
login='******',
password='******')
cmd = "echo \'multimaster.max_nodes = 3\' >> " + \
postgresql_conf_file
command_executor(cmd,
remote=True,
host=node.node_ip,
login='******',
password='******')
cmd = "echo \'multimaster.node_id = %s\' >> %s" % (
str(node_id), postgresql_conf_file)
command_executor(cmd,
remote=True,
host=node.node_ip,
login='******',
password='******')
node_id += 1
cmd = "echo \'log_min_messages = log\' >> " + postgresql_conf_file
command_executor(cmd,
remote=True,
host=node.node_ip,
login='******',
password='******')
cmd = "echo \'log_min_error_statement = log\' >> " + \
postgresql_conf_file
command_executor(cmd,
remote=True,
host=node.node_ip,
login='******',
password='******')
node.manage_psql("restart", remote=True, host=node.node_ip)
time.sleep(30)
node = install_postgres[0]
node.connstring = "host=%s user=postgres" % node.node_ip
conn = psycopg2.connect(node.connstring)
cursor = conn.cursor()
cursor.execute("CREATE EXTENSION multimaster")
cursor.close()
conn.commit()
conn.close()
for node in install_postgres:
node.connstring = "host=%s user=postgres" % node.node_ip
conn = psycopg2.connect(node.connstring)
cursor = conn.cursor()
cursor.execute("SELECT connected FROM mtm.get_nodes_state()")
state = cursor.fetchall()[0][0]
assert state is True
cursor.execute("SELECT status FROM mtm.get_cluster_state()")
status = cursor.fetchall()[0][0]
cursor.close()
conn.close()
assert status == 'Online'
def test_upload_data_to_cluster(self, create_environment,
install_postgres):
"""Scenario
"""
# Step 1
cluster_name = create_environment.keys()[0]
multimaster_conn_string = ""
for node in install_postgres:
multimaster_conn_string += \
"dbname=postgres user=postgres host=%s, " % node.node_ip
multimaster_conn_string = multimaster_conn_string.rstrip(", ")
multimaster_conn_string = "'%s'" % multimaster_conn_string
node_id = 1
for node in install_postgres:
install_postgres.connstring = "host=%s user=postgres" % \
node.node_ip
self.backp_postgresqlconf(
node['ip'], install_postgres.get_option('data_directory'))
postgresql_conf_file = '/'.join([
install_postgres.get_option('data_directory'),
'postgresql.conf'
])
cmd = "echo \'shared_preload_libraries = " \
"\'multimaster\'\' >> %s " % postgresql_conf_file
command_executor(cmd,
remote=True,
host=node.node_ip,
login='******',
password='******')
cmd = "echo \'multimaster.conn_strings = " \
"\'\\\'%s\\\'\'\' >> %s" % (
multimaster_conn_string, postgresql_conf_file)
command_executor(cmd,
remote=True,
host=node.node_ip,
login='******',
password='******')
cmd = "echo \'wal_level = logical\' >> %s " % postgresql_conf_file
command_executor(cmd,
remote=True,
host=node.node_ip,
login='******',
password='******')
cmd = "echo \'max_prepared_transactions = 300\' >> " + \
postgresql_conf_file
command_executor(cmd,
remote=True,
host=node.node_ip,
login='******',
password='******')
cmd = "echo \'max_wal_senders = 10\' >> " + postgresql_conf_file
command_executor(cmd,
remote=True,
host=node['ip'],
login='******',
password='******')
cmd = "echo \'max_replication_slots = 10\' >> " + \
postgresql_conf_file
command_executor(cmd,
remote=True,
host=node.node_ip,
login='******',
password='******')
cmd = "echo \'max_worker_processes = 250\' >> " + \
postgresql_conf_file
command_executor(cmd,
remote=True,
host=node.node_ip,
login='******',
password='******')
cmd = "echo \'multimaster.max_nodes = 3\' >> " + \
postgresql_conf_file
command_executor(cmd,
remote=True,
host=node.node_ip,
login='******',
password='******')
cmd = "echo \'multimaster.node_id = %s\' >> %s " % (
str(node_id), postgresql_conf_file)
command_executor(cmd,
remote=True,
host=node.node_ip,
login='******',
password='******')
node_id += 1
cmd = "echo \'log_min_messages = log\' >> " + \
postgresql_conf_file
command_executor(cmd,
remote=True,
host=node.node_ip,
login='******',
password='******')
cmd = "echo \'log_min_error_statement = log\' >> " + \
postgresql_conf_file
command_executor(cmd,
remote=True,
host=node.node_ip,
login='******',
password='******')
node.manage_psql("restart", remote=True, host=node.node_ip)
time.sleep(30)
node = install_postgres[0]
node.connstring = "host=%s user=postgres" % node.node_ip
conn = psycopg2.connect(node.connstring)
cursor = conn.cursor()
cursor.execute("CREATE EXTENSION multimaster")
cursor.close()
conn.commit()
conn.close()
def get_packages_in_repo(self, reponame, version):
result = []
if self.is_windows():
return result
if self.is_pm_yum():
cmd = "script -q -c \"stty cols 150; " \
"LANG=C yum -q --disablerepo='*' " \
"--enablerepo='%s' list available\"" % reponame
ysout = command_executor(cmd,
self.remote,
self.host,
REMOTE_ROOT,
REMOTE_ROOT_PASSWORD,
stdout=True).split('\n')
for line in ysout:
line = line.strip()
if line == 'Available Packages' or line == '':
continue
pkginfo = line.split()
if len(pkginfo) != 3:
print("Invalid line in yum list output:", line)
raise Exception('Invalid line in yum list output')
pkgname = re.sub(r'\.(x86_64|noarch)$', '', pkginfo[0])
result.append(pkgname)
if version == '9.6':
if 'pgbadger' in result and 'pgpro-pgbadger' in result:
result.remove('pgbadger')
elif self.is_altlinux():
# Parsing binary package info in lists/ is unfeasible,
# so the only way to find packages from the repository is
# to parse `apt-cache showpkg` output
# Use only relevant lists in separate directory to optimize parsing
cmd = "sh -c \"rm -rf /tmp/t_r 2>/dev/null; mkdir /tmp/t_r;" \
"cp /var/lib/apt/lists/%s* /tmp/t_r/;" \
"echo 'Dir::State::Lists \\\"/tmp/t_r\\\";'>/tmp/t_apt.conf;" \
"APT_CONFIG=/tmp/t_apt.conf " \
"apt-cache --names-only search .\"" % reponame
acout = command_executor(cmd,
self.remote,
self.host,
REMOTE_ROOT,
REMOTE_ROOT_PASSWORD,
stdout=True).split('\n')
ipkgs = []
for line in acout:
if line.strip() != '':
ipkgs.append(line[:line.index(' - ')])
cmd = "sh -c \"APT_CONFIG=/tmp/t_apt.conf " \
"apt-cache showpkg %s \"" % (" ".join(ipkgs))
acout = command_executor(cmd,
self.remote,
self.host,
REMOTE_ROOT,
REMOTE_ROOT_PASSWORD,
stdout=True).split('\n')
state = 0
pkgname = None
for line in acout:
if line.strip() == '':
state = 0
pkgname = None
continue
if state == 0:
if line.startswith('Package:'):
pkgname = re.sub(r'^Package:\s+', '', line)
state = 1
elif state == 1:
if line.strip() == 'Versions:':
state = 2
elif state == 2:
if ('(/tmp/t_r/' + reponame) in line:
result.append(pkgname)
elif self.is_pm_apt():
cmd = "sh -c \"grep -h -e 'Package:\\s\\+' " \
"/var/lib/apt/lists/%s*\"" % reponame
gsout = command_executor(cmd,
self.remote,
self.host,
REMOTE_ROOT,
REMOTE_ROOT_PASSWORD,
stdout=True).split('\n')
for line in gsout:
if line == '':
continue
pkgname = line.replace('Package: ', '')
if pkgname == 'Auto-Built-debug-symbols':
continue
if pkgname not in result:
result.append(pkgname)
if version == '9.6':
# PGPRO-2286
exclude = []
for pkgname in result:
if pkgname.startswith('lib'):
if ('postgrespro-' + pkgname) in result:
exclude.append(pkgname)
for pkgname in exclude:
result.remove(pkgname)
elif self.is_pm_zypper():
cmd = "sh -c \"LANG=C zypper search --repo %s\"" % reponame
zsout = command_executor(cmd,
self.remote,
self.host,
REMOTE_ROOT,
REMOTE_ROOT_PASSWORD,
stdout=True).split('\n')
for line in zsout:
pkginfo = line.split('|')
if len(pkginfo) != 4:
continue
pkgname = pkginfo[1].strip()
if (pkgname == 'Name'):
continue
result.append(pkgname)
return result
def get_all_installed_packages(self):
result = []
if self.is_windows():
return result
if self.is_pm_yum():
cmd = "script -q -c \"stty cols 150; " \
"LANG=C yum -q list installed\""
ysout = command_executor(cmd,
self.remote,
self.host,
REMOTE_ROOT,
REMOTE_ROOT_PASSWORD,
stdout=True).split('\n')
for line in ysout:
line = line.strip()
if line == 'Available Packages' or line == '' or \
line == 'Installed Packages':
continue
pkginfo = line.split()
if len(pkginfo) != 3:
print("Invalid line in yum list output:", line)
raise Exception('Invalid line in yum list output')
pkgname = re.sub(r'\.(x86_64|noarch)$', '', pkginfo[0])
result.append(pkgname)
elif self.is_altlinux():
cmd = "rpm -qa --qf '%{name}\\n'"
acout = command_executor(cmd,
self.remote,
self.host,
REMOTE_ROOT,
REMOTE_ROOT_PASSWORD,
stdout=True).split('\n')
result = acout[:]
elif self.is_pm_apt():
result = []
cmd = "apt list --installed"
gsout = command_executor(cmd,
self.remote,
self.host,
REMOTE_ROOT,
REMOTE_ROOT_PASSWORD,
stdout=True).split('\n')
passthrough = True
for line in gsout:
if line == '':
continue
if passthrough:
passthrough = line != 'Listing...'
continue
result.append(line.split('/')[0])
elif self.is_pm_zypper():
cmd = "sh -c \"LANG=C zypper packages --installed-only\""
zsout = command_executor(cmd,
self.remote,
self.host,
REMOTE_ROOT,
REMOTE_ROOT_PASSWORD,
stdout=True).split('\n')
for line in zsout:
pkginfo = line.split('|')
if len(pkginfo) != 5:
continue
pkgname = pkginfo[2].strip()
if (pkgname == 'Name'):
continue
result.append(pkgname)
return result