def get_sys_users(postgres_peers,
patroni_config='/var/lib/pgsql/patroni.yaml'):
cluster_info = get_cluster_info(nodes=postgres_peers)
host = ''
if 'members' in cluster_info:
for m in cluster_info['members']:
if 'role' in m and (m['role'] == 'leader'
or m['role'] == 'sync_standby'):
host = m['host']
if m['state'] == 'running': break
p = [
x for x in postgres_peers if x.hostname == host or host in x.ip_aliases
]
for n in p:
try:
result = patt.exec_script(nodes=[n],
src="./dscripts/patroni_info.py",
args=['-i', 'sys_user'],
sudo=True,
log_call=False)
except:
continue
else:
for r in result:
try:
c = json.loads(json.dumps(r.out.strip()))
except:
continue
else:
return dict(ast.literal_eval(c))
return {}
def nftables_configure(cluster_name,
template_src,
config_file_target,
patroni_peers=[],
etcd_peers=[],
haproxy_peers=[],
postgres_clients=[]):
nodes = list(
{n.hostname: n
for n in patroni_peers + etcd_peers + haproxy_peers}.values())
logger.debug("nftables_configure {}".format([n.hostname for n in nodes]))
nft_init(nodes)
result = patt.exec_script(
nodes=nodes,
src="./dscripts/nft_config.py",
payload=template_src,
args=['-t'] + [os.path.basename(template_src)] + ['-d'] +
[config_file_target] + ['-p'] + [p.hostname for p in patroni_peers] +
list([" ".join(p.ip_aliases) for p in patroni_peers]) + ['-e'] +
[e.hostname for e in etcd_peers] +
list([" ".join(e.ip_aliases) for e in etcd_peers]) + ['-x'] +
[x.hostname for x in haproxy_peers] +
list([" ".join(x.ip_aliases) for x in haproxy_peers]) + ['-c'] +
[c for c in postgres_clients],
sudo=True)
log_results(result)
nftables_enable(nodes)
def walg_ssh_archiving_init(nodes):
logger.info ("processing {}".format ([n.hostname for n in nodes]))
patt.host_id(nodes)
# patt.check_dup_id (nodes)
result = patt.exec_script (nodes=nodes, src="./dscripts/d27.walg.sh",
args=['ssh_archiving_init'], sudo=True)
log_results (result)
def disk_init(nodes, vol_size, mnt=None, user=None):
nodes = list({n.hostname: n for n in nodes}.values())
logger.debug("disk init {}".format(nodes))
patt.host_id(nodes)
patt.check_dup_id(nodes)
util_init(nodes)
if mnt:
result = patt.exec_script(nodes=nodes,
src="./dscripts/data_vol.py",
args=['-m'] + [mnt] + ['-s'] + [vol_size],
sudo=True)
elif user:
result = patt.exec_script(nodes=nodes,
src="./dscripts/data_vol.py",
args=['-u'] + [user] + ['-s'] + [vol_size],
sudo=True)
log_results(result)
def cluster_health(nodes):
result = patt.exec_script (nodes=nodes, src="./dscripts/d10.etcd.sh",
args=['cluster_health'], sudo=True)
for r in result:
logger.debug ("cluster_health: _{}_".format(r.out.strip()))
if r.out.strip() == "cluster is healthy":
return True
return False
def haproxy_enable(nodes):
patt.host_id(nodes)
patt.check_dup_id(nodes)
result = patt.exec_script(nodes=nodes,
src="./dscripts/d40.haproxy.sh",
args=['enable'],
sudo=True)
log_results(result)
def walg_init(walg_version, nodes):
logger.info ("processing {}".format ([n.hostname for n in nodes]))
patt.host_id(nodes)
patt.check_dup_id (nodes)
result = patt.exec_script (nodes=nodes, src="./dscripts/d27.walg.sh",
args=['init'] + [walg_version], sudo=False)
log_results (result)
return all(x == True for x in [bool(n.out) for n in result])
def floating_ip_enable(nodes, floating_ips):
patt.host_id(nodes)
patt.check_dup_id(nodes)
result = patt.exec_script(nodes=nodes,
src="./dscripts/d25.floating_ip.sh",
args=['enable', " ".join(floating_ips)],
sudo=True)
log_results(result)
def walg_archiving_add(cluster_name, nodes):
patt.host_id(nodes)
# patt.check_dup_id (nodes)
result = patt.exec_script (nodes=nodes, src="./dscripts/d27.walg.sh",
args=['ssh_archiving_add'] + [cluster_name], sudo=True)
log_results (result)
return all(x == True for x in [bool(n.out == "drwx--x--x {}.{} {}".format (
cluster_name, "walg", "/var/lib/walg/" + cluster_name)) for n in result])
def postgres_ssl_cert_init(nodes):
logger.info("processing {}".format([n.hostname for n in nodes]))
patt.host_id(nodes)
patt.check_dup_id(nodes)
result = patt.exec_script(nodes=nodes,
src="dscripts/ssl_cert_postgres.sh",
args=['init'],
sudo=True)
log_results(result)
def postgres_init(postgres_version, nodes):
logger.info("processing {}".format([n.hostname for n in nodes]))
patt.host_id(nodes)
patt.check_dup_id(nodes)
result = patt.exec_script(nodes=nodes,
src="./dscripts/d20.postgres.sh",
args=['init'] + [postgres_version],
sudo=True)
log_results(result)
def floating_ip_init(nodes, ip_takeover_version="0.9"):
patt.host_id(nodes)
patt.check_dup_id(nodes)
result = patt.exec_script(nodes=nodes,
src="./dscripts/d25.floating_ip.sh",
args=['init'] + [ip_takeover_version],
sudo=True,
timeout=1440)
log_results(result)
def walg_ssh_known_hosts(cluster_name, nodes, archiving_server):
logger.info ("processing {}".format ([n.hostname for n in nodes]))
patt.host_id(nodes)
patt.check_dup_id (nodes)
result = patt.exec_script (nodes=nodes, src="./dscripts/d27.walg.sh",
args=['ssh_known_hosts'] + [cluster_name] +
[archiving_server[0].hostname],
sudo=True)
log_results (result)
return not any(x == True for x in [bool(n.error) for n in result if hasattr(n,'error')])
def walg_ssh_gen(cluster_name, nodes, postgres_user='******'):
logger.info ("processing {}".format ([n.hostname for n in nodes]))
patt.host_id(nodes)
patt.check_dup_id (nodes)
result = patt.exec_script (nodes=nodes, src="./dscripts/d27.walg.sh",
args=['ssh_archive_keygen'] + [cluster_name] + [postgres_user],
sudo=True, log_call=True)
log_results (result, hide_stdout=True)
assert all(x == True for x in [bool(n.out) for n in result])
return [n.out for n in result]
def nft_init(nodes):
nodes = list({n.hostname: n for n in nodes}.values())
logger.debug("nft_init {}".format([n.hostname for n in nodes]))
patt.host_id(nodes)
patt.check_dup_id(nodes)
result = patt.exec_script(nodes=nodes,
src="./dscripts/d01.nft.sh",
args=['init'],
sudo=True)
log_results(result)
def patroni_enable(postgres_version, patroni_version, nodes):
patt.host_id(nodes)
patt.check_dup_id(nodes)
result = patt.exec_script(nodes=nodes,
src="./dscripts/d30.patroni.sh",
args=['enable'] + [postgres_version] +
[patroni_version],
sudo=True)
log_results(result)
random_node = [random.choice(nodes)]
result = patt.exec_script(nodes=random_node,
src="./dscripts/d30.patroni.sh",
args=['check'],
sudo=True)
for r in result:
logger.warn("hostname: {}".format(r.hostname))
return ("\n{}".format(r.out))
logger.warn("error: {}".format(r.error))
def floating_ip_build(nodes, ip_takeover_version="0.9"):
patt.host_id(nodes)
patt.check_dup_id(nodes)
result = patt.exec_script(
nodes=nodes,
src="./dscripts/d25.floating_ip.sh",
payload=["./ip_takeover.py", "./ip_takeover.make"],
args=['build'] + [ip_takeover_version],
sudo=False)
log_results(result)
def add_repo(repo_url, nodes):
nodes = list({n.hostname: n for n in nodes}.values())
logger.debug("add repo url {}".format(nodes))
patt.host_id(nodes)
patt.check_dup_id(nodes)
result = patt.exec_script(nodes=nodes,
src="./dscripts/d03.repo.sh",
args=['add'] + [" ".join(repo_url)],
sudo=True)
log_results(result)
def patroni_init(postgres_version, patroni_version, nodes):
patt.host_id(nodes)
patt.check_dup_id(nodes)
result = patt.exec_script(nodes=nodes,
src="./dscripts/d30.patroni.sh",
payload='config/patroni.te',
args=['init'] + [postgres_version] +
[patroni_version] + ['patroni.te'],
sudo=True)
log_results(result)
def tuned_postgresql(nodes):
nodes = list({n.hostname: n for n in nodes}.values())
logger.debug("tuned postgresql {}".format(nodes))
patt.host_id(nodes)
patt.check_dup_id(nodes)
result = patt.exec_script(nodes=nodes,
src="./dscripts/d22_tuned.sh",
args=['enable'],
sudo=True)
log_results(result)
def postgres_wait_ready(postgres_peers, postgres_version, timeout=120):
logger.info("processing {}".format([n.hostname for n in postgres_peers]))
patt.host_id(postgres_peers)
patt.check_dup_id(postgres_peers)
result = patt.exec_script(nodes=postgres_peers,
src="./dscripts/pg_wait_ready.sh",
args=['wait_pg_isready'] + [postgres_version] +
[timeout],
sudo=True)
log_results(result)
return not all(x == False for x in [bool(n.out) for n in result])
def walg_authorize_keys(cluster_name, nodes, keys=[]):
patt.host_id(nodes)
# patt.check_dup_id (nodes)
with tempfile.NamedTemporaryFile(mode='w+', encoding='ascii') as tmpl_file:
for k in keys + [""]:
print("{}".format (k), file=tmpl_file)
tmpl_file.flush()
result = patt.exec_script (nodes=nodes, src="./dscripts/d27.walg.sh",
payload=tmpl_file.name,
args=['ssh_authorize_keys'] + [cluster_name] +
[os.path.basename (tmpl_file.name)], sudo=True)
log_results (result)
return not any (x == True for x in [bool(n.error) for n in result if hasattr(n,'error')])
def haproxy_configure(cluster_name, template_src, nodes, postgres_nodes,
config_file_target):
haproxy_init(nodes)
result = patt.exec_script(nodes=nodes,
src="./dscripts/haproxy_config.py",
payload=template_src,
args=['-c'] + [cluster_name] + ['-t'] +
[os.path.basename(template_src)] + ['-d'] +
[config_file_target] + ['-p'] +
[p.hostname for p in postgres_nodes] + ['-x'] +
[x.hostname for x in nodes],
sudo=True)
log_results(result)
haproxy_enable(nodes)
def etcd_sort_by_version (nodes):
for i in range(10):
resp = patt.exec_script (nodes=nodes, src="./dscripts/d10.etcd.sh", args=['version'], sudo=True)
log_results (resp)
if all(x == False for x in [bool(n.error) for n in resp]): break
time.sleep(3.0)
for r in resp:
tmp = (r.hostname, r.out.strip())
for idx, item in enumerate(nodes):
if tmp[0] == item.hostname:
nodes[idx].etcd_version = tmp[1]
result = sorted(nodes, key=lambda etcd_peer: etcd_peer.etcd_version)
logger.debug ("etcd_sort_by_version: {}".format(
[str(n.hostname) + ' ' + str(n.id) + ' ' + str(n.etcd_version) for n in result]))
return result
def postgres_exec(postgres_peers, script_file):
script = None
nodes = postgres_peers
if os.path.isfile(script_file):
script = script_file
script_arg = os.path.basename(script_file)
else:
script_arg = script_file
result = patt.exec_script(nodes=nodes,
src="./dscripts/postgres_exec.sh",
payload=script,
args=[script_arg],
sudo=True,
log_call=True)
log_results(result)
def get_cluster_info(nodes):
for n in nodes:
try:
result = patt.exec_script(nodes=[n],
src="./dscripts/patroni_info.py",
args=['-i', 'cluster'],
sudo=False)
except:
continue
else:
for r in result:
try:
c = json.loads(json.dumps(r.out.strip()))
except:
continue
else:
return dict(ast.literal_eval(c))
return {}
def patroni_configure(postgres_version,
cluster_name,
template_src,
nodes,
etcd_peers,
config_file_target,
sysuser_pass,
postgres_parameters,
pg_hba_list=cert_pg_hba_list(),
user=None):
tmpl = ""
with tempfile.NamedTemporaryFile(mode='w+', encoding='utf-8') as tmpl_file:
if os.path.isfile(template_src):
with open(template_src, 'r') as p:
try:
tmpl = yaml.safe_load(p)
except yaml.YAMLError as e:
print(str(e), file=sys.stderr)
raise
except:
raise
if postgres_parameters:
for p in postgres_parameters:
key, val = p.split('=')
tmpl['postgresql']['parameters'][key.strip()] = val.strip()
if pg_hba_list:
tmpl['postgresql']['pg_hba'] = pg_hba_list
print(yaml.dump(tmpl, default_flow_style=False), file=tmpl_file)
tmpl_file.flush()
result = patt.exec_script(
nodes=nodes,
src="./dscripts/patroni_config.py",
payload=tmpl_file.name,
args=['-c'] + [cluster_name] + ['-t'] +
[os.path.basename(tmpl_file.name)] + ['-d'] +
[config_file_target] + ['-u'] + [user] + ['-v'] +
[postgres_version] + ['-p'] + [n.hostname
for n in nodes] + ['-e'] +
[n.hostname
for n in etcd_peers] + ['-s'] + ['"' + str(sysuser_pass) + '"'],
sudo=True,
log_call=False)
log_results(result)
def postgres_gc_cron(nodes, vaccum_full_df_percent, target, postgres_version):
logger.info("processing {}".format([n.hostname for n in nodes]))
patt.host_id(nodes)
patt.check_dup_id(nodes)
tmpl = "./config/postgres-gc.sh.tmpl"
vacuumdb_option = ""
if postgres_version >= 12:
vacuumdb_option = "--skip-locked"
result = patt.exec_script(
nodes=nodes,
src="./dscripts/tmpl2file.py",
payload=tmpl,
args=['-t'] + [os.path.basename(tmpl)] + ['-o'] + [target] +
['--chmod'] + ['755'] + ['--dictionary_key_val'] +
["pc={}".format(vaccum_full_df_percent)] + ['--dictionary_key_val'] +
["vacuumdb_option={}".format(vacuumdb_option)] +
['--dictionary_key_val'] +
["postgres_version={}".format(postgres_version)],
sudo=True)
log_results(result)
def postgres_get_cert(q, postgres_user='******', nodes=[]):
if q == 'root.crt':
e = '--get_ca_crt'
elif q == 'root.key':
e = '--get_ca_key'
else:
raise ValueError("unknow query {}".format(q))
for n in nodes:
try:
result = patt.exec_script(nodes=[n],
src="dscripts/ssl_cert_postgres.py",
args=['-u', postgres_user, e],
sudo=True)
except:
continue
else:
for r in result:
if r.out:
return r.out
finally:
log_results(result, hide_stdout=True)
def get_members(nodes, cluster_name, state='ok'):
members = []
if state == 'ok':
cmd = 'check_healthy'
else:
cmd = 'check_unhealthy'
result = patt.exec_script (nodes=nodes, src="./dscripts/d10.etcd.sh",
args=[cmd] + [cluster_name], sudo=True)
for r in result:
try:
line = r.out.rsplit()
line = [l.rsplit('[')[1] for l in line]
line = [l.rsplit(']')[0] for l in line]
except:
continue
if not line:
continue
for l in line:
if l not in members:
members.append(l)
return members
