def step_impl(context):
run_remotely(
host=ClusterUnderTest.INITIAL_MASTER,
timeout=2592000, # wait forever
command=
"sudo -u postgres /var/local/postgresql/full_backup_to_aws.sh >>/var/local/postgresql/log/backup.log 2>&1"
)
def detect_state(server):
try:
lsres, status = run_remotely(
host=server,
command="/bin/ls -1 /var/local/postgresql/data",
timeout=10)
files = [f.strip() for f in lsres.split("\n")]
if 'postgresql.conf.deactivated' in files:
return State.DEACTIVATED
elif 'recovery.conf' in files:
return State.CONFIGURED_SLAVE
elif 'postgresql.conf' in files:
return State.CONFIGURED_MASTER
elif status == 2:
lsres, status = run_remotely(
host=server,
command="/bin/ls -1 /var/local/postgresql",
timeout=10)
if status == 2:
return State.NOT_INITIALIZED
else:
return State.EMPTY_DATA_DIR
except Exception as ex:
print("ssh connect to '{}' failed!".format(server))
return State.NOT_REACHABLE
def step_impl(context, node):
start = time.time()
while time.time() - start < 120:
try:
run_remotely(host=host_for_node_name(node),
timeout=10,
command="uptime")
break
except Exception:
pass # ignore and retry
def server_details(name):
ip = detect_ip(servers[name])
state = detect_state(ip)
reboot = ""
version = ""
if state != State.NOT_REACHABLE:
lsres, status = run_remotely(host=ip,
command="/bin/ls -1 /run/reboot-required",
timeout=10)
if status == 0:
reboot = "REBOOT!"
lsres, status = run_remotely(
host=ip, command="/usr/local/bin/pg_config --version", timeout=10)
if status == 0:
version = lsres.strip().split(" ")[1]
res = "{:<15} {:17} {:7} {:5} {:<20} {}".format(ip,
servers[name].flavor['id'],
reboot, version,
str_state(state), name)
return res
if 'ENFORCE_MASTER' in os.environ:
enforce_master = os.environ['ENFORCE_MASTER']
if not enforce_master in servers.keys():
raise ValueError(
"Desired ENFORCE_MASTER '{}' is not found in the server list '{}'".
format(enforce_master, servers.keys()))
if master != enforce_master:
deactivate.append(master)
master = enforce_master
slave_upstream = enforce_master # override
if master in slaves:
slaves.remove(master)
for slave in slaves: # detect obsolete slaves with obsolete upstream
conninfo, status = run_remotely(
host=slave,
command=
"cat /var/local/postgresql/data/recovery.conf | grep primary_conninfo",
timeout=10)
match_upstream = re.search(r'host=(\S+)', conninfo)
if match_upstream and match_upstream.group(1) != enforce_master:
logging.info('match_upstream:' + match_upstream.group(1))
logging.info(' vs. ' + enforce_master)
deactivate.append(slave)
# clean up slave list
for x in deactivate:
if x in slaves:
slaves.remove(x)
if (master in deactivated) or (master in deactivate):
master = None
def step_impl(context, node):
run_remotely(host=host_for_node_name(node), command="reboot")
def wipe_out(context, node):
run_remotely(host=host_for_node_name(node), command=wipe_out_command)