def step_impl(context): run_remotely( host=ClusterUnderTest.INITIAL_MASTER, timeout=2592000, # wait forever command= "sudo -u postgres /var/local/postgresql/full_backup_to_aws.sh >>/var/local/postgresql/log/backup.log 2>&1" )
def detect_state(server): try: lsres, status = run_remotely( host=server, command="/bin/ls -1 /var/local/postgresql/data", timeout=10) files = [f.strip() for f in lsres.split("\n")] if 'postgresql.conf.deactivated' in files: return State.DEACTIVATED elif 'recovery.conf' in files: return State.CONFIGURED_SLAVE elif 'postgresql.conf' in files: return State.CONFIGURED_MASTER elif status == 2: lsres, status = run_remotely( host=server, command="/bin/ls -1 /var/local/postgresql", timeout=10) if status == 2: return State.NOT_INITIALIZED else: return State.EMPTY_DATA_DIR except Exception as ex: print("ssh connect to '{}' failed!".format(server)) return State.NOT_REACHABLE
def step_impl(context, node): start = time.time() while time.time() - start < 120: try: run_remotely(host=host_for_node_name(node), timeout=10, command="uptime") break except Exception: pass # ignore and retry
def server_details(name): ip = detect_ip(servers[name]) state = detect_state(ip) reboot = "" version = "" if state != State.NOT_REACHABLE: lsres, status = run_remotely(host=ip, command="/bin/ls -1 /run/reboot-required", timeout=10) if status == 0: reboot = "REBOOT!" lsres, status = run_remotely( host=ip, command="/usr/local/bin/pg_config --version", timeout=10) if status == 0: version = lsres.strip().split(" ")[1] res = "{:<15} {:17} {:7} {:5} {:<20} {}".format(ip, servers[name].flavor['id'], reboot, version, str_state(state), name) return res
if 'ENFORCE_MASTER' in os.environ: enforce_master = os.environ['ENFORCE_MASTER'] if not enforce_master in servers.keys(): raise ValueError( "Desired ENFORCE_MASTER '{}' is not found in the server list '{}'". format(enforce_master, servers.keys())) if master != enforce_master: deactivate.append(master) master = enforce_master slave_upstream = enforce_master # override if master in slaves: slaves.remove(master) for slave in slaves: # detect obsolete slaves with obsolete upstream conninfo, status = run_remotely( host=slave, command= "cat /var/local/postgresql/data/recovery.conf | grep primary_conninfo", timeout=10) match_upstream = re.search(r'host=(\S+)', conninfo) if match_upstream and match_upstream.group(1) != enforce_master: logging.info('match_upstream:' + match_upstream.group(1)) logging.info(' vs. ' + enforce_master) deactivate.append(slave) # clean up slave list for x in deactivate: if x in slaves: slaves.remove(x) if (master in deactivated) or (master in deactivate): master = None
def step_impl(context, node): run_remotely(host=host_for_node_name(node), command="reboot")
def wipe_out(context, node): run_remotely(host=host_for_node_name(node), command=wipe_out_command)