def __init__(self, config):
self.nap_time = config['loop_wait']
self.postgresql = Postgresql(config['postgresql'])
self.ha = Ha(self.postgresql, self.get_dcs(self.postgresql.name, config))
host, port = config['restapi']['listen'].split(':')
self.api = RestApiServer(self, config['restapi'])
self.skydns2 = config.get('skydns2')
self.next_run = time.time()
self.shutdown_member_ttl = 300
def run(config):
etcd = Etcd(config["etcd"])
postgresql = Postgresql(config["postgresql"])
try:
from BaseHTTPServer import HTTPServer
host, port = config["haproxy_status"]["listen"].split(":")
server = HTTPServer((host, int(port)), handler(postgresql, etcd))
logging.info('listening on %s:%s', host, port)
server.serve_forever()
except KeyboardInterrupt:
print('^C received, shutting down server')
server.socket.close()
def set_up(self):
subprocess.call = subprocess_call
shutil.copy = nop
self.p = Postgresql({'name': 'test0', 'scope': 'batman', 'data_dir': 'data/test0',
'listen': '127.0.0.1, *:5432', 'connect_address': '127.0.0.2:5432',
'pg_hba': ['hostssl all all 0.0.0.0/0 md5', 'host all all 0.0.0.0/0 md5'],
'superuser': {'password': ''},
'admin': {'username': '******', 'password': '******'},
'replication': {'username': '******',
'password': '******',
'network': '127.0.0.1/32'},
'parameters': {'foo': 'bar'}, 'recovery_conf': {'foo': 'bar'},
'callbacks': {'on_start': 'true', 'on_stop': 'true',
'on_restart': 'true', 'on_role_change': 'true',
'on_reload': 'true'
},
'restore': 'true'})
psycopg2.connect = psycopg2_connect
if not os.path.exists(self.p.data_dir):
os.makedirs(self.p.data_dir)
self.leader = Member(0, 'leader', 'postgres://*****:*****@127.0.0.1:5435/postgres', None, None, 28)
self.other = Member(0, 'test1', 'postgres://*****:*****@127.0.0.1:5433/postgres', None, None, 28)
self.me = Member(0, 'test0', 'postgres://*****:*****@127.0.0.1:5434/postgres', None, None, 28)
#!/usr/bin/env python
from BaseHTTPServer import BaseHTTPRequestHandler
from helpers.etcd import Etcd
from helpers.postgresql import Postgresql
import sys, yaml, socket
f = open(sys.argv[1], "r")
config = yaml.load(f.read())
f.close()
etcd = Etcd(config["etcd"])
postgresql = Postgresql(config["postgresql"])
class StatusHandler(BaseHTTPRequestHandler):
def do_GET(self):
return self.do_ANY()
def do_OPTIONS(self):
return self.do_ANY()
def do_ANY(self):
if postgresql.name == etcd.current_leader()["hostname"]:
self.send_response(200)
else:
self.send_response(503)
self.end_headers()
self.wfile.write('\r\n')
return
def run(config):
etcd = Etcd(config["etcd"])
postgresql = Postgresql(config["postgresql"])
ha = Ha(postgresql, etcd)
atexit.register(stop_postgresql, postgresql)
logging.info("Governor Starting up")
# is data directory empty?
if postgresql.data_directory_empty():
logging.info("Governor Starting up: Empty Data Dir")
# racing to initialize
wait_for_etcd("cannot initialize member without ETCD", etcd,
postgresql)
if etcd.race("/initialize", postgresql.name) or not etcd.members():
logging.info(
"Governor Starting up: Initialisation Race ... WON!!!")
logging.info("Governor Starting up: Initialise Postgres")
postgresql.initialize()
logging.info("Governor Starting up: Initialise Complete")
etcd.take_leader(postgresql.name)
logging.info("Governor Starting up: Starting Postgres")
postgresql.start()
else:
logging.info("Governor Starting up: Initialisation Race ... LOST")
logging.info("Governor Starting up: Sync Postgres from Leader")
synced_from_leader = False
while not synced_from_leader:
leader = etcd.current_leader()
if not leader:
time.sleep(5)
continue
if postgresql.sync_from_leader(leader):
logging.info("Governor Starting up: Sync Completed")
postgresql.write_recovery_conf(leader)
logging.info("Governor Starting up: Starting Postgres")
postgresql.start()
synced_from_leader = True
else:
time.sleep(5)
else:
logging.info("Governor Starting up: Existing Data Dir")
postgresql.follow_no_leader()
logging.info("Governor Starting up: Starting Postgres")
postgresql.start()
wait_for_etcd(
"running in readonly mode; cannot participate in cluster HA without etcd",
etcd, postgresql)
logging.info("Governor Running: Starting Running Loop")
while True:
try:
ha.run_cycle()
# create replication slots
if postgresql.is_leader():
logging.info("Governor Running: I am the Leader")
for node in etcd.get_client_path(
"/members?recursive=true").get("node",
{}).get("nodes", []):
member = node["key"].split('/')[-1]
if member != postgresql.name:
postgresql.query(
"DO LANGUAGE plpgsql $$DECLARE somevar VARCHAR; BEGIN SELECT slot_name INTO somevar FROM pg_replication_slots WHERE slot_name = '%(slot)s' LIMIT 1; IF NOT FOUND THEN PERFORM pg_create_physical_replication_slot('%(slot)s'); END IF; END$$;"
% {"slot": member})
etcd.touch_member(postgresql.name, postgresql.connection_string)
time.sleep(config["loop_wait"])
except urllib2.URLError:
logging.info(
"Lost connection to etcd, setting no leader and waiting on etcd"
)
postgresql.follow_no_leader()
wait_for_etcd(
"running in readonly mode; cannot participate in cluster HA without etcd",
etcd, postgresql)
def run(config):
etcd = Etcd(config["etcd"])
postgresql = Postgresql(config["postgresql"])
ha = Ha(postgresql, etcd)
atexit.register(stop_postgresql, postgresql)
signal.signal(signal.SIGTERM, signalhandler)
logging.info("Governor Starting up")
# is data directory empty?
if postgresql.data_directory_empty():
logging.info("Governor Starting up: Empty Data Dir")
# racing to initialize
wait_for_etcd("cannot initialize member without ETCD", etcd, postgresql)
if etcd.race("/initialize", postgresql.name):
logging.info("Governor Starting up: Initialisation Race ... WON!!!")
logging.info("Governor Starting up: Initialise Postgres")
postgresql.initialize()
logging.info("Governor Starting up: Initialise Complete")
etcd.take_leader(postgresql.name)
logging.info("Governor Starting up: Starting Postgres")
postgresql.start()
else:
logging.info("Governor Starting up: Initialisation Race ... LOST")
logging.info("Governor Starting up: Sync Postgres from Leader")
synced_from_leader = False
while not synced_from_leader:
leader = etcd.current_leader()
if not leader:
time.sleep(5)
continue
if postgresql.sync_from_leader(leader):
logging.info("Governor Starting up: Sync Completed")
postgresql.write_recovery_conf(leader)
logging.info("Governor Starting up: Starting Postgres")
postgresql.start()
synced_from_leader = True
else:
time.sleep(5)
else:
logging.info("Governor Starting up: Existing Data Dir")
postgresql.follow_no_leader()
logging.info("Governor Starting up: Starting Postgres")
postgresql.start()
wait_for_etcd("running in readonly mode; cannot participate in cluster HA without etcd", etcd, postgresql)
logging.info("Governor Running: Starting Running Loop")
while True:
try:
logging.info("Governor Running: %s" % ha.run_cycle())
# create replication slots
if postgresql.is_leader():
logging.info("Governor Running: I am the Leader")
for node in etcd.members():
member = node["hostname"]
if member != postgresql.name:
if postgresql.is_leader():
postgresql.ensure_replication_slot(
postgresql.replication_slot_name(member)
)
else:
postgresql.drop_replication_slot(
postgresql.replication_slot_name(member)
)
etcd.touch_member(postgresql.name, postgresql.connection_string)
time.sleep(config["loop_wait"])
except (urllib2.URLError, socket.timeout):
logging.info("Lost connection to etcd, setting no leader and waiting on etcd")
postgresql.follow_no_leader()
wait_for_etcd("running in readonly mode; cannot participate in cluster HA without etcd", etcd, postgresql)
def test_create_connection_users(self):
cfg = self.p.config
cfg['superuser']['username'] = '******'
p = Postgresql(cfg)
p.create_connection_users()
