def __init__(self, root_db_name, branch_db_name, leaf_db_name=None, basedir = 'st3test', howto=None):
self.root_db_name = root_db_name
self.branch_db_name = branch_db_name
self.leaf_db_name = leaf_db_name
self.basedir = basedir
self.pgqd = PgQD(basedir = self.basedir)
self.db1 = PGDatabase(self.root_db_name)
self.node1 = Londiste3Node(self.db1, 'root', 'node1', basedir = self.basedir)
self.pgb = PGBench(2,self.db1)
self.db2 = PGDatabase(self.branch_db_name)
self.node2 = Londiste3Node(self.db2, 'branch', 'node2', provider_db=self.db1, basedir = self.basedir)
if self.leaf_db_name:
self.init_leaf_node()
self._howto = howto
class PGBench:
"class for interacting with pgbench utility"
def __init__(self, scale, db=None, dbname=None, host=None, port=None, user=None, passwd=None):
self.scale = scale
if db:
self.database = db
else:
self.database = PGDatabase(dbname, host, port, user, passwd)
def pgb_init_db(self):
"just run 'pgbench -i -s'"
# run pgbench -i -s %(self.scale)s
cmd = [PGBENCH, '-i', '-s', str(self.scale), '-F', '80'] + self.database.get_db_cmdline_args()
run_cmd(cmd)
def modify_db_for_replication(self):
"add pk's, foreign keys, partitions, ..."
# add support for moddatetime triggers from contrib
cmd = ['psql'] + self.database.get_db_cmdline_args() + ['-f', '/usr/share/postgresql/8.4/contrib/moddatetime.sql']
# print cmd
run_cmd(cmd)
# modify db and commit
with open('/tmp/prepare_pgbenchdb_for_londiste.sql', 'w') as f:
f.write(pgbencdb_mods)
howto("create file /tmp/prepare_pgbenchdb_for_londiste.sql with the following ...\n----\n%s\n----\n" % pgbencdb_mods)
howto(". . . and then execute it")
cmd = ['psql'] + self.database.get_db_cmdline_args() + ['-f', '/tmp/prepare_pgbenchdb_for_londiste.sql']
run_cmd(cmd)
# run pgbench -T 10 to populate db with some more data
# self.pgb_run(10,16)
def pgb_run (self, seconds=60, concurrency=25, filename=None, background=False):
# run pgbench -T 10 to populate db with some data
cmd = [PGBENCH, '-T', '%d' % seconds, '-c', '%d' % concurrency] + self.database.get_db_cmdline_args()
if filename:
cmd += ['-f', filename]
if background:
run_in_background(cmd)
else:
run_cmd(cmd)
def preparedb(self):
"prepare db for replication"
self.pgb_init_db()
self.modify_db_for_replication()
def capture_unmodified_schema(self):
"read and save result of 'pg_dump -s'"
class Londiste2Test(unittest.TestCase):
"tests that pgbench is runnable"
def setUp(self):
from PGDatabase import PGDatabase
self.db = PGDatabase('pgbtest', port='5432')
self.db2 = PGDatabase('pgbtest2', port='5432')
self.londiste = Londiste2(self.db, self.db2)
def tearDown(self):
pass
def test_replica_setup(self):
from PGBench import PGBench
self.db.createdb()
self.pgbench = PGBench(2, db=self.db)
self.pgbench.preparedb()
self.db2.createdb()
self.db2.copy_schema_from(self.db)
if 0: # if londiste was installed on source
self.db2.cleanup_londiste_provider()
def test_create_ticker_ini(self):
self.londiste.create_ticker_ini()
def test_install_ticker(self):
self.londiste.install_ticker()
def test_start_ticker(self):
self.londiste.start_ticker()
def test_create_provider_ini(self):
self.londiste.create_provider_ini()
def test_create_subscriber_ini(self):
self.londiste.create_subscriber_ini()
def test_install_londiste_on_provider(self):
self.londiste.install_londiste_on_provider()
def test_install_londiste_on_subscriber(self):
self.londiste.install_londiste_on_subscriber()
def test_start_londiste_replay(self):
self.londiste.start_londiste_replay()
def test_add_tables_on_provider(self):
self.londiste.add_tables_on_provider(['--all'])
def test_add_tables_on_subscriber(self):
self.londiste.add_tables_on_subscriber(['--all'])
def test_wait_for_subscription(self):
self.londiste.wait_for_subscription()
def test_check_lag(self):
self.londiste.check_lag()
def test_check_data(self):
self.londiste.check_data()
def test_(self):
self.londiste.x()
def test_(self):
self.londiste.x()
def setup_londiste3_replication():
db1 = PGDatabase('l3db1')
pgb = PGBench(2,db1)
l3_node1 = Londiste3Node(db1, 'root', 'node1', basedir = 'st3test')
db2 = PGDatabase('l3db2')
l3_node2 = Londiste3Node(db2, 'branch', 'node2', provider_db=db1, basedir = 'st3test')
pgqd = PgQD(basedir = 'st3test')
if 0:
pgqd.create_ini_file()
# create and populate db
db1.createdb()
pgb.pgb_init_db()
pgb.modify_db_for_replication()
# create node
l3_node1.create_ini_file()
l3_node1.create_node()
l3_node1.start()
# create other dbs and nodes
db2.createdb()
l3_node2.create_ini_file()
l3_node2.create_node()
l3_node2.start()
# start ticker daemon
pgqd.start()
# wait a sec
time.sleep(1)
# check aall daemons are running
print 'pgqd.check()', pgqd.check()
print 'l3_node1.check()', l3_node1.check()
print 'l3_node2.check()', l3_node2.check()
# copy schema
db2.copy_schema_from(db1)
# add tables
l3_node1.add_tables()
l3_node2.add_tables()
pgb.pgb_run(10,10)
print 'waiting for node2 to sync'
l3_node2.wait_replication_state_ok()
l3_node2.compare_tables()
print 'replication graph and lags'
l3_node1.status()
print 'Adding column on provider'
l3_node1.execute('alter table pgbench_accounts add column reptestcolumn text')
# wait a few sec
time.sleep(3)
print 'now see if pgbench_accounts.reptestcolumn exists in l3db2\n'
print "use:\n"
print " psql l3db2 -c '\d pgbench_accounts'"
def __init__(self, scale, db=None, dbname=None, host=None, port=None, user=None, passwd=None):
self.scale = scale
if db:
self.database = db
else:
self.database = PGDatabase(dbname, host, port, user, passwd)
def setUp(self):
from PGDatabase import PGDatabase
self.db = PGDatabase('pgbtest', port='5432')
self.db2 = PGDatabase('pgbtest2', port='5432')
self.londiste = Londiste2(self.db, self.db2)
def init_leaf_node(self):
self.db3 = PGDatabase(self.leaf_db_name )
self.node3 = Londiste3Node(self.db3, 'leaf', 'node3', provider_db=self.db1, basedir = self.basedir)
class Londiste3Cluster:
def __init__(self, root_db_name, branch_db_name, leaf_db_name=None, basedir = 'st3test', howto=None):
self.root_db_name = root_db_name
self.branch_db_name = branch_db_name
self.leaf_db_name = leaf_db_name
self.basedir = basedir
self.pgqd = PgQD(basedir = self.basedir)
self.db1 = PGDatabase(self.root_db_name)
self.node1 = Londiste3Node(self.db1, 'root', 'node1', basedir = self.basedir)
self.pgb = PGBench(2,self.db1)
self.db2 = PGDatabase(self.branch_db_name)
self.node2 = Londiste3Node(self.db2, 'branch', 'node2', provider_db=self.db1, basedir = self.basedir)
if self.leaf_db_name:
self.init_leaf_node()
self._howto = howto
def howto(self, text):
if not self._howto: return
print text
def init_leaf_node(self):
self.db3 = PGDatabase(self.leaf_db_name )
self.node3 = Londiste3Node(self.db3, 'leaf', 'node3', provider_db=self.db1, basedir = self.basedir)
def start(self):
self.howto('== Set up schema for root database')
self.setup_root_schema()
self.setup_replicated_schema(self.db2)
if self.leaf_db_name:
self.setup_replicated_schema(self.db3)
self.setup_node(self.node1)
self.setup_node(self.node2)
if self.leaf_db_name:
self.setup_node(self.node3)
self.pgqd.create_ini_file()
self.pgqd.start()
def setup_root_schema(self):
# create database
self.howto('=== Create database ===')
self.db1.createdb()
# create schemas, populate with data and modify for replication
self.howto('=== set up schema for pgbench ===')
self.pgb.pgb_init_db()
self.howto('=== and add primary and foreign keys needed for replication ===')
self.pgb.modify_db_for_replication()
def setup_replicated_schema(self, db):
# create database
db.createdb()
# copy schema from root database
db.copy_schema_from(self.db1)
def setup_node(self, node):
# install londiste3 support and start replay process
node.create_ini_file()
node.create_node()
node.start()
def check_processes(self):
time.sleep(1) # let things settle
print 'pgqd.check()', self.pgqd.check()
print 'l3_node1.check()', self.node1.check()
print 'l3_node2.check()', self.node2.check()
# print replication tree
self.node1.status()
def add_tables(self, tablelist=['--all'], wait=True):
self.node1.add_tables(tablelist)
self.node2.add_tables(tablelist)
if wait:
print 'waiting for subscription to finish'
time.sleep(3) # let things settle
self.node2.wait_replication_state_ok()
def start_pgbench_in_background(self, seconds=120):
# starts a long pgbench run in background to run for 2 minutes
with open('/tmp/throttled.pgbench', 'w') as f:
f.write(throttled_pgbench_script)
self.pgb.pgb_run(seconds=seconds, concurrency=5, filename='/tmp/throttled.pgbench', background=True)
def check_data(self):
self.node2.compare_tables()
if self.leaf_db_name:
self.node3.compare_tables()
def tearDown(self):
self.pgqd.stop()
self.node1.stop()
self.node2.stop()
if self.leaf_db_name:
self.node3.stop()
self.db1.dropdb()
self.db2.dropdb()
if self.leaf_db_name:
self.db3.dropdb()