Ejemplo n.º 1
0
 def __init__(self, root_db_name, branch_db_name, leaf_db_name=None, basedir = 'st3test', howto=None):
     self.root_db_name = root_db_name
     self.branch_db_name = branch_db_name
     self.leaf_db_name = leaf_db_name
     self.basedir = basedir
     self.pgqd = PgQD(basedir = self.basedir)
     self.db1 = PGDatabase(self.root_db_name)
     self.node1 = Londiste3Node(self.db1, 'root', 'node1', basedir = self.basedir)
     self.pgb = PGBench(2,self.db1)
     self.db2 = PGDatabase(self.branch_db_name)
     self.node2 = Londiste3Node(self.db2, 'branch', 'node2', provider_db=self.db1, basedir = self.basedir)
     if self.leaf_db_name:
         self.init_leaf_node()
     self._howto = howto
Ejemplo n.º 2
0
class PGBench:
    "class for interacting with pgbench utility"
    def __init__(self, scale, db=None, dbname=None, host=None, port=None, user=None, passwd=None):
        self.scale = scale
        if db:
            self.database = db
        else:
            self.database = PGDatabase(dbname, host, port, user, passwd)
    def pgb_init_db(self):
        "just run 'pgbench -i -s'"
        # run pgbench -i -s %(self.scale)s
        cmd = [PGBENCH, '-i', '-s', str(self.scale), '-F', '80'] + self.database.get_db_cmdline_args()
        run_cmd(cmd)
    def modify_db_for_replication(self):
        "add pk's, foreign keys, partitions, ..."
        # add support for moddatetime triggers from contrib
        cmd = ['psql'] + self.database.get_db_cmdline_args() + ['-f', '/usr/share/postgresql/8.4/contrib/moddatetime.sql']
#        print cmd
        run_cmd(cmd)
        # modify db and commit
        with open('/tmp/prepare_pgbenchdb_for_londiste.sql', 'w') as f:
            f.write(pgbencdb_mods)
        howto("create file /tmp/prepare_pgbenchdb_for_londiste.sql with the following ...\n----\n%s\n----\n" % pgbencdb_mods)
        howto(". . . and then execute it")
        cmd = ['psql'] + self.database.get_db_cmdline_args() + ['-f', '/tmp/prepare_pgbenchdb_for_londiste.sql']
        run_cmd(cmd)
        # run pgbench -T 10 to populate db with some more data
        # self.pgb_run(10,16)
    def pgb_run (self, seconds=60, concurrency=25, filename=None, background=False):
        # run pgbench -T 10 to populate db with some data
        cmd = [PGBENCH, '-T', '%d' % seconds, '-c', '%d' % concurrency] + self.database.get_db_cmdline_args()
        if filename:
            cmd += ['-f', filename]
        if background:
            run_in_background(cmd)
        else:
            run_cmd(cmd)
    def preparedb(self):
        "prepare db for replication"
        self.pgb_init_db()
        self.modify_db_for_replication()
    def capture_unmodified_schema(self):
        "read and save result of 'pg_dump -s'"
Ejemplo n.º 3
0
class Londiste2Test(unittest.TestCase):
    "tests that pgbench is runnable"
    def setUp(self):
        from PGDatabase import PGDatabase
        self.db = PGDatabase('pgbtest', port='5432')
        self.db2 = PGDatabase('pgbtest2', port='5432')
        self.londiste = Londiste2(self.db, self.db2)
    def tearDown(self):
        pass
    def test_replica_setup(self):
        from PGBench import PGBench
        self.db.createdb()
        self.pgbench = PGBench(2, db=self.db)
        self.pgbench.preparedb()
        self.db2.createdb()
        self.db2.copy_schema_from(self.db)
        if 0: # if londiste was installed on source
            self.db2.cleanup_londiste_provider()
    def test_create_ticker_ini(self):
        self.londiste.create_ticker_ini()
    def test_install_ticker(self):
        self.londiste.install_ticker()
    def test_start_ticker(self):
        self.londiste.start_ticker()
    def test_create_provider_ini(self):
        self.londiste.create_provider_ini()
    def test_create_subscriber_ini(self):
        self.londiste.create_subscriber_ini()
    def test_install_londiste_on_provider(self):
        self.londiste.install_londiste_on_provider()
    def test_install_londiste_on_subscriber(self):
        self.londiste.install_londiste_on_subscriber()
    def test_start_londiste_replay(self):
        self.londiste.start_londiste_replay()
    def test_add_tables_on_provider(self):
        self.londiste.add_tables_on_provider(['--all'])
    def test_add_tables_on_subscriber(self):
        self.londiste.add_tables_on_subscriber(['--all'])
    def test_wait_for_subscription(self):
        self.londiste.wait_for_subscription()
    def test_check_lag(self):
        self.londiste.check_lag()
    def test_check_data(self):
        self.londiste.check_data()
    def test_(self):
        self.londiste.x()
    def test_(self):
        self.londiste.x()
Ejemplo n.º 4
0
def setup_londiste3_replication():
    db1 = PGDatabase('l3db1')
    pgb = PGBench(2,db1)
    l3_node1 = Londiste3Node(db1, 'root', 'node1', basedir = 'st3test')
    db2 = PGDatabase('l3db2')
    l3_node2 = Londiste3Node(db2, 'branch', 'node2', provider_db=db1, basedir = 'st3test')
    pgqd = PgQD(basedir = 'st3test')
    if 0:
	    pgqd.create_ini_file()
	    # create and populate db
	    db1.createdb()
	    pgb.pgb_init_db()
	    pgb.modify_db_for_replication()
	    # create node
	    l3_node1.create_ini_file()
	    l3_node1.create_node()
	    l3_node1.start()
	    # create other dbs and nodes
	    db2.createdb()
	    l3_node2.create_ini_file()
	    l3_node2.create_node()
	    l3_node2.start()
	    # start ticker daemon
	    pgqd.start()
	    # wait a sec
	    time.sleep(1)
	    # check aall daemons are running
	    print 'pgqd.check()', pgqd.check()
	    print 'l3_node1.check()', l3_node1.check()
	    print 'l3_node2.check()', l3_node2.check()
	    # copy schema
	    db2.copy_schema_from(db1)
	    # add tables
	    l3_node1.add_tables()
	    l3_node2.add_tables()
	    pgb.pgb_run(10,10)
	    print 'waiting for node2 to sync'
	    l3_node2.wait_replication_state_ok()
	    l3_node2.compare_tables()
	    print 'replication graph and lags'
    l3_node1.status()
    print 'Adding column on provider'
    l3_node1.execute('alter table pgbench_accounts add column reptestcolumn text')
    # wait a few sec
    time.sleep(3)
    print 'now see if pgbench_accounts.reptestcolumn exists in l3db2\n'
    print "use:\n"
    print "   psql l3db2 -c '\d pgbench_accounts'"
Ejemplo n.º 5
0
 def __init__(self, scale, db=None, dbname=None, host=None, port=None, user=None, passwd=None):
     self.scale = scale
     if db:
         self.database = db
     else:
         self.database = PGDatabase(dbname, host, port, user, passwd)
Ejemplo n.º 6
0
 def setUp(self):
     from PGDatabase import PGDatabase
     self.db = PGDatabase('pgbtest', port='5432')
     self.db2 = PGDatabase('pgbtest2', port='5432')
     self.londiste = Londiste2(self.db, self.db2)
Ejemplo n.º 7
0
 def init_leaf_node(self):
     self.db3 = PGDatabase(self.leaf_db_name )
     self.node3 = Londiste3Node(self.db3, 'leaf', 'node3', provider_db=self.db1, basedir = self.basedir)
Ejemplo n.º 8
0
class Londiste3Cluster:
    def __init__(self, root_db_name, branch_db_name, leaf_db_name=None, basedir = 'st3test', howto=None):
        self.root_db_name = root_db_name
        self.branch_db_name = branch_db_name
        self.leaf_db_name = leaf_db_name
        self.basedir = basedir
        self.pgqd = PgQD(basedir = self.basedir)
        self.db1 = PGDatabase(self.root_db_name)
        self.node1 = Londiste3Node(self.db1, 'root', 'node1', basedir = self.basedir)
        self.pgb = PGBench(2,self.db1)
        self.db2 = PGDatabase(self.branch_db_name)
        self.node2 = Londiste3Node(self.db2, 'branch', 'node2', provider_db=self.db1, basedir = self.basedir)
        if self.leaf_db_name:
            self.init_leaf_node()
        self._howto = howto
    def howto(self, text):
        if not self._howto: return
        print text
    def init_leaf_node(self):
        self.db3 = PGDatabase(self.leaf_db_name )
        self.node3 = Londiste3Node(self.db3, 'leaf', 'node3', provider_db=self.db1, basedir = self.basedir)
    def start(self):
        self.howto('== Set up schema for root database')
        self.setup_root_schema()
        self.setup_replicated_schema(self.db2)
        if self.leaf_db_name:
            self.setup_replicated_schema(self.db3)
        self.setup_node(self.node1)
        self.setup_node(self.node2)
        if self.leaf_db_name:
            self.setup_node(self.node3)
        self.pgqd.create_ini_file()
        self.pgqd.start()
    def setup_root_schema(self):
        # create database
        self.howto('=== Create database ===')
        self.db1.createdb()
        # create schemas, populate with data and modify for replication
        self.howto('=== set up schema for pgbench ===')
        self.pgb.pgb_init_db()
        self.howto('=== and add primary and foreign keys needed for replication ===')
        self.pgb.modify_db_for_replication()
    def setup_replicated_schema(self, db):
        # create database
        db.createdb()
        # copy schema from root database
        db.copy_schema_from(self.db1)
    def setup_node(self, node):
        # install londiste3 support and start replay process
        node.create_ini_file()
        node.create_node()
        node.start()
    def check_processes(self):
        time.sleep(1) # let things settle
        print 'pgqd.check()', self.pgqd.check()
        print 'l3_node1.check()', self.node1.check()
        print 'l3_node2.check()', self.node2.check()
        # print replication tree
        self.node1.status()
    def add_tables(self, tablelist=['--all'], wait=True):
        self.node1.add_tables(tablelist)
        self.node2.add_tables(tablelist)
        if wait:
            print 'waiting for subscription to finish'
            time.sleep(3) # let things settle
            self.node2.wait_replication_state_ok()
    def start_pgbench_in_background(self, seconds=120):
        # starts a long pgbench run in background to run for 2 minutes
        with open('/tmp/throttled.pgbench', 'w') as f:
            f.write(throttled_pgbench_script)
        self.pgb.pgb_run(seconds=seconds, concurrency=5, filename='/tmp/throttled.pgbench', background=True)
    def check_data(self):
        self.node2.compare_tables()
        if self.leaf_db_name:
            self.node3.compare_tables()
    def tearDown(self):
        self.pgqd.stop()
        self.node1.stop()
        self.node2.stop()
        if self.leaf_db_name:
            self.node3.stop()
        self.db1.dropdb()
        self.db2.dropdb()
        if self.leaf_db_name:
            self.db3.dropdb()