from __future__ import print_function
import os
import sys
import re
import yaml
from lib.tarantool_server import TarantoolServer
server = TarantoolServer(server.ini)
server.script = "long_run-py/lua/finalizers.lua"
server.vardir = os.path.join(server.vardir, "finalizers")
server.crash_expected = True
try:
server.deploy()
except:
print("Expected error:", sys.exc_info()[0])
else:
print("Error! exception did not occur")
master.admin(
"space = box.schema.space.create('test', { id = 99999, engine = \"sophia\" })"
)
master.admin("index = space:create_index('primary', { type = 'tree'})")
master.admin('for k = 1, 123 do space:insert{k, k*k} end')
master.admin('box.snapshot()')
lsn = master.get_lsn(master_id)
print '-------------------------------------------------------------'
print 'replica JOIN'
print '-------------------------------------------------------------'
# replica server
replica = TarantoolServer(server.ini)
replica.script = 'replication/replica.lua'
replica.vardir = server.vardir #os.path.join(server.vardir,'replica')
replica.rpl_master = master
replica.deploy()
replica.wait_lsn(master_id, lsn)
replica.admin('box.space.test:select()')
replica.stop()
replica.cleanup(True)
# remove space
master.admin("space:drop()")
master.admin('box.snapshot()')
master.admin("ffi = require('ffi')")
master.admin("ffi.cdef(\"int sophia_schedule(void);\")")
master.admin("ffi.C.sophia_schedule() >= 0")
server.deploy()
print '-------------------------------------------------------------'
print 'Start a new replica and check box.info on the start'
print '-------------------------------------------------------------'
# master server
master = server
master_id = master.get_param('server')['id']
master.admin("box.schema.user.grant('guest', 'replication')")
replica = TarantoolServer(server.ini)
replica.script = 'replication-py/replica.lua'
replica.vardir = server.vardir
replica.rpl_master = master
replica.deploy()
replica.wait_lsn(master_id, master.get_lsn(master_id))
replica_id = replica.get_param('server')['id']
replica_uuid = replica.get_param('server')['uuid']
sys.stdout.push_filter(replica_uuid, '<replica uuid>')
replica.admin('box.info.server.id == %d' % replica_id)
replica.admin('not box.info.server.ro')
replica.admin('box.info.server.lsn == 0')
replica.admin('box.info.vclock[%d] == 0' % replica_id)
print '-------------------------------------------------------------'
print 'Modify data to change LSN and check box.info'
print '-------------------------------------------------------------'
replica.admin('box.space._schema:insert{"test", 48}')
replica.admin('box.info.server.lsn == 1')
server.deploy()
print '-------------------------------------------------------------'
print 'Start a new replica and check box.info on the start'
print '-------------------------------------------------------------'
# master server
master = server
master_id = master.get_param('server')['id']
master.admin("box.schema.user.grant('guest', 'replication')")
replica = TarantoolServer(server.ini)
replica.script = 'replication/replica.lua'
replica.vardir = server.vardir
replica.rpl_master = master
replica.deploy()
replica.wait_lsn(master_id, master.get_lsn(master_id))
replica_id = replica.get_param('server')['id']
replica_uuid = replica.get_param('server')['uuid']
sys.stdout.push_filter(replica_uuid, '<replica uuid>')
replica.admin('box.info.server.id == %d' % replica_id)
replica.admin('not box.info.server.ro')
replica.admin('box.info.server.lsn == 0')
replica.admin('box.info.vclock[%d] == 0' % replica_id)
print '-------------------------------------------------------------'
print 'Modify data to change LSN and check box.info'
print '-------------------------------------------------------------'
replica.admin('box.space._schema:insert{"test", 48}')
replica.admin('box.info.server.lsn == 1')
import os
from glob import iglob as glob
from lib.tarantool_server import TarantoolServer
# master server
master = server
master_id = master.get_param('server')['id']
master.admin("box.schema.user.grant('guest', 'replication')")
replica = TarantoolServer(server.ini)
replica.script = 'replication/replica.lua'
replica.vardir = server.vardir #os.path.join(server.vardir, 'replica')
replica.rpl_master = master
replica.deploy()
replica.wait_lsn(master_id, master.get_lsn(master_id))
replica_id = replica.get_param('server')['id']
replica.admin('box.info.server.id')
replica.admin('box.info.server.ro')
replica.admin('box.info.server.lsn')
replica.stop()
print '-------------------------------------------------------------'
print 'replica is read-only until receive self server_id in _cluster'
print '-------------------------------------------------------------'
# Remove xlog retrived by SUBSCRIBE
filename = str(0).zfill(20) + ".xlog"
wal = os.path.join(os.path.join(replica.vardir, replica.name), filename)
os.remove(wal)
master.admin("box.schema.user.grant('guest', 'replication')")
print("-------------------------------------------------------------")
print("gh-484: JOIN doesn't save data to snapshot with TREE index")
print("-------------------------------------------------------------")
master.admin("space = box.schema.space.create('test', {id = 42})")
master.admin("index = space:create_index('primary', { type = 'tree'})")
master.admin("for k = 1, 9 do space:insert{k, k*k} end")
replica = TarantoolServer(server.ini)
replica.script = "replication-py/replica.lua"
replica.vardir = server.vardir
replica.rpl_master = master
replica.deploy()
replica.admin("box.space.test:select()")
replica.restart()
replica.admin("box.space.test:select()")
replica.stop()
replica.cleanup()
print("-------------------------------------------------------------")
print("replica test 2 (must be ok)")
print("-------------------------------------------------------------")
master.restart()
master.admin("for k = 10, 19 do box.space[42]:insert{k, k*k*k} end")
master.admin("for k = 20, 29 do box.space[42]:upsert({k}, {}) end")
lsn = master.get_lsn(master_id)
# encoding: utf-8
import os
import time
from lib.tarantool_server import TarantoolServer
# master server
master = server
master_sql = master.sql
# hot standby server
hot_standby = TarantoolServer()
hot_standby.deploy("replication/cfg/hot_standby.cfg",
hot_standby.find_exe(self.args.builddir),
os.path.join(self.args.vardir, "hot_standby"), need_init=False)
hot_standby_sql = hot_standby.sql
# replica server
replica = TarantoolServer()
replica.deploy("replication/cfg/replica.cfg",
replica.find_exe(self.args.builddir),
os.path.join(self.args.vardir, "replica"))
replica_sql = replica.sql
# Begin tuple id
id = 1
print """
# Insert 10 tuples to master
"""
for i in range(id, id + 10):
import os
import sys
import re
import yaml
from lib.tarantool_server import TarantoolServer
server = TarantoolServer(server.ini)
server.script = 'long_run/lua/finalizers.lua'
server.vardir = os.path.join(server.vardir, 'finalizers')
try:
server.deploy()
except:
print "Expected error:", sys.exc_info()[0]
else:
print "Error! exception did not occur"
import yaml
from lib.memcached_connection import MemcachedConnection
from lib.tarantool_server import TarantoolServer
sonet = """The expense of spirit
in a waste of shame
Is lust in action;
and till action, lust""".split('\n')
master = server
master_memcached = master.memcached
replica = TarantoolServer()
replica.deploy("replication/cfg/replica.cfg",
replica.find_exe(self.args.builddir),
os.path.join(self.args.vardir, "replica"))
replica_memcached = replica.memcached
###################################
def get_lsn(serv):
serv_admin = serv.admin
resp = serv_admin("box.info.lsn", silent=True)
return yaml.load(resp)[0]
def wait(serv_master = master, serv_replica = replica):
lsn = get_lsn(serv_master)
serv_replica.wait_lsn(lsn)
return lsn
def get_memcached_len(serv):
replica = TarantoolServer(server.ini)
replica.script = 'replication-py/replica.lua'
replica.vardir = server.vardir #os.path.join(server.vardir, 'replica')
replica.rpl_master = master
# #1075: Box.once should wait before the server enters RW mode
#
# We expect the replica to get blocked in box.cfg{}, hence wait = False.
# Since xlog files on master were deleted, they aren't delivered,
# and replica waits indefinitely.
#
# Note: replica waits for a log entry indicating that this very replica
# joined the cluster. Once the entry is fetched we assume that the
# replica is relatively up to date and enter RW mode. Never happens in
# this particular test case.
replica.deploy(wait = False)
replica.admin('box.space.test')
replica.admin('box_cfg_done') # blocked in box.cfg it should be
replica.stop()
replica.cleanup(True)
print '-------------------------------------------------------------'
print 'replica JOIN'
print '-------------------------------------------------------------'
master.admin('box.snapshot()')
master.restart()
# encoding: utf-8
import os
import time
from lib.tarantool_server import TarantoolServer
# master server
master = server
master_sql = master.sql
# hot standby server
hot_standby = TarantoolServer()
hot_standby.deploy("replication/cfg/hot_standby.cfg",
hot_standby.find_exe(self.args.builddir),
os.path.join(self.args.vardir, "hot_standby"),
need_init=False)
hot_standby_sql = hot_standby.sql
# replica server
replica = TarantoolServer()
replica.deploy("replication/cfg/replica.cfg",
replica.find_exe(self.args.builddir),
os.path.join(self.args.vardir, "replica"))
replica_sql = replica.sql
# Begin tuple id
id = 1
print """
# Insert 10 tuples to master
"""
for i in range(id, id + 10):
server.deploy()
print("-------------------------------------------------------------")
print("Start a new replica and check box.info on the start")
print("-------------------------------------------------------------")
# master server
master = server
master_id = master.get_param("id")
master.admin("box.schema.user.grant('guest', 'replication')")
replica = TarantoolServer(server.ini)
replica.script = "replication-py/replica.lua"
replica.vardir = server.vardir
replica.rpl_master = master
replica.deploy()
replica_id = replica.get_param("id")
replica_uuid = replica.get_param("uuid")
sys.stdout.push_filter(replica_uuid, "<replica uuid>")
replica.admin("box.info.id == {}".format(replica_id))
replica.admin("not box.info.ro")
replica.admin("box.info.lsn == 0")
replica.admin("box.info.vclock[{}] == nil".format(replica_id))
print("-------------------------------------------------------------")
print("Modify data to bump LSN and check box.info")
print("-------------------------------------------------------------")
replica.admin("box.space._schema:insert{\"test\", 48}")
replica.admin("box.info.lsn == 1")
replica.admin("box.info.vclock[{}] == 1".format(replica_id))
class Supervisor(object):
def __init__(self, suite_path, args):
self.args = args
self.tests = []
self.suite_path = suite_path
self.ini = {
'core': 'tarantool',
'script': os.path.join(suite_path, 'parallel.lua'),
}
# read suite config
config = ConfigParser.ConfigParser()
config.read(os.path.join(suite_path, "suite.ini"))
self.ini.update(dict(config.items("default")))
self.ini.update(self.args.__dict__)
self.jobs = int(self.ini.get('jobs', 1))
self.count = int(self.ini.get('count', 0))
for i in ["script"]:
self.ini[i] = os.path.join(suite_path,
self.ini[i]) if i in self.ini else None
self.server = TarantoolServer(self.ini)
self.pool = None
self.iterator = None
def find_tests(self):
self.tests += [Parallel_PythonTest(k) \
for k in sorted(glob.glob(os.path.join(self.suite_path, "*.test.py" )))]
def take_rand(self):
if self.count != 0:
for test in self.tests:
sql = self.server.sql.clone()
admin = self.server.admin.clone()
yield [test, [sql, admin]]
else:
while True:
sql = self.server.sql.clone()
admin = self.server.admin.clone()
yield [random.choice(self.tests), [sql, admin]]
def run_all(self):
self.find_tests()
if self.count != 0:
self.tests *= self.count
random.shuffle(self.tests)
self.pool = GopherPool(processes=self.jobs)
self.iterator = self.pool.run()
self.filler = self.pool.fill(self.take_rand())
try:
self.server.cleanup()
logger.info("Tarantool.Instance > Server cleaned up")
logger.info("Tarantool.Instance > Server's path: %s",
self.server.binary)
self.server.deploy()
logger.info("Tarantool.Instance > Server deployed")
try:
while True:
self.filler.next()
logger.debug("BigBrother.run > Jobs filled %d %d" %
(self.pool.queuein.qsize(),
self.pool.queueout.qsize()))
while True:
try:
logger.debug("BigBrother.run > waiting for task")
task = self.iterator.next(1)
logger.debug("BigBrother.run > took task")
if task is None:
logger.info('>>>> Test return NONE')
continue
stat = task.get_status()
if stat.status != 3:
logger.info('>>>> Test %s finished' %
repr(task.name))
else:
logger.error(
'>>>> Test %s failed with %s (%s)' % (repr(
task.name), stat.message, stat.reject))
except (QueueEmpty, StopIteration):
break
except StopIteration:
pass
finally:
self.server.stop()
logger.info("Tarantool.Instance > Server stopped")
class Supervisor(object):
def __init__(self, suite_path, args):
self.args = args
self.tests = []
self.suite_path = suite_path
self.ini = {
'core': 'tarantool',
'script': os.path.join(suite_path, 'parallel.lua'),
}
# read suite config
config = ConfigParser.ConfigParser()
config.read(os.path.join(suite_path, "suite.ini"))
self.ini.update(dict(config.items("default")))
self.ini.update(self.args.__dict__)
self.jobs = int(self.ini.get('jobs', 1))
self.count = int(self.ini.get('count', 0))
for i in ["script"]:
self.ini[i] = os.path.join(suite_path, self.ini[i]) if i in self.ini else None
self.server = TarantoolServer(self.ini)
self.pool = None
self.iterator = None
def find_tests(self):
self.tests += [Parallel_PythonTest(k) \
for k in sorted(glob.glob(os.path.join(self.suite_path, "*.test.py" )))]
def take_rand(self):
if self.count != 0:
for test in self.tests:
sql = self.server.sql.clone()
admin = self.server.admin.clone()
yield [test, [sql, admin]]
else:
while True:
sql = self.server.sql.clone()
admin = self.server.admin.clone()
yield [random.choice(self.tests), [sql, admin]]
def run_all(self):
self.find_tests()
if self.count != 0:
self.tests *= self.count
random.shuffle(self.tests)
self.pool = GopherPool(processes = self.jobs)
self.iterator = self.pool.run()
self.filler = self.pool.fill(self.take_rand())
try:
self.server.cleanup()
logger.info("Tarantool.Instance > Server cleaned up")
logger.info("Tarantool.Instance > Server's path: %s", self.server.binary)
self.server.deploy()
logger.info("Tarantool.Instance > Server deployed")
try:
while True:
self.filler.next()
logger.debug("BigBrother.run > Jobs filled %d %d" %
(self.pool.queuein.qsize(), self.pool.queueout.qsize()))
while True:
try:
logger.debug("BigBrother.run > waiting for task")
task = self.iterator.next(1)
logger.debug("BigBrother.run > took task")
if task is None:
logger.info('>>>> Test return NONE')
continue
stat = task.get_status()
if stat.status != 3:
logger.info('>>>> Test %s finished' % repr(task.name))
else:
logger.error('>>>> Test %s failed with %s (%s)' %
(repr(task.name), stat.message, stat.reject))
except (QueueEmpty, StopIteration):
break
except StopIteration:
pass
finally:
self.server.stop()
logger.info("Tarantool.Instance > Server stopped")
