def setUp(self):
if self.vttop is None:
raise EnvironmentError("VTTOP not defined")
if self.vtroot is None:
raise EnvironmentError("VTROOT not defined")
try:
os.makedirs(utils.tmp_root)
except OSError:
pass
utils.setup()
framework.execute('go build',
verbose=utils.options.verbose,
cwd=self.vttop + '/go/cmd/vtocc')
framework.execute('go build',
verbose=utils.options.verbose,
cwd=self.vttop + '/go/cmd/mysqlctl')
# start mysql
res = subprocess.call([
self.vtroot + "/bin/mysqlctl", "-tablet-uid", self.tabletuid,
"-port",
str(self.vtoccport), "-mysql-port",
str(self.mysqlport), "init"
])
if res != 0:
raise EnvironmentError("Cannot start mysql")
res = subprocess.call([
"mysql", "-S", self.mysqldir + "/mysql.sock", "-u", "vt_dba", "-e",
"create database vt_test_keyspace ; set global read_only = off"
])
if res != 0:
raise Exception("Cannot create vt_test_keyspace database")
dbconfig = self.mysqldir + "/dbconf.json"
with open(dbconfig, 'w') as f:
conf = {
'charset': 'utf8',
'dbname': 'vt_test_keyspace',
'host': 'localhost',
'unix_socket': self.mysqldir + "/mysql.sock",
'uname': 'vt_dba', # use vt_dba as some tests depend on 'drop'
'keyspace': 'test_keyspace',
'shard': '0',
}
json.dump(conf, f)
self.mysql_conn = self.mysql_connect()
mcu = self.mysql_conn.cursor()
self.clean_sqls = []
self.init_sqls = []
clean_mode = False
with open(
os.path.join(self.vttop, "test", "test_data",
"test_schema.sql")) as f:
for line in f:
line = line.rstrip()
if line == "# clean":
clean_mode = True
if line == '' or line.startswith("#"):
continue
if clean_mode:
self.clean_sqls.append(line)
else:
self.init_sqls.append(line)
try:
for line in self.init_sqls:
mcu.execute(line, {})
finally:
mcu.close()
customrules = '/tmp/customrules.json'
self.create_customrules(customrules)
schema_override = '/tmp/schema_override.json'
self.create_schema_override(schema_override)
occ_args = [
self.vtroot + "/bin/vtocc",
"-port",
"9461",
"-dbconfig",
dbconfig,
"-customrules",
customrules,
"-schema-override",
schema_override,
]
if utils.options.memcache:
memcache = self.mysqldir + "/memcache.sock"
config = self.mysqldir + "/config.json"
with open(config, 'w') as f:
json.dump({"RowCache": ["memcached", "-s", memcache]}, f)
occ_args.extend(["-queryserver-config-file", config])
self.vtstderr = open("/tmp/vtocc_stderr.log", "a+")
self.vtocc = subprocess.Popen(occ_args, stderr=self.vtstderr)
for i in range(30):
try:
self.conn = self.connect()
self.txlogger = subprocess.Popen(
['curl', '-s', '-N', 'http://localhost:9461/debug/txlog'],
stdout=open('/tmp/vtocc_txlog.log', 'w'))
self.txlog = framework.Tailer(open('/tmp/vtocc_txlog.log',
'r'))
def flush():
utils.run([
'curl', '-s', '-N',
'http://localhost:9461/debug/flushlogs'
],
trap_output=True)
self.log = framework.Tailer(open('/tmp/vtocc.INFO'),
flush=flush)
utils.run_bg([
'curl', '-s', '-N',
'http://localhost:9461/debug/querylog?full=true'
],
stdout=open('/tmp/vtocc_streamlog_9461.log', 'w'))
self.querylog = framework.Tailer(
open('/tmp/vtocc_streamlog_9461.log'), sleep=0.1)
return
except dbexceptions.OperationalError:
if i == 29:
raise
time.sleep(1)
def setUp(self):
utils.zk_setup()
utils.setup()
if self.vttop is None:
raise EnvironmentError("VTTOP not defined")
if self.vtroot is None:
raise EnvironmentError("VTROOT not defined")
framework.execute('go build',
verbose=utils.options.verbose,
cwd=self.vttop + '/go/cmd/mysqlctl')
utils.wait_procs([self.tablet.init_mysql()])
self.tablet.mquery(
"",
["create database vt_test_keyspace", "set global read_only = off"])
self.mysql_conn, mcu = self.tablet.connect('vt_test_keyspace')
self.clean_sqls = []
self.init_sqls = []
clean_mode = False
with open(
os.path.join(self.vttop, "test", "test_data",
"test_schema.sql")) as f:
for line in f:
line = line.rstrip()
if line == "# clean":
clean_mode = True
if line == '' or line.startswith("#"):
continue
if clean_mode:
self.clean_sqls.append(line)
else:
self.init_sqls.append(line)
try:
for line in self.init_sqls:
mcu.execute(line, {})
finally:
mcu.close()
utils.run_vtctl(
'CreateKeyspace -force /zk/global/vt/keyspaces/test_keyspace')
self.tablet.init_tablet('master', 'test_keyspace', '0')
customrules = '/tmp/customrules.json'
self.create_customrules(customrules)
schema_override = '/tmp/schema_override.json'
self.create_schema_override(schema_override)
if utils.options.memcache:
self.tablet.start_vttablet(memcache=True,
customrules=customrules,
schema_override=schema_override)
else:
self.tablet.start_vttablet(customrules=customrules,
schema_override=schema_override)
# FIXME(szopa): This is necessary here only because of a bug that
# makes the qs reload its config only after an action.
utils.run_vtctl('Ping ' + self.tablet.zk_tablet_path)
for i in range(30):
try:
self.conn = self.connect()
self.txlogger = subprocess.Popen(
['curl', '-s', '-N', 'http://localhost:9461/debug/txlog'],
stdout=open('/tmp/vtocc_txlog.log', 'w'))
self.txlog = framework.Tailer(open('/tmp/vtocc_txlog.log'),
flush=self.tablet.flush)
self.log = framework.Tailer(open(
os.path.join(self.tablet.tablet_dir, 'vttablet.INFO')),
flush=self.tablet.flush)
querylog_file = '/tmp/vtocc_streamlog_%s.log' % self.tablet.port
utils.run_bg([
'curl', '-s', '-N',
'http://localhost:9461/debug/querylog?full=true'
],
stdout=open(querylog_file, 'w'))
time.sleep(1)
self.querylog = framework.Tailer(open(querylog_file),
sleep=0.1)
return
except dbexceptions.OperationalError:
if i == 29:
raise
time.sleep(1)
def setUp(self):
if self.vttop is None:
raise EnvironmentError("VTTOP not defined")
if self.vtroot is None:
raise EnvironmentError("VTROOT not defined")
try:
os.makedirs(utils.tmp_root)
except OSError:
pass
utils.setup()
framework.execute('go build', verbose=utils.options.verbose, cwd=self.vttop+'/go/cmd/vtocc')
framework.execute('go build', verbose=utils.options.verbose, cwd=self.vttop+'/go/cmd/mysqlctl')
# start mysql
res = subprocess.call([
self.vtroot+"/bin/mysqlctl",
"-tablet-uid", self.tabletuid,
"-port", str(self.vtoccport),
"-mysql-port", str(self.mysqlport),
"init"
])
if res != 0:
raise EnvironmentError("Cannot start mysql")
res = subprocess.call([
"mysql",
"-S", self.mysqldir+"/mysql.sock",
"-u", "vt_dba",
"-e", "create database vt_test_keyspace ; set global read_only = off"])
if res != 0:
raise Exception("Cannot create vt_test_keyspace database")
dbconfig = self.mysqldir+"/dbconf.json"
with open(dbconfig, 'w') as f:
conf = {
'charset': 'utf8',
'dbname': 'vt_test_keyspace',
'host': 'localhost',
'unix_socket': self.mysqldir+"/mysql.sock",
'uname': 'vt_dba', # use vt_dba as some tests depend on 'drop'
'keyspace': 'test_keyspace',
'shard' : '0',
}
json.dump(conf, f)
self.mysql_conn = self.mysql_connect()
mcu = self.mysql_conn.cursor()
self.clean_sqls = []
self.init_sqls = []
clean_mode = False
with open(os.path.join(self.vttop, "test", "test_data", "test_schema.sql")) as f:
for line in f:
line = line.rstrip()
if line == "# clean":
clean_mode = True
if line=='' or line.startswith("#"):
continue
if clean_mode:
self.clean_sqls.append(line)
else:
self.init_sqls.append(line)
try:
for line in self.init_sqls:
mcu.execute(line, {})
finally:
mcu.close()
customrules = '/tmp/customrules.json'
self.create_customrules(customrules)
schema_override = '/tmp/schema_override.json'
self.create_schema_override(schema_override)
occ_args = [
self.vtroot+"/bin/vtocc",
"-port", "9461",
"-dbconfig", dbconfig,
"-customrules", customrules,
"-schema-override", schema_override,
]
if utils.options.memcache:
memcache = self.mysqldir+"/memcache.sock"
config = self.mysqldir+"/config.json"
with open(config, 'w') as f:
json.dump({"RowCache": ["memcached", "-s", memcache]}, f)
occ_args.extend(["-queryserver-config-file", config])
self.vtstderr = open("/tmp/vtocc_stderr.log", "a+")
self.vtocc = subprocess.Popen(occ_args, stderr=self.vtstderr)
for i in range(30):
try:
self.conn = self.connect()
self.txlogger = subprocess.Popen(['curl', '-s', '-N', 'http://localhost:9461/debug/txlog'], stdout=open('/tmp/vtocc_txlog.log', 'w'))
self.txlog = framework.Tailer(open('/tmp/vtocc_txlog.log', 'r'))
def flush():
utils.run(['curl', '-s', '-N', 'http://localhost:9461/debug/flushlogs'], trap_output=True)
self.log = framework.Tailer(open('/tmp/vtocc.INFO'), flush=flush)
utils.run_bg(['curl', '-s', '-N', 'http://localhost:9461/debug/querylog?full=true'], stdout=open('/tmp/vtocc_streamlog_9461.log', 'w'))
self.querylog = framework.Tailer(open('/tmp/vtocc_streamlog_9461.log'), sleep=0.1)
return
except dbexceptions.OperationalError:
if i == 29:
raise
time.sleep(1)
def setUp(self):
utils.zk_setup()
utils.setup()
if self.vttop is None:
raise EnvironmentError("VTTOP not defined")
if self.vtroot is None:
raise EnvironmentError("VTROOT not defined")
framework.execute('go build', verbose=utils.options.verbose, cwd=self.vttop+'/go/cmd/mysqlctl')
utils.wait_procs([self.tablet.init_mysql()])
self.tablet.mquery("", ["create database vt_test_keyspace", "set global read_only = off"])
self.mysql_conn, mcu = self.tablet.connect('vt_test_keyspace')
self.clean_sqls = []
self.init_sqls = []
clean_mode = False
with open(os.path.join(self.vttop, "test", "test_data", "test_schema.sql")) as f:
for line in f:
line = line.rstrip()
if line == "# clean":
clean_mode = True
if line=='' or line.startswith("#"):
continue
if clean_mode:
self.clean_sqls.append(line)
else:
self.init_sqls.append(line)
try:
for line in self.init_sqls:
mcu.execute(line, {})
finally:
mcu.close()
utils.run_vtctl('CreateKeyspace -force /zk/global/vt/keyspaces/test_keyspace')
self.tablet.init_tablet('master', 'test_keyspace', '0')
customrules = '/tmp/customrules.json'
self.create_customrules(customrules)
schema_override = '/tmp/schema_override.json'
self.create_schema_override(schema_override)
if utils.options.memcache:
self.tablet.start_vttablet(memcache=True, customrules=customrules, schema_override=schema_override)
else:
self.tablet.start_vttablet(customrules=customrules, schema_override=schema_override)
# FIXME(szopa): This is necessary here only because of a bug that
# makes the qs reload its config only after an action.
utils.run_vtctl('Ping ' + self.tablet.zk_tablet_path)
for i in range(30):
try:
self.conn = self.connect()
self.txlogger = subprocess.Popen(['curl', '-s', '-N', 'http://localhost:9461/debug/txlog'], stdout=open('/tmp/vtocc_txlog.log', 'w'))
self.txlog = framework.Tailer(open('/tmp/vtocc_txlog.log'), flush=self.tablet.flush)
self.log = framework.Tailer(open(os.path.join(self.tablet.tablet_dir, 'vttablet.INFO')), flush=self.tablet.flush)
querylog_file = '/tmp/vtocc_streamlog_%s.log' % self.tablet.port
utils.run_bg(['curl', '-s', '-N', 'http://localhost:9461/debug/querylog?full=true'], stdout=open(querylog_file, 'w'))
time.sleep(1)
self.querylog = framework.Tailer(open(querylog_file), sleep=0.1)
return
except dbexceptions.OperationalError:
if i == 29:
raise
time.sleep(1)
def setUp(self):
if self.vttop is None:
raise EnvironmentError("VTTOP not defined")
if self.vtroot is None:
raise EnvironmentError("VTROOT not defined")
framework.execute("go build", verbose=utils.options.verbose, cwd=self.vttop + "/go/cmd/vtocc")
framework.execute("go build", verbose=utils.options.verbose, cwd=self.vttop + "/go/cmd/mysqlctl")
# start mysql
res = subprocess.call(
[
self.vtroot + "/bin/mysqlctl",
"-tablet-uid",
self.tabletuid,
"-port",
str(self.vtoccport),
"-mysql-port",
str(self.mysqlport),
"init",
]
)
if res != 0:
raise EnvironmentError("Cannot start mysql")
res = subprocess.call(
[
"mysql",
"-S",
self.mysqldir + "/mysql.sock",
"-u",
"vt_dba",
"-e",
"create database vt_test_keyspace ; set global read_only = off",
]
)
if res != 0:
raise Exception("Cannot create vt_test_keyspace database")
dbconfig = self.mysqldir + "/dbconf.json"
with open(dbconfig, "w") as f:
conf = {
"charset": "utf8",
"dbname": "vt_test_keyspace",
"host": "localhost",
"unix_socket": self.mysqldir + "/mysql.sock",
"uname": "vt_dba", # use vt_dba as some tests depend on 'drop'
"keyspace": "test_keyspace",
"shard": "0",
}
json.dump(conf, f)
self.mysql_conn = self.mysql_connect()
mcu = self.mysql_conn.cursor()
self.clean_sqls = []
self.init_sqls = []
clean_mode = False
with open(os.path.join(self.vttop, "test", "test_data", "test_schema.sql")) as f:
for line in f:
line = line.rstrip()
if line == "# clean":
clean_mode = True
if line == "" or line.startswith("#"):
continue
if clean_mode:
self.clean_sqls.append(line)
else:
self.init_sqls.append(line)
try:
for line in self.init_sqls:
mcu.execute(line, {})
finally:
mcu.close()
customrules = "/tmp/customrules.json"
self.create_customrules(customrules)
schema_override = "/tmp/schema_override.json"
self.create_schema_override(schema_override)
occ_args = [
self.vtroot + "/bin/vtocc",
"-port",
"9461",
"-dbconfig",
dbconfig,
"-logfile",
self.logfile,
"-querylog",
self.querylogfile,
"-customrules",
customrules,
"-schema-override",
schema_override,
]
if utils.options.memcache:
memcache = self.mysqldir + "/memcache.sock"
config = self.mysqldir + "/config.json"
with open(config, "w") as f:
json.dump({"RowCache": ["memcached", "-s", memcache]}, f)
occ_args.extend(["-config", config])
self.vtstderr = open("/tmp/vtocc_stderr.log", "a+")
self.vtocc = subprocess.Popen(occ_args, stderr=self.vtstderr)
for i in range(30):
try:
self.conn = self.connect()
self.querylog = framework.Tailer(open(self.querylogfile, "r"))
self.log = framework.Tailer(open(self.logfile, "r"))
self.txlogger = subprocess.Popen(
["curl", "-s", "-N", "http://localhost:9461/debug/txlog"], stdout=open("/tmp/vtocc_txlog.log", "w")
)
self.txlog = framework.Tailer(open("/tmp/vtocc_txlog.log", "r"))
return
except dbexceptions.OperationalError:
if i == 29:
raise
time.sleep(1)
def setUp(self):
utils.zk_setup()
if self.vttop is None:
raise EnvironmentError("VTTOP not defined")
if self.vtroot is None:
raise EnvironmentError("VTROOT not defined")
framework.execute("go build", verbose=utils.options.verbose, cwd=self.vttop + "/go/cmd/mysqlctl")
utils.wait_procs([self.tablet.init_mysql()])
self.tablet.mquery("", ["create database vt_test_keyspace", "set global read_only = off"])
self.mysql_conn, mcu = self.tablet.connect("vt_test_keyspace")
self.clean_sqls = []
self.init_sqls = []
clean_mode = False
with open(os.path.join(self.vttop, "test", "test_data", "test_schema.sql")) as f:
for line in f:
line = line.rstrip()
if line == "# clean":
clean_mode = True
if line == "" or line.startswith("#"):
continue
if clean_mode:
self.clean_sqls.append(line)
else:
self.init_sqls.append(line)
try:
for line in self.init_sqls:
mcu.execute(line, {})
finally:
mcu.close()
utils.run_vtctl("CreateKeyspace -force /zk/global/vt/keyspaces/test_keyspace")
self.tablet.init_tablet("master", "test_keyspace", "0")
customrules = "/tmp/customrules.json"
self.create_customrules(customrules)
schema_override = "/tmp/schema_override.json"
self.create_schema_override(schema_override)
if utils.options.memcache:
self.tablet.start_vttablet(memcache=True, customrules=customrules, schema_override=schema_override)
else:
self.tablet.start_vttablet(customrules=customrules, schema_override=schema_override)
# FIXME(szopa): This is necessary here only because of a bug that
# makes the qs reload its config only after an action.
utils.run_vtctl("Ping " + self.tablet.zk_tablet_path)
for i in range(30):
try:
self.conn = self.connect()
self.querylog = framework.Tailer(open(self.tablet.querylog_file, "r"))
self.log = framework.Tailer(open(self.tablet.logfile, "r"))
self.txlogger = subprocess.Popen(
["curl", "-s", "-N", "http://localhost:9461/debug/txlog"], stdout=open("/tmp/vtocc_txlog.log", "w")
)
self.txlog = framework.Tailer(open("/tmp/vtocc_txlog.log", "r"))
return
except dbexceptions.OperationalError:
if i == 29:
raise
time.sleep(1)
