def __init__(self, env):
self.env = env
self.id = str(uuid.uuid4())
self.curl = utils.curl(self.env.url('/debug/querylog'),
background=True,
stdout=open(self.path, 'w'))
self.curl_full = utils.curl(self.env.url('/debug/querylog?full=true'),
background=True,
stdout=open(self.path_full, 'w'))
time.sleep(0.3)
self.tailer = framework.Tailer(open(self.path), sleep=0.02)
self.tailer_full = framework.Tailer(open(self.path_full), sleep=0.02)
def setUp(self):
if self.vttop is None:
raise EnvironmentError("VTTOP not defined")
if self.vtroot is None:
raise EnvironmentError("VTROOT not defined")
try:
os.makedirs(utils.tmp_root)
except OSError:
pass
utils.setup()
framework.execute('go build',
verbose=utils.options.verbose,
cwd=self.vttop + '/go/cmd/vtocc')
framework.execute('go build',
verbose=utils.options.verbose,
cwd=self.vttop + '/go/cmd/mysqlctl')
# start mysql
res = subprocess.call([
self.vtroot + "/bin/mysqlctl", "-tablet-uid", self.tabletuid,
"-port",
str(self.vtoccport), "-mysql-port",
str(self.mysqlport), "init"
])
if res != 0:
raise EnvironmentError("Cannot start mysql")
res = subprocess.call([
"mysql", "-S", self.mysqldir + "/mysql.sock", "-u", "vt_dba", "-e",
"create database vt_test_keyspace ; set global read_only = off"
])
if res != 0:
raise Exception("Cannot create vt_test_keyspace database")
dbconfig = self.mysqldir + "/dbconf.json"
with open(dbconfig, 'w') as f:
conf = {
'charset': 'utf8',
'dbname': 'vt_test_keyspace',
'host': 'localhost',
'unix_socket': self.mysqldir + "/mysql.sock",
'uname': 'vt_dba', # use vt_dba as some tests depend on 'drop'
'keyspace': 'test_keyspace',
'shard': '0',
}
json.dump(conf, f)
self.mysql_conn = self.mysql_connect()
mcu = self.mysql_conn.cursor()
self.clean_sqls = []
self.init_sqls = []
clean_mode = False
with open(
os.path.join(self.vttop, "test", "test_data",
"test_schema.sql")) as f:
for line in f:
line = line.rstrip()
if line == "# clean":
clean_mode = True
if line == '' or line.startswith("#"):
continue
if clean_mode:
self.clean_sqls.append(line)
else:
self.init_sqls.append(line)
try:
for line in self.init_sqls:
mcu.execute(line, {})
finally:
mcu.close()
customrules = '/tmp/customrules.json'
self.create_customrules(customrules)
schema_override = '/tmp/schema_override.json'
self.create_schema_override(schema_override)
occ_args = [
self.vtroot + "/bin/vtocc",
"-port",
"9461",
"-dbconfig",
dbconfig,
"-customrules",
customrules,
"-schema-override",
schema_override,
]
if utils.options.memcache:
memcache = self.mysqldir + "/memcache.sock"
config = self.mysqldir + "/config.json"
with open(config, 'w') as f:
json.dump({"RowCache": ["memcached", "-s", memcache]}, f)
occ_args.extend(["-queryserver-config-file", config])
self.vtstderr = open("/tmp/vtocc_stderr.log", "a+")
self.vtocc = subprocess.Popen(occ_args, stderr=self.vtstderr)
for i in range(30):
try:
self.conn = self.connect()
self.txlogger = subprocess.Popen(
['curl', '-s', '-N', 'http://localhost:9461/debug/txlog'],
stdout=open('/tmp/vtocc_txlog.log', 'w'))
self.txlog = framework.Tailer(open('/tmp/vtocc_txlog.log',
'r'))
def flush():
utils.run([
'curl', '-s', '-N',
'http://localhost:9461/debug/flushlogs'
],
trap_output=True)
self.log = framework.Tailer(open('/tmp/vtocc.INFO'),
flush=flush)
utils.run_bg([
'curl', '-s', '-N',
'http://localhost:9461/debug/querylog?full=true'
],
stdout=open('/tmp/vtocc_streamlog_9461.log', 'w'))
self.querylog = framework.Tailer(
open('/tmp/vtocc_streamlog_9461.log'), sleep=0.1)
return
except dbexceptions.OperationalError:
if i == 29:
raise
time.sleep(1)
def setUp(self):
utils.zk_setup()
utils.setup()
if self.vttop is None:
raise EnvironmentError("VTTOP not defined")
if self.vtroot is None:
raise EnvironmentError("VTROOT not defined")
framework.execute('go build',
verbose=utils.options.verbose,
cwd=self.vttop + '/go/cmd/mysqlctl')
utils.wait_procs([self.tablet.init_mysql()])
self.tablet.mquery(
"",
["create database vt_test_keyspace", "set global read_only = off"])
self.mysql_conn, mcu = self.tablet.connect('vt_test_keyspace')
self.clean_sqls = []
self.init_sqls = []
clean_mode = False
with open(
os.path.join(self.vttop, "test", "test_data",
"test_schema.sql")) as f:
for line in f:
line = line.rstrip()
if line == "# clean":
clean_mode = True
if line == '' or line.startswith("#"):
continue
if clean_mode:
self.clean_sqls.append(line)
else:
self.init_sqls.append(line)
try:
for line in self.init_sqls:
mcu.execute(line, {})
finally:
mcu.close()
utils.run_vtctl(
'CreateKeyspace -force /zk/global/vt/keyspaces/test_keyspace')
self.tablet.init_tablet('master', 'test_keyspace', '0')
customrules = '/tmp/customrules.json'
self.create_customrules(customrules)
schema_override = '/tmp/schema_override.json'
self.create_schema_override(schema_override)
if utils.options.memcache:
self.tablet.start_vttablet(memcache=True,
customrules=customrules,
schema_override=schema_override)
else:
self.tablet.start_vttablet(customrules=customrules,
schema_override=schema_override)
# FIXME(szopa): This is necessary here only because of a bug that
# makes the qs reload its config only after an action.
utils.run_vtctl('Ping ' + self.tablet.zk_tablet_path)
for i in range(30):
try:
self.conn = self.connect()
self.txlogger = subprocess.Popen(
['curl', '-s', '-N', 'http://localhost:9461/debug/txlog'],
stdout=open('/tmp/vtocc_txlog.log', 'w'))
self.txlog = framework.Tailer(open('/tmp/vtocc_txlog.log'),
flush=self.tablet.flush)
self.log = framework.Tailer(open(
os.path.join(self.tablet.tablet_dir, 'vttablet.INFO')),
flush=self.tablet.flush)
querylog_file = '/tmp/vtocc_streamlog_%s.log' % self.tablet.port
utils.run_bg([
'curl', '-s', '-N',
'http://localhost:9461/debug/querylog?full=true'
],
stdout=open(querylog_file, 'w'))
time.sleep(1)
self.querylog = framework.Tailer(open(querylog_file),
sleep=0.1)
return
except dbexceptions.OperationalError:
if i == 29:
raise
time.sleep(1)
def setUp(self):
# start mysql
res = subprocess.call(
environment.binary_args("mysqlctl") + [
"-tablet_uid", self.tabletuid, "-port",
str(self.port), "-mysql_port",
str(self.mysqlport), "init"
])
if res != 0:
raise EnvironmentError("Cannot start mysql")
res = subprocess.call([
environment.mysql_binary_path('mysql'), "-S",
self.mysqldir + "/mysql.sock", "-u", "vt_dba", "-e",
"create database vt_test_keyspace ; set global read_only = off"
])
if res != 0:
raise Exception("Cannot create vt_test_keyspace database")
self.mysql_conn = self.mysql_connect()
mcu = self.mysql_conn.cursor()
self.clean_sqls = []
self.init_sqls = []
clean_mode = False
with open(
os.path.join(environment.vttop, "test", "test_data",
"test_schema.sql")) as f:
for line in f:
line = line.rstrip()
if line == "# clean":
clean_mode = True
if line == '' or line.startswith("#"):
continue
if clean_mode:
self.clean_sqls.append(line)
else:
self.init_sqls.append(line)
try:
for line in self.init_sqls:
mcu.execute(line, {})
finally:
mcu.close()
customrules = os.path.join(environment.tmproot, 'customrules.json')
self.create_customrules(customrules)
schema_override = os.path.join(environment.tmproot,
'schema_override.json')
self.create_schema_override(schema_override)
table_acl_config = os.path.join(environment.vttop, 'test', 'test_data',
'table_acl_config.json')
occ_args = environment.binary_args('vtocc') + [
"-port",
str(self.port),
"-customrules",
customrules,
"-log_dir",
environment.vtlogroot,
"-schema-override",
schema_override,
"-table-acl-config",
table_acl_config,
"-queryserver-config-strict-table-acl",
"-db-config-app-charset",
"utf8",
"-db-config-app-dbname",
"vt_test_keyspace",
"-db-config-app-host",
"localhost",
"-db-config-app-unixsocket",
self.mysqldir + "/mysql.sock",
"-db-config-app-uname",
'vt_dba', # use vt_dba as some tests depend on 'drop'
"-db-config-app-keyspace",
"test_keyspace",
"-db-config-app-shard",
"0",
"-auth-credentials",
os.path.join(environment.vttop, 'test', 'test_data',
'authcredentials_test.json'),
]
if self.memcache:
memcache = self.mysqldir + "/memcache.sock"
occ_args.extend(["-rowcache-bin", environment.memcached_bin()])
occ_args.extend(["-rowcache-socket", memcache])
occ_args.extend(["-enable-rowcache"])
self.vtocc = subprocess.Popen(occ_args,
stdout=utils.devnull,
stderr=utils.devnull)
for i in range(30):
try:
self.conn = self.connect()
self.txlogger = utils.curl(self.url('/debug/txlog'),
background=True,
stdout=open(self.txlog_file, 'w'))
self.txlog = framework.Tailer(open(self.txlog_file, 'r'))
def flush():
utils.curl(self.url(environment.flush_logs_url),
trap_output=True)
self.log = framework.Tailer(open(
os.path.join(environment.vtlogroot, 'vtocc.INFO')),
flush=flush)
break
except (dbexceptions.OperationalError, dbexceptions.RetryError):
if i == 29:
raise
time.sleep(1)
self.postSetup()
def setUp(self):
environment.topo_server_setup()
utils.wait_procs([self.tablet.init_mysql()])
self.tablet.mquery(
"",
["create database vt_test_keyspace", "set global read_only = off"])
self.mysql_conn, mcu = self.tablet.connect('vt_test_keyspace')
self.clean_sqls = []
self.init_sqls = []
clean_mode = False
with open(
os.path.join(self.vttop, "test", "test_data",
"test_schema.sql")) as f:
for line in f:
line = line.rstrip()
if line == "# clean":
clean_mode = True
if line == '' or line.startswith("#"):
continue
if clean_mode:
self.clean_sqls.append(line)
else:
self.init_sqls.append(line)
try:
for line in self.init_sqls:
mcu.execute(line, {})
finally:
mcu.close()
utils.run_vtctl('CreateKeyspace -force test_keyspace')
self.tablet.init_tablet('master', 'test_keyspace', '0')
customrules = os.path.join(environment.tmproot, 'customrules.json')
self.create_customrules(customrules)
schema_override = os.path.join(environment.tmproot,
'schema_override.json')
self.create_schema_override(schema_override)
table_acl_config = os.path.join(environment.vttop, 'test', 'test_data',
'table_acl_config.json')
self.tablet.start_vttablet(
memcache=self.memcache,
customrules=customrules,
schema_override=schema_override,
table_acl_config=table_acl_config,
auth=True,
)
# FIXME(szopa): This is necessary here only because of a bug that
# makes the qs reload its config only after an action.
utils.run_vtctl('Ping ' + self.tablet.tablet_alias)
for i in range(30):
try:
self.conn = self.connect()
self.txlogger = utils.curl(self.url('/debug/txlog'),
background=True,
stdout=open(self.txlog_file, 'w'))
self.txlog = framework.Tailer(open(self.txlog_file),
flush=self.tablet.flush)
self.log = framework.Tailer(open(
os.path.join(environment.vtlogroot, 'vttablet.INFO')),
flush=self.tablet.flush)
break
except dbexceptions.OperationalError:
if i == 29:
raise
time.sleep(1)
self.postSetup()