def main():
print "Content-Type: application/json\n"
try:
conn = vtgatev2.connect(["localhost:12345"], 10.0)
cursor = conn.cursor(None, "master", writable=True)
args = cgi.FieldStorage()
query = args.getvalue("query")
response = {}
try:
queries = []
stats = capture_log(12345, queries)
time.sleep(0.25)
exec_query(cursor, "result", query, response)
finally:
stats.terminate()
time.sleep(0.25)
response["queries"] = queries
exec_query(
cursor, "user0",
"select * from user where keyrange('','\x80')", response)
exec_query(
cursor, "user1",
"select * from user where keyrange('\x80', '')", response)
exec_query(
cursor, "user_extra0",
"select * from user_extra where keyrange('','\x80')", response)
exec_query(
cursor, "user_extra1",
"select * from user_extra where keyrange('\x80', '')", response)
exec_query(
cursor, "music0",
"select * from music where keyrange('','\x80')", response)
exec_query(
cursor, "music1",
"select * from music where keyrange('\x80', '')", response)
exec_query(
cursor, "music_extra0",
"select * from music_extra where keyrange('','\x80')", response)
exec_query(
cursor, "music_extra1",
"select * from music_extra where keyrange('\x80', '')", response)
exec_query(
cursor, "user_idx", "select * from user_idx", response)
exec_query(
cursor, "name_user_idx", "select * from name_user_idx", response)
exec_query(
cursor, "music_user_idx",
"select * from music_user_idx", response)
print json.dumps(response)
except Exception as e:
print json.dumps({"error": str(e)})
def main():
print "Content-Type: application/json\n"
try:
conn = vtgatev2.connect(["localhost:12345"], 10.0)
cursor = conn.cursor(None, "master", writable=True)
args = cgi.FieldStorage()
query = args.getvalue("query")
response = {}
try:
queries = []
stats = capture_log(12345, queries)
time.sleep(0.25)
exec_query(cursor, "result", query, response)
finally:
stats.terminate()
time.sleep(0.25)
response["queries"] = queries
exec_query(cursor, "user0",
"select * from user where keyrange('','\x80')", response)
exec_query(cursor, "user1",
"select * from user where keyrange('\x80', '')", response)
exec_query(cursor, "user_extra0",
"select * from user_extra where keyrange('','\x80')",
response)
exec_query(cursor, "user_extra1",
"select * from user_extra where keyrange('\x80', '')",
response)
exec_query(cursor, "music0",
"select * from music where keyrange('','\x80')", response)
exec_query(cursor, "music1",
"select * from music where keyrange('\x80', '')", response)
exec_query(cursor, "music_extra0",
"select * from music_extra where keyrange('','\x80')",
response)
exec_query(cursor, "music_extra1",
"select * from music_extra where keyrange('\x80', '')",
response)
exec_query(cursor, "user_idx", "select * from user_idx", response)
exec_query(cursor, "name_user_idx", "select * from name_user_idx",
response)
exec_query(cursor, "music_user_idx", "select * from music_user_idx",
response)
print json.dumps(response)
except Exception as e:
print json.dumps({"error": str(e)})
from zk import zkocc
# Constants and params
UNSHARDED = [keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE)]
cursorclass = vtgate_cursor.VTGateCursor
# Parse args
parser = argparse.ArgumentParser()
parser.add_argument('--server', dest='server', default='localhost:15001')
parser.add_argument('--timeout', dest='timeout', type=float, default='10.0')
args = parser.parse_args()
vtgate_addrs = {"_vt": [args.server]}
# Connect
conn = vtgatev2.connect(vtgate_addrs, args.timeout)
# Read topology
# This is a temporary work-around until the VTGate V2 client is topology-free.
topoconn = zkocc.ZkOccConnection(args.server, 'test_cell', args.timeout)
topology.read_topology(topoconn)
topoconn.close()
# Insert something.
print('Inserting into master...')
cursor = conn.cursor(cursorclass, conn, 'test_keyspace', 'master',
keyranges=UNSHARDED, writable=True)
cursor.begin()
cursor.execute(
"INSERT INTO test_table (msg) VALUES (%(msg)s)",
{'msg': 'V is for speed'})
'time_created_ns': int(time.time() * 1e9),
'keyspace_id': keyspace_id_int,
'message': value,
})
cursor.commit()
# Read the list back from master (critical read) because it's
# important that the user sees his own addition immediately.
cursor.execute(
'SELECT message FROM messages WHERE page=%(page)s ORDER BY time_created_ns',
{'page': page})
entries = [row[0] for row in cursor.fetchall()]
cursor.close()
return json.dumps(entries)
@app.route("/env")
def env():
return json.dumps(dict(os.environ))
if __name__ == "__main__":
timeout = 10 # connect timeout in seconds
# Get vtgate service address from Kubernetes environment.
addr = '%s:%s' % (os.environ['VTGATE_SERVICE_HOST'], os.environ['VTGATE_SERVICE_PORT'])
# Connect to vtgate.
conn = vtgatev2.connect({'vt': [addr]}, timeout)
app.run(host='0.0.0.0', port=8080, debug=True)
def _vtdb_conn(self): conn = vtgatev2.connect(self.vtgate_addrs['_vt'], 30) return conn
def test_secure(self):
zkocc_server = utils.zkocc_start()
# start the tablets
shard_0_master.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
shard_0_slave.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem",
repl_extra_flags={
'flags': "2048",
'ssl-ca': cert_dir + "/ca-cert.pem",
'ssl-cert': cert_dir + "/client-cert.pem",
'ssl-key': cert_dir + "/client-key.pem",
})
# Reparent using SSL
for t in [shard_0_master, shard_0_slave]:
t.reset_replication()
utils.run_vtctl('ReparentShard -force test_keyspace/0 ' + shard_0_master.tablet_alias, auto_log=True)
# then get the topology and check it
zkocc_client = zkocc.ZkOccConnection("localhost:%u" % environment.zkocc_port_base,
"test_nj", 30.0)
topology.read_keyspaces(zkocc_client)
shard_0_master_addrs = topology.get_host_port_by_name(zkocc_client, "test_keyspace.0.master:_vts")
if len(shard_0_master_addrs) != 1:
self.fail('topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts", got: %s' % " ".join(["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs]))
if shard_0_master_addrs[0][2] != True:
self.fail('topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts" is not encrypted')
logging.debug("shard 0 master addrs: %s", " ".join(["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs]))
# make sure asking for optionally secure connections works too
auto_addrs = topology.get_host_port_by_name(zkocc_client, "test_keyspace.0.master:_vtocc", encrypted=True)
if auto_addrs != shard_0_master_addrs:
self.fail('topology.get_host_port_by_name doesn\'t resolve encrypted addresses properly: %s != %s' % (str(shard_0_master_addrs), str(auto_addrs)))
# try to connect with regular client
try:
conn = tablet3.TabletConnection("%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]),
"", "test_keyspace", "0", 10.0)
conn.dial()
self.fail("No exception raised to secure port")
except dbexceptions.FatalError as e:
if not e.args[0][0].startswith('Unexpected EOF in handshake to'):
self.fail("Unexpected exception: %s" % str(e))
sconn = utils.get_vars(shard_0_master.port)["SecureConnections"]
if sconn != 0:
self.fail("unexpected conns %s" % sconn)
# connect to encrypted port
conn = tablet3.TabletConnection("%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]),
"", "test_keyspace", "0", 5.0, encrypted=True)
conn.dial()
(results, rowcount, lastrowid, fields) = conn._execute("select 1 from dual", {})
self.assertEqual(results, [(1,),], 'wrong conn._execute output: %s' % str(results))
sconn = utils.get_vars(shard_0_master.port)["SecureConnections"]
if sconn != 1:
self.fail("unexpected conns %s" % sconn)
saccept = utils.get_vars(shard_0_master.port)["SecureAccepts"]
if saccept == 0:
self.fail("unexpected accepts %s" % saccept)
# trigger a time out on a secure connection, see what exception we get
try:
conn._execute("select sleep(100) from dual", {})
self.fail("No timeout exception")
except dbexceptions.TimeoutError as e:
logging.debug("Got the right exception for SSL timeout: %s", str(e))
# start a vtgate to connect to that tablet
gate_proc, gate_port, gate_secure_port = utils.vtgate_start(
tablet_bson_encrypted=True,
cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
# try to connect to vtgate with regular client
timeout = 2.0
try:
conn = vtgatev2.connect(["localhost:%s" % (gate_secure_port),],
timeout)
self.fail("No exception raised to VTGate secure port")
except dbexceptions.OperationalError as e:
exception_type = e.args[2]
exception_msg = str(e.args[2][0][0])
self.assertIsInstance(exception_type, dbexceptions.FatalError,
"unexpected exception type")
if not exception_msg.startswith('Unexpected EOF in handshake to'):
self.fail("Unexpected exception message: %s" % exception_msg)
sconn = utils.get_vars(gate_port)["SecureConnections"]
if sconn != 0:
self.fail("unexpected conns %s" % sconn)
# connect to vtgate with encrypted port
conn = vtgatev2.connect(["localhost:%s" % (gate_secure_port),],
timeout, encrypted=True)
(results, rowcount, lastrowid, fields) = conn._execute(
"select 1 from dual",
{},
"test_keyspace",
"master",
keyranges=[keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE),])
self.assertEqual(rowcount, 1, "want 1, got %d" % (rowcount))
self.assertEqual(len(fields), 1, "want 1, got %d" % (len(fields)))
self.assertEqual(results, [(1,),], 'wrong conn._execute output: %s' % str(results))
sconn = utils.get_vars(gate_port)["SecureConnections"]
if sconn != 1:
self.fail("unexpected conns %s" % sconn)
saccept = utils.get_vars(gate_port)["SecureAccepts"]
if saccept == 0:
self.fail("unexpected accepts %s" % saccept)
# trigger a time out on a vtgate secure connection, see what exception we get
try:
conn._execute("select sleep(4) from dual",
{},
"test_keyspace",
"master",
keyranges=[keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE),])
self.fail("No timeout exception")
except dbexceptions.TimeoutError as e:
logging.debug("Got the right exception for SSL timeout: %s", str(e))
conn.close()
utils.vtgate_kill(gate_proc)
# kill everything
utils.kill_sub_process(zkocc_server)
})
cursor.commit()
# Read the list back from master (critical read) because it's
# important that the user sees his own addition immediately.
cursor.execute(
'SELECT message FROM messages WHERE page=%(page)s ORDER BY time_created_ns',
{'page': page})
entries = [row[0] for row in cursor.fetchall()]
cursor.close()
return json.dumps(entries)
@app.route("/env")
def env():
return json.dumps(dict(os.environ))
if __name__ == "__main__":
timeout = 10 # connect timeout in seconds
# Get vtgate service address from Kubernetes environment.
addr = '%s:%s' % (os.environ['VTGATE_SERVICE_HOST'],
os.environ['VTGATE_SERVICE_PORT'])
# Connect to vtgate.
conn = vtgatev2.connect({'vt': [addr]}, timeout)
app.run(host='0.0.0.0', port=8080, debug=True)
def test_secure(self):
zkocc_server = utils.zkocc_start()
# start the tablets
shard_0_master.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
shard_0_slave.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem",
repl_extra_flags={
'flags': "2048",
'ssl-ca': cert_dir + "/ca-cert.pem",
'ssl-cert':
cert_dir + "/client-cert.pem",
'ssl-key':
cert_dir + "/client-key.pem",
})
# Reparent using SSL
for t in [shard_0_master, shard_0_slave]:
t.reset_replication()
utils.run_vtctl('ReparentShard -force test_keyspace/0 ' +
shard_0_master.tablet_alias,
auto_log=True)
# then get the topology and check it
zkocc_client = zkocc.ZkOccConnection(
"localhost:%u" % environment.zkocc_port_base, "test_nj", 30.0)
topology.read_keyspaces(zkocc_client)
shard_0_master_addrs = topology.get_host_port_by_name(
zkocc_client, "test_keyspace.0.master:_vts")
if len(shard_0_master_addrs) != 1:
self.fail(
'topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts", got: %s'
% " ".join([
"%s:%u(%s)" % (h, p, str(e))
for (h, p, e) in shard_0_master_addrs
]))
if shard_0_master_addrs[0][2] != True:
self.fail(
'topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts" is not encrypted'
)
logging.debug(
"shard 0 master addrs: %s", " ".join([
"%s:%u(%s)" % (h, p, str(e))
for (h, p, e) in shard_0_master_addrs
]))
# make sure asking for optionally secure connections works too
auto_addrs = topology.get_host_port_by_name(
zkocc_client, "test_keyspace.0.master:_vtocc", encrypted=True)
if auto_addrs != shard_0_master_addrs:
self.fail(
'topology.get_host_port_by_name doesn\'t resolve encrypted addresses properly: %s != %s'
% (str(shard_0_master_addrs), str(auto_addrs)))
# try to connect with regular client
try:
conn = tablet3.TabletConnection(
"%s:%u" %
(shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]), "",
"test_keyspace", "0", 10.0)
conn.dial()
self.fail("No exception raised to secure port")
except dbexceptions.FatalError as e:
if not e.args[0][0].startswith('Unexpected EOF in handshake to'):
self.fail("Unexpected exception: %s" % str(e))
sconn = utils.get_vars(shard_0_master.port)["SecureConnections"]
if sconn != 0:
self.fail("unexpected conns %s" % sconn)
# connect to encrypted port
conn = tablet3.TabletConnection(
"%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]),
"",
"test_keyspace",
"0",
5.0,
encrypted=True)
conn.dial()
(results, rowcount, lastrowid,
fields) = conn._execute("select 1 from dual", {})
self.assertEqual(results, [
(1, ),
], 'wrong conn._execute output: %s' % str(results))
sconn = utils.get_vars(shard_0_master.port)["SecureConnections"]
if sconn != 1:
self.fail("unexpected conns %s" % sconn)
saccept = utils.get_vars(shard_0_master.port)["SecureAccepts"]
if saccept == 0:
self.fail("unexpected accepts %s" % saccept)
# trigger a time out on a secure connection, see what exception we get
try:
conn._execute("select sleep(100) from dual", {})
self.fail("No timeout exception")
except dbexceptions.TimeoutError as e:
logging.debug("Got the right exception for SSL timeout: %s",
str(e))
# start a vtgate to connect to that tablet
gate_proc, gate_port, gate_secure_port = utils.vtgate_start(
tablet_bson_encrypted=True,
cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
# try to connect to vtgate with regular client
timeout = 2.0
try:
conn = vtgatev2.connect([
"localhost:%s" % (gate_secure_port),
], timeout)
self.fail("No exception raised to VTGate secure port")
except dbexceptions.OperationalError as e:
exception_type = e.args[2]
exception_msg = str(e.args[2][0][0])
self.assertIsInstance(exception_type, dbexceptions.FatalError,
"unexpected exception type")
if not exception_msg.startswith('Unexpected EOF in handshake to'):
self.fail("Unexpected exception message: %s" % exception_msg)
sconn = utils.get_vars(gate_port)["SecureConnections"]
if sconn != 0:
self.fail("unexpected conns %s" % sconn)
# connect to vtgate with encrypted port
conn = vtgatev2.connect([
"localhost:%s" % (gate_secure_port),
],
timeout,
encrypted=True)
(results, rowcount, lastrowid, fields) = conn._execute(
"select 1 from dual", {},
"test_keyspace",
"master",
keyranges=[
keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE),
])
self.assertEqual(rowcount, 1, "want 1, got %d" % (rowcount))
self.assertEqual(len(fields), 1, "want 1, got %d" % (len(fields)))
self.assertEqual(results, [
(1, ),
], 'wrong conn._execute output: %s' % str(results))
sconn = utils.get_vars(gate_port)["SecureConnections"]
if sconn != 1:
self.fail("unexpected conns %s" % sconn)
saccept = utils.get_vars(gate_port)["SecureAccepts"]
if saccept == 0:
self.fail("unexpected accepts %s" % saccept)
# trigger a time out on a vtgate secure connection, see what exception we get
try:
conn._execute("select sleep(4) from dual", {},
"test_keyspace",
"master",
keyranges=[
keyrange.KeyRange(
keyrange_constants.NON_PARTIAL_KEYRANGE),
])
self.fail("No timeout exception")
except dbexceptions.TimeoutError as e:
logging.debug("Got the right exception for SSL timeout: %s",
str(e))
conn.close()
utils.vtgate_kill(gate_proc)
# kill everything
utils.kill_sub_process(zkocc_server)
def __connect(self):
return vtgatev2.connect({'vt': [self.vtgate]}, self.timeout)
from vtdb import topology
from zk import zkocc
# Constants and params
UNSHARDED = [keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE)]
# Parse args
parser = argparse.ArgumentParser()
parser.add_argument('--server', dest='server', default='localhost:15001')
parser.add_argument('--timeout', dest='timeout', type=float, default='10.0')
args = parser.parse_args()
vtgate_addrs = {"vt": [args.server]}
# Connect
conn = vtgatev2.connect(vtgate_addrs, args.timeout)
# Read topology
# This is a temporary work-around until the VTGate V2 client is topology-free.
topoconn = zkocc.ZkOccConnection(args.server, 'test', args.timeout)
topology.read_topology(topoconn)
topoconn.close()
# Insert something.
print('Inserting into master...')
cursor = conn.cursor('test_keyspace', 'master',
keyranges=UNSHARDED, writable=True)
cursor.begin()
cursor.execute(
"INSERT INTO test_table (msg) VALUES (%(msg)s)",
{'msg': 'V is for speed'})
def _vtdb_conn(self):
conn = vtgatev2.connect(self.vtgate_addrs['_vt'], 30)
return conn
