def main(): print "Content-Type: application/json\n" try: conn = vtgatev2.connect(["localhost:12345"], 10.0) cursor = conn.cursor(None, "master", writable=True) args = cgi.FieldStorage() query = args.getvalue("query") response = {} try: queries = [] stats = capture_log(12345, queries) time.sleep(0.25) exec_query(cursor, "result", query, response) finally: stats.terminate() time.sleep(0.25) response["queries"] = queries exec_query( cursor, "user0", "select * from user where keyrange('','\x80')", response) exec_query( cursor, "user1", "select * from user where keyrange('\x80', '')", response) exec_query( cursor, "user_extra0", "select * from user_extra where keyrange('','\x80')", response) exec_query( cursor, "user_extra1", "select * from user_extra where keyrange('\x80', '')", response) exec_query( cursor, "music0", "select * from music where keyrange('','\x80')", response) exec_query( cursor, "music1", "select * from music where keyrange('\x80', '')", response) exec_query( cursor, "music_extra0", "select * from music_extra where keyrange('','\x80')", response) exec_query( cursor, "music_extra1", "select * from music_extra where keyrange('\x80', '')", response) exec_query( cursor, "user_idx", "select * from user_idx", response) exec_query( cursor, "name_user_idx", "select * from name_user_idx", response) exec_query( cursor, "music_user_idx", "select * from music_user_idx", response) print json.dumps(response) except Exception as e: print json.dumps({"error": str(e)})
def main(): print "Content-Type: application/json\n" try: conn = vtgatev2.connect(["localhost:12345"], 10.0) cursor = conn.cursor(None, "master", writable=True) args = cgi.FieldStorage() query = args.getvalue("query") response = {} try: queries = [] stats = capture_log(12345, queries) time.sleep(0.25) exec_query(cursor, "result", query, response) finally: stats.terminate() time.sleep(0.25) response["queries"] = queries exec_query(cursor, "user0", "select * from user where keyrange('','\x80')", response) exec_query(cursor, "user1", "select * from user where keyrange('\x80', '')", response) exec_query(cursor, "user_extra0", "select * from user_extra where keyrange('','\x80')", response) exec_query(cursor, "user_extra1", "select * from user_extra where keyrange('\x80', '')", response) exec_query(cursor, "music0", "select * from music where keyrange('','\x80')", response) exec_query(cursor, "music1", "select * from music where keyrange('\x80', '')", response) exec_query(cursor, "music_extra0", "select * from music_extra where keyrange('','\x80')", response) exec_query(cursor, "music_extra1", "select * from music_extra where keyrange('\x80', '')", response) exec_query(cursor, "user_idx", "select * from user_idx", response) exec_query(cursor, "name_user_idx", "select * from name_user_idx", response) exec_query(cursor, "music_user_idx", "select * from music_user_idx", response) print json.dumps(response) except Exception as e: print json.dumps({"error": str(e)})
from zk import zkocc # Constants and params UNSHARDED = [keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE)] cursorclass = vtgate_cursor.VTGateCursor # Parse args parser = argparse.ArgumentParser() parser.add_argument('--server', dest='server', default='localhost:15001') parser.add_argument('--timeout', dest='timeout', type=float, default='10.0') args = parser.parse_args() vtgate_addrs = {"_vt": [args.server]} # Connect conn = vtgatev2.connect(vtgate_addrs, args.timeout) # Read topology # This is a temporary work-around until the VTGate V2 client is topology-free. topoconn = zkocc.ZkOccConnection(args.server, 'test_cell', args.timeout) topology.read_topology(topoconn) topoconn.close() # Insert something. print('Inserting into master...') cursor = conn.cursor(cursorclass, conn, 'test_keyspace', 'master', keyranges=UNSHARDED, writable=True) cursor.begin() cursor.execute( "INSERT INTO test_table (msg) VALUES (%(msg)s)", {'msg': 'V is for speed'})
'time_created_ns': int(time.time() * 1e9), 'keyspace_id': keyspace_id_int, 'message': value, }) cursor.commit() # Read the list back from master (critical read) because it's # important that the user sees his own addition immediately. cursor.execute( 'SELECT message FROM messages WHERE page=%(page)s ORDER BY time_created_ns', {'page': page}) entries = [row[0] for row in cursor.fetchall()] cursor.close() return json.dumps(entries) @app.route("/env") def env(): return json.dumps(dict(os.environ)) if __name__ == "__main__": timeout = 10 # connect timeout in seconds # Get vtgate service address from Kubernetes environment. addr = '%s:%s' % (os.environ['VTGATE_SERVICE_HOST'], os.environ['VTGATE_SERVICE_PORT']) # Connect to vtgate. conn = vtgatev2.connect({'vt': [addr]}, timeout) app.run(host='0.0.0.0', port=8080, debug=True)
def _vtdb_conn(self): conn = vtgatev2.connect(self.vtgate_addrs['_vt'], 30) return conn
def test_secure(self): zkocc_server = utils.zkocc_start() # start the tablets shard_0_master.start_vttablet(cert=cert_dir + "/vt-server-cert.pem", key=cert_dir + "/vt-server-key.pem") shard_0_slave.start_vttablet(cert=cert_dir + "/vt-server-cert.pem", key=cert_dir + "/vt-server-key.pem", repl_extra_flags={ 'flags': "2048", 'ssl-ca': cert_dir + "/ca-cert.pem", 'ssl-cert': cert_dir + "/client-cert.pem", 'ssl-key': cert_dir + "/client-key.pem", }) # Reparent using SSL for t in [shard_0_master, shard_0_slave]: t.reset_replication() utils.run_vtctl('ReparentShard -force test_keyspace/0 ' + shard_0_master.tablet_alias, auto_log=True) # then get the topology and check it zkocc_client = zkocc.ZkOccConnection("localhost:%u" % environment.zkocc_port_base, "test_nj", 30.0) topology.read_keyspaces(zkocc_client) shard_0_master_addrs = topology.get_host_port_by_name(zkocc_client, "test_keyspace.0.master:_vts") if len(shard_0_master_addrs) != 1: self.fail('topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts", got: %s' % " ".join(["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs])) if shard_0_master_addrs[0][2] != True: self.fail('topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts" is not encrypted') logging.debug("shard 0 master addrs: %s", " ".join(["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs])) # make sure asking for optionally secure connections works too auto_addrs = topology.get_host_port_by_name(zkocc_client, "test_keyspace.0.master:_vtocc", encrypted=True) if auto_addrs != shard_0_master_addrs: self.fail('topology.get_host_port_by_name doesn\'t resolve encrypted addresses properly: %s != %s' % (str(shard_0_master_addrs), str(auto_addrs))) # try to connect with regular client try: conn = tablet3.TabletConnection("%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]), "", "test_keyspace", "0", 10.0) conn.dial() self.fail("No exception raised to secure port") except dbexceptions.FatalError as e: if not e.args[0][0].startswith('Unexpected EOF in handshake to'): self.fail("Unexpected exception: %s" % str(e)) sconn = utils.get_vars(shard_0_master.port)["SecureConnections"] if sconn != 0: self.fail("unexpected conns %s" % sconn) # connect to encrypted port conn = tablet3.TabletConnection("%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]), "", "test_keyspace", "0", 5.0, encrypted=True) conn.dial() (results, rowcount, lastrowid, fields) = conn._execute("select 1 from dual", {}) self.assertEqual(results, [(1,),], 'wrong conn._execute output: %s' % str(results)) sconn = utils.get_vars(shard_0_master.port)["SecureConnections"] if sconn != 1: self.fail("unexpected conns %s" % sconn) saccept = utils.get_vars(shard_0_master.port)["SecureAccepts"] if saccept == 0: self.fail("unexpected accepts %s" % saccept) # trigger a time out on a secure connection, see what exception we get try: conn._execute("select sleep(100) from dual", {}) self.fail("No timeout exception") except dbexceptions.TimeoutError as e: logging.debug("Got the right exception for SSL timeout: %s", str(e)) # start a vtgate to connect to that tablet gate_proc, gate_port, gate_secure_port = utils.vtgate_start( tablet_bson_encrypted=True, cert=cert_dir + "/vt-server-cert.pem", key=cert_dir + "/vt-server-key.pem") # try to connect to vtgate with regular client timeout = 2.0 try: conn = vtgatev2.connect(["localhost:%s" % (gate_secure_port),], timeout) self.fail("No exception raised to VTGate secure port") except dbexceptions.OperationalError as e: exception_type = e.args[2] exception_msg = str(e.args[2][0][0]) self.assertIsInstance(exception_type, dbexceptions.FatalError, "unexpected exception type") if not exception_msg.startswith('Unexpected EOF in handshake to'): self.fail("Unexpected exception message: %s" % exception_msg) sconn = utils.get_vars(gate_port)["SecureConnections"] if sconn != 0: self.fail("unexpected conns %s" % sconn) # connect to vtgate with encrypted port conn = vtgatev2.connect(["localhost:%s" % (gate_secure_port),], timeout, encrypted=True) (results, rowcount, lastrowid, fields) = conn._execute( "select 1 from dual", {}, "test_keyspace", "master", keyranges=[keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE),]) self.assertEqual(rowcount, 1, "want 1, got %d" % (rowcount)) self.assertEqual(len(fields), 1, "want 1, got %d" % (len(fields))) self.assertEqual(results, [(1,),], 'wrong conn._execute output: %s' % str(results)) sconn = utils.get_vars(gate_port)["SecureConnections"] if sconn != 1: self.fail("unexpected conns %s" % sconn) saccept = utils.get_vars(gate_port)["SecureAccepts"] if saccept == 0: self.fail("unexpected accepts %s" % saccept) # trigger a time out on a vtgate secure connection, see what exception we get try: conn._execute("select sleep(4) from dual", {}, "test_keyspace", "master", keyranges=[keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE),]) self.fail("No timeout exception") except dbexceptions.TimeoutError as e: logging.debug("Got the right exception for SSL timeout: %s", str(e)) conn.close() utils.vtgate_kill(gate_proc) # kill everything utils.kill_sub_process(zkocc_server)
}) cursor.commit() # Read the list back from master (critical read) because it's # important that the user sees his own addition immediately. cursor.execute( 'SELECT message FROM messages WHERE page=%(page)s ORDER BY time_created_ns', {'page': page}) entries = [row[0] for row in cursor.fetchall()] cursor.close() return json.dumps(entries) @app.route("/env") def env(): return json.dumps(dict(os.environ)) if __name__ == "__main__": timeout = 10 # connect timeout in seconds # Get vtgate service address from Kubernetes environment. addr = '%s:%s' % (os.environ['VTGATE_SERVICE_HOST'], os.environ['VTGATE_SERVICE_PORT']) # Connect to vtgate. conn = vtgatev2.connect({'vt': [addr]}, timeout) app.run(host='0.0.0.0', port=8080, debug=True)
def test_secure(self): zkocc_server = utils.zkocc_start() # start the tablets shard_0_master.start_vttablet(cert=cert_dir + "/vt-server-cert.pem", key=cert_dir + "/vt-server-key.pem") shard_0_slave.start_vttablet(cert=cert_dir + "/vt-server-cert.pem", key=cert_dir + "/vt-server-key.pem", repl_extra_flags={ 'flags': "2048", 'ssl-ca': cert_dir + "/ca-cert.pem", 'ssl-cert': cert_dir + "/client-cert.pem", 'ssl-key': cert_dir + "/client-key.pem", }) # Reparent using SSL for t in [shard_0_master, shard_0_slave]: t.reset_replication() utils.run_vtctl('ReparentShard -force test_keyspace/0 ' + shard_0_master.tablet_alias, auto_log=True) # then get the topology and check it zkocc_client = zkocc.ZkOccConnection( "localhost:%u" % environment.zkocc_port_base, "test_nj", 30.0) topology.read_keyspaces(zkocc_client) shard_0_master_addrs = topology.get_host_port_by_name( zkocc_client, "test_keyspace.0.master:_vts") if len(shard_0_master_addrs) != 1: self.fail( 'topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts", got: %s' % " ".join([ "%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs ])) if shard_0_master_addrs[0][2] != True: self.fail( 'topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts" is not encrypted' ) logging.debug( "shard 0 master addrs: %s", " ".join([ "%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs ])) # make sure asking for optionally secure connections works too auto_addrs = topology.get_host_port_by_name( zkocc_client, "test_keyspace.0.master:_vtocc", encrypted=True) if auto_addrs != shard_0_master_addrs: self.fail( 'topology.get_host_port_by_name doesn\'t resolve encrypted addresses properly: %s != %s' % (str(shard_0_master_addrs), str(auto_addrs))) # try to connect with regular client try: conn = tablet3.TabletConnection( "%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]), "", "test_keyspace", "0", 10.0) conn.dial() self.fail("No exception raised to secure port") except dbexceptions.FatalError as e: if not e.args[0][0].startswith('Unexpected EOF in handshake to'): self.fail("Unexpected exception: %s" % str(e)) sconn = utils.get_vars(shard_0_master.port)["SecureConnections"] if sconn != 0: self.fail("unexpected conns %s" % sconn) # connect to encrypted port conn = tablet3.TabletConnection( "%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]), "", "test_keyspace", "0", 5.0, encrypted=True) conn.dial() (results, rowcount, lastrowid, fields) = conn._execute("select 1 from dual", {}) self.assertEqual(results, [ (1, ), ], 'wrong conn._execute output: %s' % str(results)) sconn = utils.get_vars(shard_0_master.port)["SecureConnections"] if sconn != 1: self.fail("unexpected conns %s" % sconn) saccept = utils.get_vars(shard_0_master.port)["SecureAccepts"] if saccept == 0: self.fail("unexpected accepts %s" % saccept) # trigger a time out on a secure connection, see what exception we get try: conn._execute("select sleep(100) from dual", {}) self.fail("No timeout exception") except dbexceptions.TimeoutError as e: logging.debug("Got the right exception for SSL timeout: %s", str(e)) # start a vtgate to connect to that tablet gate_proc, gate_port, gate_secure_port = utils.vtgate_start( tablet_bson_encrypted=True, cert=cert_dir + "/vt-server-cert.pem", key=cert_dir + "/vt-server-key.pem") # try to connect to vtgate with regular client timeout = 2.0 try: conn = vtgatev2.connect([ "localhost:%s" % (gate_secure_port), ], timeout) self.fail("No exception raised to VTGate secure port") except dbexceptions.OperationalError as e: exception_type = e.args[2] exception_msg = str(e.args[2][0][0]) self.assertIsInstance(exception_type, dbexceptions.FatalError, "unexpected exception type") if not exception_msg.startswith('Unexpected EOF in handshake to'): self.fail("Unexpected exception message: %s" % exception_msg) sconn = utils.get_vars(gate_port)["SecureConnections"] if sconn != 0: self.fail("unexpected conns %s" % sconn) # connect to vtgate with encrypted port conn = vtgatev2.connect([ "localhost:%s" % (gate_secure_port), ], timeout, encrypted=True) (results, rowcount, lastrowid, fields) = conn._execute( "select 1 from dual", {}, "test_keyspace", "master", keyranges=[ keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE), ]) self.assertEqual(rowcount, 1, "want 1, got %d" % (rowcount)) self.assertEqual(len(fields), 1, "want 1, got %d" % (len(fields))) self.assertEqual(results, [ (1, ), ], 'wrong conn._execute output: %s' % str(results)) sconn = utils.get_vars(gate_port)["SecureConnections"] if sconn != 1: self.fail("unexpected conns %s" % sconn) saccept = utils.get_vars(gate_port)["SecureAccepts"] if saccept == 0: self.fail("unexpected accepts %s" % saccept) # trigger a time out on a vtgate secure connection, see what exception we get try: conn._execute("select sleep(4) from dual", {}, "test_keyspace", "master", keyranges=[ keyrange.KeyRange( keyrange_constants.NON_PARTIAL_KEYRANGE), ]) self.fail("No timeout exception") except dbexceptions.TimeoutError as e: logging.debug("Got the right exception for SSL timeout: %s", str(e)) conn.close() utils.vtgate_kill(gate_proc) # kill everything utils.kill_sub_process(zkocc_server)
def __connect(self): return vtgatev2.connect({'vt': [self.vtgate]}, self.timeout)
from vtdb import topology from zk import zkocc # Constants and params UNSHARDED = [keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE)] # Parse args parser = argparse.ArgumentParser() parser.add_argument('--server', dest='server', default='localhost:15001') parser.add_argument('--timeout', dest='timeout', type=float, default='10.0') args = parser.parse_args() vtgate_addrs = {"vt": [args.server]} # Connect conn = vtgatev2.connect(vtgate_addrs, args.timeout) # Read topology # This is a temporary work-around until the VTGate V2 client is topology-free. topoconn = zkocc.ZkOccConnection(args.server, 'test', args.timeout) topology.read_topology(topoconn) topoconn.close() # Insert something. print('Inserting into master...') cursor = conn.cursor('test_keyspace', 'master', keyranges=UNSHARDED, writable=True) cursor.begin() cursor.execute( "INSERT INTO test_table (msg) VALUES (%(msg)s)", {'msg': 'V is for speed'})