def read_topology(zkocc_client, read_fqdb_keys=True):
fqdb_keys = []
db_keys = []
keyspace_list = zkocc_client.get_srv_keyspace_names('local')
# validate step
if len(keyspace_list) == 0:
vtdb_logger.get_logger().topo_empty_keyspace_list()
raise Exception('zkocc returned empty keyspace list')
for keyspace_name in keyspace_list:
try:
ks = keyspace.read_keyspace(zkocc_client, keyspace_name)
__set_keyspace(ks)
for db_type, partition in ks.partitions.iteritems():
for shard_reference in partition['ShardReferences']:
db_key_parts = [ks.name, shard_reference['Name'], db_type]
db_key = '.'.join(db_key_parts)
db_keys.append(db_key)
if read_fqdb_keys:
db_instances = len(
get_host_port_by_name(zkocc_client, db_key))
for db_i in xrange(db_instances):
fqdb_keys.append('.'.join(db_key_parts +
[str(db_i)]))
except Exception:
vtdb_logger.get_logger().topo_bad_keyspace_data(keyspace_name)
return db_keys, fqdb_keys
def connect(self):
try:
return self._connect()
except dbexceptions.OperationalError as e:
vtdb_logger.get_logger().vtclient_exception(
self.keyspace, self.shard, self.db_type, e)
raise
def _run_with_reconnect(self, *args, **kargs):
attempt = 0
while True:
try:
return method(self, *args, **kargs)
except (dbexceptions.RetryError, dbexceptions.FatalError, dbexceptions.TxPoolFull) as e:
attempt += 1
# Execution attempt failed with OperationalError, re-read the keyspace.
if not isinstance(e, dbexceptions.TxPoolFull):
self.resolve_topology()
if attempt >= self.max_attempts or self.in_txn:
self.close()
vtdb_logger.get_logger().vtclient_exception(self.keyspace, self.shard, self.db_type, e)
raise dbexceptions.FatalError(*e.args)
if method.__name__ == 'begin':
time.sleep(BEGIN_RECONNECT_DELAY)
else:
time.sleep(RECONNECT_DELAY)
if not isinstance(e, dbexceptions.TxPoolFull):
logging.info("Attempting to reconnect, %d", attempt)
self.close()
self.connect()
logging.info("Successfully reconnected to %s", str(self.conn))
else:
logging.info("Waiting to retry for dbexceptions.TxPoolFull to %s, attempt %d", str(self.conn), attempt)
def read_topology(zkocc_client, read_fqdb_keys=True):
fqdb_keys = []
db_keys = []
keyspace_list = zkocc_client.get_srv_keyspace_names('local')
# validate step
if len(keyspace_list) == 0:
vtdb_logger.get_logger().topo_empty_keyspace_list()
raise Exception('zkocc returned empty keyspace list')
for keyspace_name in keyspace_list:
try:
ks = keyspace.read_keyspace(zkocc_client, keyspace_name)
__set_keyspace(ks)
for db_type in ks.db_types:
for shard_name in ks.get_shard_names(db_type):
db_key_parts = [ks.name, shard_name, db_type]
db_key = '.'.join(db_key_parts)
db_keys.append(db_key)
if read_fqdb_keys:
db_instances = len(get_host_port_by_name(zkocc_client, db_key))
for db_i in xrange(db_instances):
fqdb_keys.append('.'.join(db_key_parts + [str(db_i)]))
except Exception:
vtdb_logger.get_logger().topo_bad_keyspace_data(keyspace_name)
return db_keys, fqdb_keys
def _run_with_reconnect(self, *args, **kargs):
attempt = 0
while True:
try:
return method(self, *args, **kargs)
except (dbexceptions.RetryError, dbexceptions.FatalError,
dbexceptions.TxPoolFull) as e:
attempt += 1
# Execution attempt failed with OperationalError, re-read the keyspace.
if not isinstance(e, dbexceptions.TxPoolFull):
self.resolve_topology()
if attempt >= self.max_attempts or self.in_txn:
self.close()
vtdb_logger.get_logger().vtclient_exception(
self.keyspace, self.shard, self.db_type, e)
raise dbexceptions.FatalError(*e.args)
if method.__name__ == 'begin':
time.sleep(BEGIN_RECONNECT_DELAY)
else:
time.sleep(RECONNECT_DELAY)
if not isinstance(e, dbexceptions.TxPoolFull):
logging.info("Attempting to reconnect, %d", attempt)
self.close()
self.connect()
logging.info("Successfully reconnected to %s",
str(self.conn))
else:
logging.info(
"Waiting to retry for dbexceptions.TxPoolFull to %s, attempt %d",
str(self.conn), attempt)
def connect(self):
try:
return self._connect()
except dbexceptions.OperationalError as e:
vtdb_logger.get_logger().vtclient_exception(
self.keyspace, self.shard, self.db_type, e)
raise
def test_topo_read_threshold(self):
before_topo_rtt = vtdb_logger.get_logger().get_topo_rtt()
# Check original state.
keyspace_obj = topology.get_keyspace('test_keyspace')
self.assertNotEqual(keyspace_obj, None, "test_keyspace should be not None")
self.assertEqual(keyspace_obj.sharding_col_type, keyrange_constants.KIT_UINT64, "ShardingColumnType be %s" % keyrange_constants.KIT_UINT64)
# Change the keyspace object.
utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', 'test_keyspace',
'keyspace_id', keyrange_constants.KIT_BYTES])
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
# sleep throttle interval and check values again.
# the keyspace should have changed and also caused a rtt to topo server.
time.sleep(self.keyspace_fetch_throttle)
topology.refresh_keyspace(self.vtgate_client, 'test_keyspace')
keyspace_obj = topology.get_keyspace('test_keyspace')
after_1st_clear = vtdb_logger.get_logger().get_topo_rtt()
self.assertEqual(after_1st_clear - before_topo_rtt, 1, "One additional round-trips to topo server")
self.assertEqual(keyspace_obj.sharding_col_type, keyrange_constants.KIT_BYTES, "ShardingColumnType be %s" % keyrange_constants.KIT_BYTES)
# Refresh without sleeping for throttle time shouldn't cause additional rtt.
topology.refresh_keyspace(self.vtgate_client, 'test_keyspace')
keyspace_obj = topology.get_keyspace('test_keyspace')
after_2nd_clear = vtdb_logger.get_logger().get_topo_rtt()
self.assertEqual(after_2nd_clear - after_1st_clear, 0, "No additional round-trips to topo server")
def test_topo_read_threshold(self):
before_topo_rtt = vtdb_logger.get_logger().get_topo_rtt()
# Check original state.
keyspace_obj = topology.get_keyspace('test_keyspace')
self.assertNotEqual(keyspace_obj, None, "test_keyspace should be not None")
self.assertEqual(keyspace_obj.sharding_col_type, keyrange_constants.KIT_UINT64, "ShardingColumnType be %s" % keyrange_constants.KIT_UINT64)
# Change the keyspace object.
utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', 'test_keyspace',
'keyspace_id', keyrange_constants.KIT_BYTES])
utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
# sleep throttle interval and check values again.
# the keyspace should have changed and also caused a rtt to topo server.
time.sleep(self.keyspace_fetch_throttle)
topology.refresh_keyspace(self.vtgate_client, 'test_keyspace')
keyspace_obj = topology.get_keyspace('test_keyspace')
after_1st_clear = vtdb_logger.get_logger().get_topo_rtt()
self.assertEqual(after_1st_clear - before_topo_rtt, 1, "One additional round-trips to topo server")
self.assertEqual(keyspace_obj.sharding_col_type, keyrange_constants.KIT_BYTES, "ShardingColumnType be %s" % keyrange_constants.KIT_BYTES)
# Refresh without sleeping for throttle time shouldn't cause additional rtt.
topology.refresh_keyspace(self.vtgate_client, 'test_keyspace')
keyspace_obj = topology.get_keyspace('test_keyspace')
after_2nd_clear = vtdb_logger.get_logger().get_topo_rtt()
self.assertEqual(after_2nd_clear - after_1st_clear, 0, "No additional round-trips to topo server")
def _execute(self, sql, bind_variables):
sql, bind_variables = dbapi.prepare_query_bind_vars(sql, bind_variables)
try:
result = self.conn._execute(sql, bind_variables)
except dbexceptions.IntegrityError as e:
vtdb_logger.get_logger().integrity_error(e)
raise
return result
def _execute(self, sql, bind_variables):
sql, bind_variables = dbapi.prepare_query_bind_vars(sql, bind_variables)
try:
result = self.conn._execute(sql, bind_variables)
except dbexceptions.IntegrityError as e:
vtdb_logger.get_logger().integrity_error(e)
raise
return result
def test_keyspace_reresolve_on_conn_failure(self):
before_topo_rtt = vtdb_logger.get_logger().get_topo_rtt()
self.replica_tablet.kill_vttablet()
time.sleep(self.keyspace_fetch_throttle)
try:
replica_conn = get_connection(db_type='replica',
shard_index=self.shard_index)
except Exception:
pass
after_tablet_conn_error = vtdb_logger.get_logger().get_topo_rtt()
self.assertEqual(after_tablet_conn_error - before_topo_rtt, 1,
'One additional round-trips to topo server')
self.replica_tablet.start_vttablet()
def _execute_batch(self, sql_list, bind_variables_list):
sane_sql_list = []
sane_bind_vars_list = []
for sql, bind_variables in zip(sql_list, bind_variables_list):
sane_sql, sane_bind_vars = dbapi.prepare_query_bind_vars(sql, bind_variables)
sane_sql_list.append(sane_sql)
sane_bind_vars_list.append(sane_bind_vars)
try:
result = self.conn._execute_batch(sane_sql_list, sane_bind_vars_list)
except dbexceptions.IntegrityError as e:
vtdb_logger.get_logger().integrity_error(e)
raise
return result
def test_keyspace_reresolve_on_conn_failure(self):
before_topo_rtt = vtdb_logger.get_logger().get_topo_rtt()
self.replica_tablet.kill_vttablet()
time.sleep(self.keyspace_fetch_throttle)
try:
replica_conn = get_connection(
db_type='replica', shard_index=self.shard_index)
except Exception:
pass
after_tablet_conn_error = vtdb_logger.get_logger().get_topo_rtt()
self.assertEqual(
after_tablet_conn_error - before_topo_rtt, 1,
'One additional round-trips to topo server')
self.replica_tablet.start_vttablet()
def _execute_batch(self, sql_list, bind_variables_list):
sane_sql_list = []
sane_bind_vars_list = []
for sql, bind_variables in zip(sql_list, bind_variables_list):
sane_sql, sane_bind_vars = dbapi.prepare_query_bind_vars(sql, bind_variables)
sane_sql_list.append(sane_sql)
sane_bind_vars_list.append(sane_bind_vars)
try:
result = self.conn._execute_batch(sane_sql_list, sane_bind_vars_list)
except dbexceptions.IntegrityError as e:
vtdb_logger.get_logger().integrity_error(e)
raise
return result
def __init__(self,
addr,
tablet_type,
keyspace,
shard,
timeout,
user=None,
password=None,
keyfile=None,
certfile=None,
caller_id=None):
self.transaction_id = 0
self.session_id = 0
self.addr = addr
self.caller_id = caller_id
self.certfile = certfile
self.keyfile = keyfile
self.keyspace = keyspace
self.password = password
self.shard = shard
self.tablet_type = tablet_type
self.timeout = timeout
self.user = user
self.client = self._create_client()
self.logger_object = vtdb_logger.get_logger()
def setUp(self): self.shard_index = 1 self.keyrange = get_keyrange(shard_names[self.shard_index]) self.master_tablet = shard_1_master self.replica_tablet = shard_1_replica vtdb_logger.register_vtdb_logger(VTGateTestLogger()) self.logger = vtdb_logger.get_logger()
def __init__(self, addr, timeout, user=None, password=None,
keyfile=None, certfile=None):
self.addr = addr
self.timeout = timeout
self.client = bsonrpc.BsonRpcClient(
addr, timeout, user, password, keyfile=keyfile, certfile=certfile)
self.logger_object = vtdb_logger.get_logger()
def setUp(self):
self.shard_index = 1
self.keyrange = get_keyrange(shard_names[self.shard_index])
self.master_tablet = shard_1_master
self.replica_tablet = shard_1_replica
vtdb_logger.register_vtdb_logger(VTGateTestLogger())
self.logger = vtdb_logger.get_logger()
def __init__(self,
addr,
timeout,
root_certificates=None,
private_key=None,
certificate_chain=None,
auth_static_client_creds=None,
**kwargs):
"""Creates a new GRPCVTGateConnection.
Args:
addr: address to connect to.
timeout: connection time out.
root_certificates: PEM_encoded root certificates.
private_key: PEM-encoded private key.
certificate_chain: PEM-encoded certificate chain.
auth_static_client_creds: basic auth credentials file path.
**kwargs: passed up.
"""
super(GRPCVTGateConnection, self).__init__(addr, timeout, **kwargs)
self.stub = None
self.root_certificates = root_certificates
self.private_key = private_key
self.certificate_chain = certificate_chain
self.auth_static_client_creds = auth_static_client_creds
self.logger_object = vtdb_logger.get_logger()
def refresh_keyspace(zkocc_client, name):
global __keyspace_fetch_throttle
time_last_fetch = get_time_last_fetch(name)
if time_last_fetch is None:
return
if (time_last_fetch + __keyspace_fetch_throttle) > time.time():
return
start_time = time.time()
ks = keyspace.read_keyspace(zkocc_client, name)
topo_rtt = time.time() - start_time
if ks is not None:
__set_keyspace(ks)
vtdb_logger.get_logger().topo_keyspace_fetch(name, topo_rtt)
def wrapper(self, *args, **kargs):
attempt = 0
delay = initial_delay_ms
while True:
try:
return method(self, *args, **kargs)
except retry_exceptions as e:
attempt += 1
if attempt > num_retries or self.session:
vtdb_logger.get_logger().vtgatev2_exception(e)
raise e
logging.error(
"retryable error: %s, retrying in %d ms, attempt %d of %d",
e, delay, attempt, num_retries)
time.sleep(delay / 1000.0)
delay *= backoff_multiplier
delay = min(max_delay_ms, delay)
def test_keyspace_reresolve_on_conn_failure(self):
before_topo_rtt = vtdb_logger.get_logger().get_topo_rtt()
self.replica_tablet.kill_vttablet()
time.sleep(self.keyspace_fetch_throttle)
try:
replica_conn = get_connection(db_type='replica', shard_index=self.shard_index)
except Exception, e:
pass
def refresh_keyspace(zkocc_client, name):
global __keyspace_fetch_throttle
time_last_fetch = get_time_last_fetch(name)
if time_last_fetch is None:
return
if (time_last_fetch + __keyspace_fetch_throttle) > time.time():
return
start_time = time.time()
ks = keyspace.read_keyspace(zkocc_client, name)
topo_rtt = time.time() - start_time
if ks is not None:
__set_keyspace(ks)
vtdb_logger.get_logger().topo_keyspace_fetch(name, topo_rtt)
def __init__(self, addr, tablet_type, keyspace, shard, timeout, user=None, password=None, encrypted=False, keyfile=None, certfile=None): self.addr = addr self.tablet_type = tablet_type self.keyspace = keyspace self.shard = shard self.timeout = timeout self.client = bsonrpc.BsonRpcClient(addr, timeout, user, password, encrypted=encrypted, keyfile=keyfile, certfile=certfile) self.logger_object = vtdb_logger.get_logger()
def test_keyspace_reresolve_on_execute(self):
before_topo_rtt = vtdb_logger.get_logger().get_topo_rtt()
try:
replica_conn = get_connection(db_type='replica',
shard_index=self.shard_index)
except Exception, e:
self.fail("Connection to shard %s replica failed with error %s" %
(shard_names[self.shard_index], str(e)))
def test_keyspace_reresolve_on_conn_failure(self):
before_topo_rtt = vtdb_logger.get_logger().get_topo_rtt()
self.replica_tablet.kill_vttablet()
time.sleep(self.keyspace_fetch_throttle)
try:
replica_conn = get_connection(db_type='replica', shard_index=self.shard_index)
except Exception, e:
pass
def __init__(self, addr, timeout, user=None, password=None,
keyfile=None, certfile=None):
super(VTGateConnection, self).__init__(addr, timeout)
self.user = user
self.password = password
self.keyfile = keyfile
self.certfile = certfile
self.client = self._create_client()
self.logger_object = vtdb_logger.get_logger()
def convert_exception(exc, *args):
new_args = exc.args + args
vtdb_logger.get_logger().vtgatev2_exception(exc)
if isinstance(exc, gorpc.TimeoutError):
return dbexceptions.TimeoutError(new_args)
elif isinstance(exc, gorpc.AppError):
msg = str(exc[0]).lower()
match = _errno_pattern.search(msg)
if match:
mysql_errno = int(match.group(1))
if mysql_errno == 1062:
return dbexceptions.IntegrityError(new_args)
return dbexceptions.DatabaseError(new_args)
elif isinstance(exc, gorpc.ProgrammingError):
return dbexceptions.ProgrammingError(new_args)
elif isinstance(exc, gorpc.GoRpcError):
return dbexceptions.FatalError(new_args)
return exc
def convert_exception(exc, *args):
new_args = exc.args + args
vtdb_logger.get_logger().vtgatev2_exception(exc)
if isinstance(exc, gorpc.TimeoutError):
return dbexceptions.TimeoutError(new_args)
elif isinstance(exc, gorpc.AppError):
msg = str(exc[0]).lower()
match = _errno_pattern.search(msg)
if match:
mysql_errno = int(match.group(1))
if mysql_errno == 1062:
return dbexceptions.IntegrityError(new_args)
return dbexceptions.DatabaseError(new_args)
elif isinstance(exc, gorpc.ProgrammingError):
return dbexceptions.ProgrammingError(new_args)
elif isinstance(exc, gorpc.GoRpcError):
return dbexceptions.FatalError(new_args)
return exc
def test_keyspace_reresolve_on_execute(self):
before_topo_rtt = vtdb_logger.get_logger().get_topo_rtt()
try:
replica_conn = get_connection(
db_type='replica', shard_index=self.shard_index)
except Exception, e:
self.fail(
'Connection to shard %s replica failed with error %s' %
(shard_names[self.shard_index], str(e)))
def _log_exception(exc, *args):
logger_object = vtdb_logger.get_logger()
new_exception = method(exc, *args)
if isinstance(new_exception, dbexceptions.IntegrityError):
logger_object.integrity_error(new_exception)
else:
logger_object.vtgatev2_exception(new_exception)
return new_exception
def __init__(self, vtgate_addrs=None, lag_tolerant_mode=False, master_access_disabled=False): self.vtgate_addrs = vtgate_addrs self.lag_tolerant_mode = lag_tolerant_mode self.master_access_disabled = master_access_disabled self.vtgate_connection = None self.change_master_read_to_replica = False self._transaction_stack_depth = 0 self.event_logger = vtdb_logger.get_logger() self.connection_timeout = DEFAULT_CONNECTION_TIMEOUT self._tablet_type = None
def __init__(self, vtgate_addrs=None, lag_tolerant_mode=False, master_access_disabled=False):
self.vtgate_addrs = vtgate_addrs
self.lag_tolerant_mode = lag_tolerant_mode
self.master_access_disabled = master_access_disabled
self.vtgate_connection = None
self.change_master_read_to_replica = False
self._transaction_stack_depth = 0
self.event_logger = vtdb_logger.get_logger()
self.connection_timeout = DEFAULT_CONNECTION_TIMEOUT
self._tablet_type = None
def __init__(self, addr, timeout, user=None, password=None, keyfile=None, certfile=None):
self.session = None
self.addr = addr
self.user = user
self.password = password
self.keyfile = keyfile
self.certfile = certfile
self.timeout = timeout
self.client = self._create_client()
self.logger_object = vtdb_logger.get_logger()
def _log_exception(exc, *args):
logger_object = vtdb_logger.get_logger()
new_exception = method(exc, *args)
if isinstance(new_exception, dbexceptions.IntegrityError):
logger_object.integrity_error(new_exception)
else:
logger_object.vtgatev2_exception(new_exception)
return new_exception
def __init__(self, addr, timeout, user=None, password=None,
keyfile=None, certfile=None):
self.session = None
self.addr = addr
self.user = user
self.password = password
self.keyfile = keyfile
self.certfile = certfile
self.timeout = timeout
self.client = self._create_client()
self.logger_object = vtdb_logger.get_logger()
def __init__(self, addr, timeout, user=None, password=None,
keyfile=None, certfile=None):
# TODO: Merge. This is very similar to vtgatev2.
self.session = None
self.addr = addr
self.user = user
self.password = password
self.keyfile = keyfile
self.certfile = certfile
self.timeout = timeout
self.client = self._create_client()
self.logger_object = vtdb_logger.get_logger()
def __init__(self, addr, timeout, user=None, password=None,
keyfile=None, certfile=None):
# TODO: Merge. This is very similar to vtgatev2.
self.session = None
self.addr = addr
self.user = user
self.password = password
self.keyfile = keyfile
self.certfile = certfile
self.timeout = timeout
self.client = self._create_client()
self.logger_object = vtdb_logger.get_logger()
def get_host_port_by_name(topo_client, db_key):
parts = db_key.split(':')
if len(parts) == 2:
service = parts[1]
else:
service = 'mysql'
db_key = parts[0]
ks, shard, tablet_type = db_key.split('.')
try:
data = topo_client.get_end_points('local', ks, shard, tablet_type)
except zkocc.ZkOccError as e:
vtdb_logger.get_logger().topo_zkocc_error('do data', db_key, e)
return []
except Exception as e:
vtdb_logger.get_logger().topo_exception(
'failed to get or parse topo data', db_key, e)
return []
if 'Entries' not in data:
vtdb_logger.get_logger().topo_exception(
'topo server returned: ' + str(data), db_key, e)
raise Exception('zkocc returned: %s' % str(data))
host_port_list = []
for entry in data['Entries']:
if service in entry['PortMap']:
host_port = (entry['Host'], entry['PortMap'][service])
host_port_list.append(host_port)
random.shuffle(host_port_list)
return host_port_list
def get_host_port_by_name(topo_client, db_key):
parts = db_key.split(':')
if len(parts) == 2:
service = parts[1]
else:
service = 'mysql'
db_key = parts[0]
ks, shard, tablet_type = db_key.split('.')
try:
data = topo_client.get_end_points('local', ks, shard, tablet_type)
except zkocc.ZkOccError as e:
vtdb_logger.get_logger().topo_zkocc_error('do data', db_key, e)
return []
except Exception as e:
vtdb_logger.get_logger().topo_exception(
'failed to get or parse topo data', db_key, e)
return []
if 'Entries' not in data:
vtdb_logger.get_logger().topo_exception(
'topo server returned: ' + str(data), db_key, e)
raise Exception('zkocc returned: %s' % str(data))
host_port_list = []
for entry in data['Entries']:
if service in entry['PortMap']:
host_port = (entry['Host'], entry['PortMap'][service])
host_port_list.append(host_port)
random.shuffle(host_port_list)
return host_port_list
def __init__(self,
addr,
timeout,
user=None,
password=None,
keyfile=None,
certfile=None):
super(VTGateConnection, self).__init__(addr, timeout)
self.user = user
self.password = password
self.keyfile = keyfile
self.certfile = certfile
self.client = self._create_client()
self.logger_object = vtdb_logger.get_logger()
def log_exception(exc, keyspace=None, tablet_type=None):
"""This method logs the exception.
Args:
exc: exception raised by calling code
keyspace: keyspace for the exception
tablet_type: tablet_type for the exception
"""
logger_object = vtdb_logger.get_logger()
shard_name = None
if isinstance(exc, dbexceptions.IntegrityError):
logger_object.integrity_error(exc)
else:
logger_object.vtclient_exception(keyspace, shard_name, tablet_type, exc)
def log_exception(exc, keyspace=None, tablet_type=None):
"""This method logs the exception.
Args:
exc: exception raised by calling code
keyspace: keyspace for the exception
tablet_type: tablet_type for the exception
"""
logger_object = vtdb_logger.get_logger()
shard_name = None
if isinstance(exc, dbexceptions.IntegrityError):
logger_object.integrity_error(exc)
else:
logger_object.vtclient_exception(keyspace, shard_name, tablet_type, exc)
def __init__(self,
addr,
timeout,
user=None,
password=None,
keyfile=None,
certfile=None):
self.addr = addr
self.timeout = timeout
self.client = bsonrpc.BsonRpcClient(addr,
timeout,
user,
password,
keyfile=keyfile,
certfile=certfile)
self.logger_object = vtdb_logger.get_logger()
def get_db_params_for_tablet_conn(topo_client, keyspace_name, shard, db_type, timeout, encrypted, user, password):
db_params_list = []
encrypted_service = 'vts'
if encrypted:
service = encrypted_service
else:
service = 'vt'
db_key = "%s.%s.%s:%s" % (keyspace_name, shard, db_type, service)
# This will read the cached keyspace.
keyspace_object = topology.get_keyspace(keyspace_name)
# Handle vertical split by checking 'ServedFrom' field.
new_keyspace = None
served_from = keyspace_object.served_from
if served_from is not None:
new_keyspace = served_from.get(db_type, None)
if new_keyspace is not None:
keyspace_name = new_keyspace
try:
end_points_data = topo_client.get_end_points('local', keyspace_name, shard, db_type)
except zkocc.ZkOccError as e:
vtdb_logger.get_logger().topo_zkocc_error('do data', db_key, e)
return []
except Exception as e:
vtdb_logger.get_logger().topo_exception('failed to get or parse topo data', db_key, e)
return []
end_points_list = []
host_port_list = []
encrypted_host_port_list = []
if 'Entries' not in end_points_data:
vtdb_logger.get_logger().topo_exception('topo server returned: ' + str(end_points_data), db_key, e)
raise Exception('zkocc returned: %s' % str(end_points_data))
for entry in end_points_data['Entries']:
if service in entry['NamedPortMap']:
host_port = (entry['Host'], entry['NamedPortMap'][service],
service == 'vts')
host_port_list.append(host_port)
if encrypted and encrypted_service in entry['NamedPortMap']:
host_port = (entry['Host'], entry['NamedPortMap'][encrypted_service],
True)
encrypted_host_port_list.append(host_port)
if encrypted and len(encrypted_host_port_list) > 0:
random.shuffle(encrypted_host_port_list)
end_points_list = encrypted_host_port_list
else:
random.shuffle(host_port_list)
end_points_list = host_port_list
for host, port, encrypted in end_points_list:
vt_params = VTConnParams(keyspace_name, shard, db_type, "%s:%s" % (host, port), timeout, encrypted, user, password).__dict__
db_params_list.append(vt_params)
return db_params_list
def __init__(
self, addr, tablet_type, keyspace, shard, timeout, user=None,
password=None, keyfile=None, certfile=None, caller_id=None):
self.transaction_id = 0
self.session_id = 0
self.addr = addr
self.caller_id = caller_id
self.certfile = certfile
self.keyfile = keyfile
self.keyspace = keyspace
self.password = password
self.shard = shard
self.tablet_type = tablet_type
self.timeout = timeout
self.user = user
self.client = self._create_client()
self.logger_object = vtdb_logger.get_logger()
def __init__(self, addr, timeout,
root_certificates=None, private_key=None, certificate_chain=None,
**kwargs):
"""Creates a new GRPCVTGateConnection.
Args:
addr: address to connect to.
timeout: connection time out.
root_certificates: PEM_encoded root certificates.
private_key: PEM-encoded private key.
certificate_chain: PEM-encoded certificate chain.
**kwargs: passed up.
"""
super(GRPCVTGateConnection, self).__init__(addr, timeout, **kwargs)
self.stub = None
self.root_certificates = root_certificates
self.private_key = private_key
self.certificate_chain = certificate_chain
self.logger_object = vtdb_logger.get_logger()
def __init__(self,
addr,
tablet_type,
keyspace,
shard,
timeout,
user=None,
password=None,
encrypted=False,
keyfile=None,
certfile=None):
self.addr = addr
self.tablet_type = tablet_type
self.keyspace = keyspace
self.shard = shard
self.timeout = timeout
self.client = bsonrpc.BsonRpcClient(addr,
timeout,
user,
password,
encrypted=encrypted,
keyfile=keyfile,
certfile=certfile)
self.logger_object = vtdb_logger.get_logger()
def setUp(self):
self.shard_index = 0
self.master_tablet = shard_0_master
self.replica_tablet = shard_0_replica
vtdb_logger.register_vtdb_logger(LocalLogger())
self.logger = vtdb_logger.get_logger()
def test_keyspace_reresolve_on_execute(self):
before_topo_rtt = vtdb_logger.get_logger().get_topo_rtt()
try:
replica_conn = get_connection(db_type='replica', shard_index=self.shard_index)
except Exception, e:
self.fail("Connection to shard %s replica failed with error %s" % (shard_names[self.shard_index], str(e)))
self.replica_tablet.kill_vttablet()
time.sleep(self.keyspace_fetch_throttle)
# this should cause a refresh of the topology.
try:
results = replica_conn._execute("select 1 from vt_insert_test", {})
except Exception, e:
pass
after_tablet_error = vtdb_logger.get_logger().get_topo_rtt()
self.assertEqual(after_tablet_error - before_topo_rtt, 1, "One additional round-trips to topo server")
self.replica_tablet.start_vttablet()
def test_keyspace_reresolve_on_conn_failure(self):
before_topo_rtt = vtdb_logger.get_logger().get_topo_rtt()
self.replica_tablet.kill_vttablet()
time.sleep(self.keyspace_fetch_throttle)
try:
replica_conn = get_connection(db_type='replica', shard_index=self.shard_index)
except Exception, e:
pass
after_tablet_conn_error = vtdb_logger.get_logger().get_topo_rtt()
self.assertEqual(after_tablet_conn_error - before_topo_rtt, 1, "One additional round-trips to topo server")
self.replica_tablet.start_vttablet()
replica_conn = get_connection(
db_type='replica', shard_index=self.shard_index)
except Exception, e:
self.fail(
'Connection to shard %s replica failed with error %s' %
(shard_names[self.shard_index], str(e)))
self.replica_tablet.kill_vttablet()
time.sleep(self.keyspace_fetch_throttle)
# this should cause a refresh of the topology.
try:
results = replica_conn._execute('select 1 from vt_insert_test', {})
except Exception, e:
pass
after_tablet_error = vtdb_logger.get_logger().get_topo_rtt()
self.assertEqual(
after_tablet_error - before_topo_rtt, 1,
'One additional round-trips to topo server')
self.replica_tablet.start_vttablet()
def test_keyspace_reresolve_on_conn_failure(self):
before_topo_rtt = vtdb_logger.get_logger().get_topo_rtt()
self.replica_tablet.kill_vttablet()
time.sleep(self.keyspace_fetch_throttle)
try:
replica_conn = get_connection(
db_type='replica', shard_index=self.shard_index)
except Exception:
pass
def __init__(self, addr, timeout): super(GRPCVTGateConnection, self).__init__(addr, timeout) self.stub = None self.logger_object = vtdb_logger.get_logger()
def __init__(self, addr, timeout):
self.addr = addr
self.timeout = timeout
self.stub = None
self.session = None
self.logger_object = vtdb_logger.get_logger()
def __init__(self, addr, timeout): self.addr = addr self.timeout = timeout self.stub = None self.session = None self.logger_object = vtdb_logger.get_logger()
def __init__(self, addr, timeout, **kwargs): super(GRPCVTGateConnection, self).__init__(addr, timeout, **kwargs) self.stub = None self.logger_object = vtdb_logger.get_logger()
def setUp(self): self.shard_index = 0 self.master_tablet = shard_0_master self.replica_tablet = shard_0_replica vtdb_logger.register_vtdb_logger(LocalLogger()) self.logger = vtdb_logger.get_logger()
