def read_topology(zkocc_client, read_fqdb_keys=True):
fqdb_keys = []
db_keys = []
keyspace_list = zkocc_client.get_srv_keyspace_names('local')
# validate step
if len(keyspace_list) == 0:
logging.exception('zkocc returned empty keyspace list')
raise Exception('zkocc returned empty keyspace list')
for keyspace_name in keyspace_list:
try:
ks = keyspace.read_keyspace(zkocc_client, keyspace_name)
__add_keyspace(ks)
for shard_name in ks.shard_names:
for db_type in ks.db_types:
db_key_parts = [ks.name, shard_name, db_type]
db_key = '.'.join(db_key_parts)
db_keys.append(db_key)
if read_fqdb_keys:
db_instances = len(get_host_port_by_name(zkocc_client, db_key))
for db_i in xrange(db_instances):
fqdb_keys.append('.'.join(db_key_parts + [str(db_i)]))
except Exception:
logging.exception('error getting or parsing keyspace data for %s',
keyspace_name)
return db_keys, fqdb_keys
def read_topology(zkocc_client, read_fqdb_keys=True):
fqdb_keys = []
db_keys = []
keyspace_list = zkocc_client.get_srv_keyspace_names('local')
# validate step
if len(keyspace_list) == 0:
logging.exception('zkocc returned empty keyspace list')
raise Exception('zkocc returned empty keyspace list')
for keyspace_name in keyspace_list:
try:
ks = keyspace.read_keyspace(zkocc_client, keyspace_name)
__add_keyspace(ks)
for shard_name in ks.shard_names:
for db_type in ks.db_types:
db_key_parts = [ks.name, shard_name, db_type]
db_key = '.'.join(db_key_parts)
db_keys.append(db_key)
if read_fqdb_keys:
db_instances = len(get_host_port_by_name(zkocc_client, db_key))
for db_i in xrange(db_instances):
fqdb_keys.append('.'.join(db_key_parts + [str(db_i)]))
except Exception:
logging.exception('error getting or parsing keyspace data for %s',
keyspace_name)
return db_keys, fqdb_keys
def read_topology(zkocc_client, read_fqdb_keys=True):
fqdb_keys = []
db_keys = []
keyspace_list = zkocc_client.get_srv_keyspace_names('local')
# validate step
if len(keyspace_list) == 0:
vtdb_logger.get_logger().topo_empty_keyspace_list()
raise Exception('zkocc returned empty keyspace list')
for keyspace_name in keyspace_list:
try:
ks = keyspace.read_keyspace(zkocc_client, keyspace_name)
__set_keyspace(ks)
for db_type, partition in ks.partitions.iteritems():
for shard_reference in partition['ShardReferences']:
db_key_parts = [ks.name, shard_reference['Name'], db_type]
db_key = '.'.join(db_key_parts)
db_keys.append(db_key)
if read_fqdb_keys:
db_instances = len(
get_host_port_by_name(zkocc_client, db_key))
for db_i in xrange(db_instances):
fqdb_keys.append('.'.join(db_key_parts +
[str(db_i)]))
except Exception:
vtdb_logger.get_logger().topo_bad_keyspace_data(keyspace_name)
return db_keys, fqdb_keys
def read_topology(zkocc_client, read_fqdb_keys=True):
fqdb_keys = []
db_keys = []
keyspace_list = zkocc_client.children(
keyspace.ZK_KEYSPACE_PATH)['Children']
for keyspace_name in keyspace_list:
try:
ks = keyspace.read_keyspace(zkocc_client, keyspace_name)
__add_keyspace(ks)
for shard_name in ks.shard_names:
for db_type in ks.db_types:
db_key_parts = [ks.name, shard_name, db_type]
db_key = '.'.join(db_key_parts)
db_keys.append(db_key)
if read_fqdb_keys:
db_instances = len(
get_host_port_by_name(zkocc_client, db_key))
for db_i in xrange(db_instances):
fqdb_keys.append('.'.join(db_key_parts +
[str(db_i)]))
except Exception:
logging.exception('error getting or parsing keyspace data for %s',
keyspace_name)
return db_keys, fqdb_keys
def read_topology(zkocc_client, read_fqdb_keys=True):
fqdb_keys = []
db_keys = []
keyspace_list = zkocc_client.get_srv_keyspace_names('local')
# validate step
if len(keyspace_list) == 0:
vtdb_logger.get_logger().topo_empty_keyspace_list()
raise Exception('zkocc returned empty keyspace list')
for keyspace_name in keyspace_list:
try:
ks = keyspace.read_keyspace(zkocc_client, keyspace_name)
__set_keyspace(ks)
for db_type in ks.db_types:
for shard_name in ks.get_shard_names(db_type):
db_key_parts = [ks.name, shard_name, db_type]
db_key = '.'.join(db_key_parts)
db_keys.append(db_key)
if read_fqdb_keys:
db_instances = len(get_host_port_by_name(zkocc_client, db_key))
for db_i in xrange(db_instances):
fqdb_keys.append('.'.join(db_key_parts + [str(db_i)]))
except Exception:
vtdb_logger.get_logger().topo_bad_keyspace_data(keyspace_name)
return db_keys, fqdb_keys
def get_db_params_for_tablet_conn(topo_client, keyspace_name, shard, db_type,
timeout, encrypted, user, password):
db_params_list = []
encrypted_service = '_vts'
if encrypted:
service = encrypted_service
else:
service = '_vtocc'
db_key = "%s.%s.%s:%s" % (keyspace_name, shard, db_type, service)
keyspace_object = keyspace.read_keyspace(topo_client, keyspace_name)
# Handle vertical split by checking 'ServedFrom' field.
new_keyspace = None
served_from = keyspace_object.served_from
if served_from is not None:
new_keyspace = served_from.get(db_type, None)
if new_keyspace is not None:
keyspace_name = new_keyspace
try:
end_points_data = topo_client.get_end_points('local', keyspace_name,
shard, db_type)
except zkocc.ZkOccError as e:
logging.warning('no data for %s: %s', db_key, e)
return []
except Exception as e:
logging.warning('failed to get or parse topo data %s (%s): %s', db_key,
e, end_points_data)
return []
end_points_list = []
host_port_list = []
encrypted_host_port_list = []
if 'Entries' not in end_points_data:
raise Exception('zkocc returned: %s' % str(end_points_data))
for entry in end_points_data['Entries']:
if service in entry['NamedPortMap']:
host_port = (entry['Host'], entry['NamedPortMap'][service],
service == '_vts')
host_port_list.append(host_port)
if encrypted and encrypted_service in entry['NamedPortMap']:
host_port = (entry['Host'],
entry['NamedPortMap'][encrypted_service], True)
encrypted_host_port_list.append(host_port)
if encrypted and len(encrypted_host_port_list) > 0:
random.shuffle(encrypted_host_port_list)
end_points_list = encrypted_host_port_list
else:
random.shuffle(host_port_list)
end_points_list = host_port_list
for host, port, encrypted in end_points_list:
vt_params = VTConnParams(keyspace_name, shard, db_type,
"%s:%s" % (host, port), timeout, encrypted,
user, password).__dict__
db_params_list.append(vt_params)
return db_params_list
def read_and_get_keyspace(zkocc_client, name):
ks = None
try:
ks = __keyspace_map[name]
except KeyError:
ks = keyspace.read_keyspace(zkocc_client, name)
if ks is not None:
__add_keyspace(ks)
else:
raise
return ks
def read_and_get_keyspace(zkocc_client, name):
ks = None
try:
ks = __keyspace_map[name]
except KeyError:
ks = keyspace.read_keyspace(zkocc_client, name)
if ks is not None:
__add_keyspace(ks)
else:
raise
return ks
def get_db_params_for_tablet_conn(topo_client, keyspace_name, shard, db_type, timeout, encrypted, user, password):
db_params_list = []
encrypted_service = '_vts'
if encrypted:
service = encrypted_service
else:
service = '_vtocc'
db_key = "%s.%s.%s:%s" % (keyspace_name, shard, db_type, service)
keyspace_object = keyspace.read_keyspace(topo_client, keyspace_name)
# Handle vertical split by checking 'ServedFrom' field.
new_keyspace = None
served_from = keyspace_object.served_from
if served_from is not None:
new_keyspace = served_from.get(db_type, None)
if new_keyspace is not None:
keyspace_name = new_keyspace
try:
end_points_data = topo_client.get_end_points('local', keyspace_name, shard, db_type)
except zkocc.ZkOccError as e:
logging.warning('no data for %s: %s', db_key, e)
return []
except Exception as e:
logging.warning('failed to get or parse topo data %s (%s): %s', db_key, e,
end_points_data)
return []
end_points_list = []
host_port_list = []
encrypted_host_port_list = []
if 'Entries' not in end_points_data:
raise Exception('zkocc returned: %s' % str(end_points_data))
for entry in end_points_data['Entries']:
if service in entry['NamedPortMap']:
host_port = (entry['Host'], entry['NamedPortMap'][service],
service == '_vts')
host_port_list.append(host_port)
if encrypted and encrypted_service in entry['NamedPortMap']:
host_port = (entry['Host'], entry['NamedPortMap'][encrypted_service],
True)
encrypted_host_port_list.append(host_port)
if encrypted and len(encrypted_host_port_list) > 0:
random.shuffle(encrypted_host_port_list)
end_points_list = encrypted_host_port_list
else:
random.shuffle(host_port_list)
end_points_list = host_port_list
for host, port, encrypted in end_points_list:
vt_params = VTConnParams(keyspace_name, shard, db_type, "%s:%s" % (host, port), timeout, encrypted, user, password).__dict__
db_params_list.append(vt_params)
return db_params_list
def __init__(self, zkocc_client, keyspace_name, db_type, use_streaming, timeout, user, password, dbname): self.zkocc_client = zkocc_client self.keyspace_name = keyspace_name self.db_type = db_type self.use_streaming = use_streaming self.timeout = timeout self.user = user self.password = password self.dbname = dbname self.txn = None self.keyspace = keyspace.read_keyspace(self.zkocc_client, keyspace_name) self.conns = [None] * self.keyspace.shard_count self._streaming_shard_list = [] # shards with active streams
def __init__(self, zkocc_client, keyspace_name, db_type, use_streaming,
timeout, user, password, dbname):
self.zkocc_client = zkocc_client
self.keyspace_name = keyspace_name
self.db_type = db_type
self.use_streaming = use_streaming
self.timeout = timeout
self.user = user
self.password = password
self.dbname = dbname
self.txn = None
self.keyspace = keyspace.read_keyspace(self.zkocc_client,
keyspace_name)
self.conns = [None] * self.keyspace.shard_count
self._streaming_shard_list = [] # shards with active streams
def refresh_keyspace(zkocc_client, name):
global __keyspace_fetch_throttle
time_last_fetch = get_time_last_fetch(name)
if time_last_fetch is None:
return
if (time_last_fetch + __keyspace_fetch_throttle) > time.time():
return
start_time = time.time()
ks = keyspace.read_keyspace(zkocc_client, name)
topo_rtt = time.time() - start_time
if ks is not None:
__set_keyspace(ks)
vtdb_logger.get_logger().topo_keyspace_fetch(name, topo_rtt)
def refresh_keyspace(zkocc_client, name):
global __keyspace_fetch_throttle
time_last_fetch = get_time_last_fetch(name)
if time_last_fetch is None:
return
if (time_last_fetch + __keyspace_fetch_throttle) > time.time():
return
start_time = time.time()
ks = keyspace.read_keyspace(zkocc_client, name)
topo_rtt = time.time() - start_time
if ks is not None:
__set_keyspace(ks)
vtdb_logger.get_logger().topo_keyspace_fetch(name, topo_rtt)
def refresh_keyspace(zkocc_client, name):
global __keyspace_fetch_throttle
global __keyspace_fetch_logging
time_last_fetch = get_time_last_fetch(name)
if time_last_fetch is None:
return
if (time_last_fetch + __keyspace_fetch_throttle) > time.time():
return
start_time = time.time()
ks = keyspace.read_keyspace(zkocc_client, name)
topo_rtt = time.time() - start_time
if ks is not None:
__set_keyspace(ks)
if __keyspace_fetch_logging is not None:
__keyspace_fetch_logging(name, topo_rtt)
else:
log_keyspace_fetch(name, topo_rtt)
def read_topology(zkocc_client, read_fqdb_keys=True):
fqdb_keys = []
db_keys = []
keyspace_list = zkocc_client.children(keyspace.ZK_KEYSPACE_PATH)['Children']
for keyspace_name in keyspace_list:
try:
ks = keyspace.read_keyspace(zkocc_client, keyspace_name)
__add_keyspace(ks)
for shard_name in ks.shard_names:
for db_type in ks.db_types:
db_key_parts = [ks.name, shard_name, db_type]
db_key = '.'.join(db_key_parts)
db_keys.append(db_key)
if read_fqdb_keys:
db_instances = len(get_host_port_by_name(zkocc_client, db_key))
for db_i in xrange(db_instances):
fqdb_keys.append('.'.join(db_key_parts + [str(db_i)]))
except Exception:
logging.exception('error getting or parsing keyspace data for %s',
keyspace_name)
return db_keys, fqdb_keys
def _read_keyspace(self, keyspace_name):
vtgate_client = zkocc.ZkOccConnection(utils.vtgate.addr(), "test_nj",
30.0)
return keyspace.read_keyspace(vtgate_client, keyspace_name)
def _read_keyspace(self, keyspace_name):
vtgate_client = zkocc.ZkOccConnection(utils.vtgate.addr(),
"test_nj", 30.0)
return keyspace.read_keyspace(vtgate_client, keyspace_name)
def _read_keyspace(self, keyspace_name):
global vtgate_port
vtgate_client = zkocc.ZkOccConnection("localhost:%u" % vtgate_port,
"test_nj", 30.0)
return keyspace.read_keyspace(vtgate_client, keyspace_name)
def _read_keyspace(self, keyspace_name):
global vtgate_port
vtgate_client = zkocc.ZkOccConnection("localhost:%u" % vtgate_port,
"test_nj", 30.0)
return keyspace.read_keyspace(vtgate_client, keyspace_name)
def get_keyspace(name):
try:
return __keyspace_map[name]
except KeyError:
global __topo_client
return keyspace.read_keyspace(__topo_client, name)
