def read_topology(zkocc_client, read_fqdb_keys=True): fqdb_keys = [] db_keys = [] keyspace_list = zkocc_client.get_srv_keyspace_names('local') # validate step if len(keyspace_list) == 0: logging.exception('zkocc returned empty keyspace list') raise Exception('zkocc returned empty keyspace list') for keyspace_name in keyspace_list: try: ks = keyspace.read_keyspace(zkocc_client, keyspace_name) __add_keyspace(ks) for shard_name in ks.shard_names: for db_type in ks.db_types: db_key_parts = [ks.name, shard_name, db_type] db_key = '.'.join(db_key_parts) db_keys.append(db_key) if read_fqdb_keys: db_instances = len(get_host_port_by_name(zkocc_client, db_key)) for db_i in xrange(db_instances): fqdb_keys.append('.'.join(db_key_parts + [str(db_i)])) except Exception: logging.exception('error getting or parsing keyspace data for %s', keyspace_name) return db_keys, fqdb_keys
def read_topology(zkocc_client, read_fqdb_keys=True): fqdb_keys = [] db_keys = [] keyspace_list = zkocc_client.get_srv_keyspace_names('local') # validate step if len(keyspace_list) == 0: vtdb_logger.get_logger().topo_empty_keyspace_list() raise Exception('zkocc returned empty keyspace list') for keyspace_name in keyspace_list: try: ks = keyspace.read_keyspace(zkocc_client, keyspace_name) __set_keyspace(ks) for db_type, partition in ks.partitions.iteritems(): for shard_reference in partition['ShardReferences']: db_key_parts = [ks.name, shard_reference['Name'], db_type] db_key = '.'.join(db_key_parts) db_keys.append(db_key) if read_fqdb_keys: db_instances = len( get_host_port_by_name(zkocc_client, db_key)) for db_i in xrange(db_instances): fqdb_keys.append('.'.join(db_key_parts + [str(db_i)])) except Exception: vtdb_logger.get_logger().topo_bad_keyspace_data(keyspace_name) return db_keys, fqdb_keys
def read_topology(zkocc_client, read_fqdb_keys=True): fqdb_keys = [] db_keys = [] keyspace_list = zkocc_client.children( keyspace.ZK_KEYSPACE_PATH)['Children'] for keyspace_name in keyspace_list: try: ks = keyspace.read_keyspace(zkocc_client, keyspace_name) __add_keyspace(ks) for shard_name in ks.shard_names: for db_type in ks.db_types: db_key_parts = [ks.name, shard_name, db_type] db_key = '.'.join(db_key_parts) db_keys.append(db_key) if read_fqdb_keys: db_instances = len( get_host_port_by_name(zkocc_client, db_key)) for db_i in xrange(db_instances): fqdb_keys.append('.'.join(db_key_parts + [str(db_i)])) except Exception: logging.exception('error getting or parsing keyspace data for %s', keyspace_name) return db_keys, fqdb_keys
def read_topology(zkocc_client, read_fqdb_keys=True): fqdb_keys = [] db_keys = [] keyspace_list = zkocc_client.get_srv_keyspace_names('local') # validate step if len(keyspace_list) == 0: vtdb_logger.get_logger().topo_empty_keyspace_list() raise Exception('zkocc returned empty keyspace list') for keyspace_name in keyspace_list: try: ks = keyspace.read_keyspace(zkocc_client, keyspace_name) __set_keyspace(ks) for db_type in ks.db_types: for shard_name in ks.get_shard_names(db_type): db_key_parts = [ks.name, shard_name, db_type] db_key = '.'.join(db_key_parts) db_keys.append(db_key) if read_fqdb_keys: db_instances = len(get_host_port_by_name(zkocc_client, db_key)) for db_i in xrange(db_instances): fqdb_keys.append('.'.join(db_key_parts + [str(db_i)])) except Exception: vtdb_logger.get_logger().topo_bad_keyspace_data(keyspace_name) return db_keys, fqdb_keys
def get_db_params_for_tablet_conn(topo_client, keyspace_name, shard, db_type, timeout, encrypted, user, password): db_params_list = [] encrypted_service = '_vts' if encrypted: service = encrypted_service else: service = '_vtocc' db_key = "%s.%s.%s:%s" % (keyspace_name, shard, db_type, service) keyspace_object = keyspace.read_keyspace(topo_client, keyspace_name) # Handle vertical split by checking 'ServedFrom' field. new_keyspace = None served_from = keyspace_object.served_from if served_from is not None: new_keyspace = served_from.get(db_type, None) if new_keyspace is not None: keyspace_name = new_keyspace try: end_points_data = topo_client.get_end_points('local', keyspace_name, shard, db_type) except zkocc.ZkOccError as e: logging.warning('no data for %s: %s', db_key, e) return [] except Exception as e: logging.warning('failed to get or parse topo data %s (%s): %s', db_key, e, end_points_data) return [] end_points_list = [] host_port_list = [] encrypted_host_port_list = [] if 'Entries' not in end_points_data: raise Exception('zkocc returned: %s' % str(end_points_data)) for entry in end_points_data['Entries']: if service in entry['NamedPortMap']: host_port = (entry['Host'], entry['NamedPortMap'][service], service == '_vts') host_port_list.append(host_port) if encrypted and encrypted_service in entry['NamedPortMap']: host_port = (entry['Host'], entry['NamedPortMap'][encrypted_service], True) encrypted_host_port_list.append(host_port) if encrypted and len(encrypted_host_port_list) > 0: random.shuffle(encrypted_host_port_list) end_points_list = encrypted_host_port_list else: random.shuffle(host_port_list) end_points_list = host_port_list for host, port, encrypted in end_points_list: vt_params = VTConnParams(keyspace_name, shard, db_type, "%s:%s" % (host, port), timeout, encrypted, user, password).__dict__ db_params_list.append(vt_params) return db_params_list
def read_and_get_keyspace(zkocc_client, name): ks = None try: ks = __keyspace_map[name] except KeyError: ks = keyspace.read_keyspace(zkocc_client, name) if ks is not None: __add_keyspace(ks) else: raise return ks
def __init__(self, zkocc_client, keyspace_name, db_type, use_streaming, timeout, user, password, dbname): self.zkocc_client = zkocc_client self.keyspace_name = keyspace_name self.db_type = db_type self.use_streaming = use_streaming self.timeout = timeout self.user = user self.password = password self.dbname = dbname self.txn = None self.keyspace = keyspace.read_keyspace(self.zkocc_client, keyspace_name) self.conns = [None] * self.keyspace.shard_count self._streaming_shard_list = [] # shards with active streams
def refresh_keyspace(zkocc_client, name): global __keyspace_fetch_throttle time_last_fetch = get_time_last_fetch(name) if time_last_fetch is None: return if (time_last_fetch + __keyspace_fetch_throttle) > time.time(): return start_time = time.time() ks = keyspace.read_keyspace(zkocc_client, name) topo_rtt = time.time() - start_time if ks is not None: __set_keyspace(ks) vtdb_logger.get_logger().topo_keyspace_fetch(name, topo_rtt)
def refresh_keyspace(zkocc_client, name): global __keyspace_fetch_throttle global __keyspace_fetch_logging time_last_fetch = get_time_last_fetch(name) if time_last_fetch is None: return if (time_last_fetch + __keyspace_fetch_throttle) > time.time(): return start_time = time.time() ks = keyspace.read_keyspace(zkocc_client, name) topo_rtt = time.time() - start_time if ks is not None: __set_keyspace(ks) if __keyspace_fetch_logging is not None: __keyspace_fetch_logging(name, topo_rtt) else: log_keyspace_fetch(name, topo_rtt)
def read_topology(zkocc_client, read_fqdb_keys=True): fqdb_keys = [] db_keys = [] keyspace_list = zkocc_client.children(keyspace.ZK_KEYSPACE_PATH)['Children'] for keyspace_name in keyspace_list: try: ks = keyspace.read_keyspace(zkocc_client, keyspace_name) __add_keyspace(ks) for shard_name in ks.shard_names: for db_type in ks.db_types: db_key_parts = [ks.name, shard_name, db_type] db_key = '.'.join(db_key_parts) db_keys.append(db_key) if read_fqdb_keys: db_instances = len(get_host_port_by_name(zkocc_client, db_key)) for db_i in xrange(db_instances): fqdb_keys.append('.'.join(db_key_parts + [str(db_i)])) except Exception: logging.exception('error getting or parsing keyspace data for %s', keyspace_name) return db_keys, fqdb_keys
def _read_keyspace(self, keyspace_name): vtgate_client = zkocc.ZkOccConnection(utils.vtgate.addr(), "test_nj", 30.0) return keyspace.read_keyspace(vtgate_client, keyspace_name)
def _read_keyspace(self, keyspace_name): global vtgate_port vtgate_client = zkocc.ZkOccConnection("localhost:%u" % vtgate_port, "test_nj", 30.0) return keyspace.read_keyspace(vtgate_client, keyspace_name)
def get_keyspace(name): try: return __keyspace_map[name] except KeyError: global __topo_client return keyspace.read_keyspace(__topo_client, name)