def connect(self):
db_key = "%s.%s.%s" % (self.keyspace, self.shard, self.db_type)
db_params_list = get_vt_connection_params_list(
self.zkocc_client, self.keyspace, self.shard, self.db_type,
self.timeout, self.encrypted, self.user, self.password,
self.vtgate_protocol, self.vtgate_addrs)
if not db_params_list:
raise dbexceptions.OperationalError(
"empty db params list - no db instance available for key %s" %
db_key)
db_exception = None
host_addr = None
# no retries here, since there is a higher level retry with reconnect.
for params in db_params_list:
try:
db_params = params.copy()
host_addr = db_params['addr']
if self.vtgate_protocol in vtclient_conn_classes:
self.conn = vtclient_conn_classes[self.vtgate_protocol](
**db_params)
else:
raise dbexceptions.OperationalError(
'unknown vtgate protocol: %s' % self.vtgate_protocol)
self.conn.dial()
self.conn_db_params = db_params
return self.conn
except Exception as e:
db_exception = e
logging.warning('db connection failed: %s %s, %s', db_key,
host_addr, e)
raise dbexceptions.OperationalError('unable to create vt connection',
db_key, host_addr, db_exception)
def connect(vtgate_addrs, timeout, user=None, password=None):
"""Return opened connection to vtgate."""
db_params_list = get_params_for_vtgate_conn(vtgate_addrs,
timeout,
user=user,
password=password)
if not db_params_list:
raise dbexceptions.OperationalError(
'empty db params list - no db instance available for vtgate_addrs %s'
% vtgate_addrs)
db_exception = None
host_addr = None
for params in db_params_list:
try:
db_params = params.copy()
host_addr = db_params['addr']
conn = VTGateConnection(**db_params)
conn.dial()
return conn
except Exception as e:
db_exception = e
logging.warning('db connection failed: %s, %s', host_addr, e)
raise dbexceptions.OperationalError('unable to create vt connection',
host_addr, db_exception)
def _connect(self):
db_key = "%s.%s.%s" % (self.keyspace, self.shard, self.db_type)
db_params_list = get_vt_connection_params_list(
self.topo_client, self.keyspace, self.shard, self.db_type,
self.timeout, self.encrypted, self.user, self.password)
if not db_params_list:
# no valid end-points were found, re-read the keyspace
self.resolve_topology()
raise dbexceptions.OperationalError(
"empty db params list - no db instance available for key %s" %
db_key)
db_exception = None
host_addr = None
# no retries here, since there is a higher level retry with reconnect.
for params in db_params_list:
try:
db_params = params.copy()
host_addr = db_params['addr']
self.conn = tablet.TabletConnection(**db_params)
self.conn.dial()
self.conn_db_params = db_params
return self.conn
except Exception as e:
db_exception = e
logging.warning('db connection failed: %s %s, %s', db_key,
host_addr, e)
# vttablet threw an Operational Error on connect, re-read the keyspace
if isinstance(e, dbexceptions.OperationalError):
self.resolve_topology()
raise dbexceptions.OperationalError('unable to create vt connection',
db_key, host_addr, db_exception)
def read_keyspace(zkocc_client, keyspace_name):
try:
data = zkocc_client.get_srv_keyspace('local', keyspace_name)
if not data:
raise dbexceptions.OperationalError('invalid empty keyspace',
keyspace_name)
return Keyspace(keyspace_name, data)
except Exception as e:
raise dbexceptions.OperationalError('invalid keyspace', keyspace_name,
e)
def read_keyspace(zkocc_client, keyspace_name):
keyspace_path = ZK_KEYSPACE_PATH + '/' + keyspace_name
try:
data = zkocc_client.get(keyspace_path)['Data']
if not data:
raise dbexceptions.OperationalError('invalid empty keyspace',
keyspace_path)
return Keyspace(keyspace_name, json.loads(data))
except Exception as e:
raise dbexceptions.OperationalError('invalid keyspace', keyspace_path,
e)
def read_keyspace(topo_client, keyspace_name):
try:
data = topo_client.get_srv_keyspace('local', keyspace_name)
if not data:
raise dbexceptions.OperationalError('invalid empty keyspace',
keyspace_name)
return Keyspace(keyspace_name,
keyrange_constants.srv_keyspace_proto3_to_old(data))
except dbexceptions.OperationalError as e:
raise e
except Exception as e:
raise dbexceptions.OperationalError('invalid keyspace', keyspace_name,
e)
def stream_update(self, position, timeout=3600.0):
try:
req = binlogdata_pb2.StreamUpdateRequest(position=position)
it = self.stub.StreamUpdate(req, timeout)
for response in it:
stream_event = response.stream_event
fields = []
rows = []
if stream_event.primary_key_fields:
conversions = []
for field in stream_event.primary_key_fields:
fields.append(field.name)
conversions.append(
field_types_proto3.conversions.get(field.type))
for r in stream_event.primary_key_values:
row = tuple(_make_row(r, conversions))
rows.append(row)
yield update_stream.StreamEvent(
category=int(stream_event.category),
table_name=stream_event.table_name,
fields=fields,
rows=rows,
sql=stream_event.sql,
timestamp=stream_event.timestamp,
transaction_id=stream_event.transaction_id)
except face.AbortionError as e:
# FIXME(alainjobart) These exceptions don't print well, so raise
# one that will. The real fix is to define a set of exceptions
# for this library and raise that, but it's more work.
raise dbexceptions.OperationalError(e.details, e)
def commit(self):
try:
if self.txn:
err_conns = []
for i, conn in enumerate(self.txn.conns):
if conn.is_closed():
err_conns.append(conn)
if err_conns:
raise dbexceptions.OperationalError(
'tablets offline', [str(x) for x in err_conn])
for i, conn in enumerate(self.txn.conns):
try:
conn.commit()
except dbexceptions.DatabaseError as e:
err_conns.append(conn)
# If our first commit fails, just raise the error after rolling back
# everything else.
if i == 0:
try:
self.rollback()
except dbexceptions.DatabaseError:
pass
raise e
if err_conns and len(err_conns) != len(self.txn.conns):
raise dbexceptions.PartialCommitError(err_conns)
except dbexceptions.DatabaseError:
# If a DatabaseError occurred, scan for dead connections and remove
# them so they will be recreated.
for i, conn in enumerate(self.conns):
if conn.is_closed():
self.conns[i] = None
finally:
self.txn = None
def _execute_batch(self, query_list, bind_vars_list, shard_idx):
new_query_list = []
new_bind_vars_list = []
for query, bind_vars in zip(query_list, bind_vars_list):
query, bind_vars = dbapi.prepare_query_bind_vars(query, bind_vars)
new_query_list.append(query)
new_bind_vars_list.append(bind_vars)
query_list = new_query_list
bind_vars_list = new_bind_vars_list
for x in xrange(self.max_attempts):
try:
conn = self.conns[shard_idx]
if conn is None:
conn = self._dial_shard(shard_idx)
return conn._execute_batch(query_list, bind_vars_list)
except dbexceptions.OperationalError as e:
# Tear down regardless of the precise failure.
self.conns[shard_idx] = None
if isinstance(e, tablet3.TimeoutError):
# On any timeout let the error bubble up and just redial next time.
raise e
if isinstance(e, tablet3.RetryError):
# Give the tablet a moment to restart itself. This isn't
# strictly necessary since there is a significant chance you
# will end up talking to another host.
time.sleep(self.reconnect_delay)
raise dbexceptions.OperationalError('tablets unreachable',
self.keyspace_name, shard_idx,
self.db_type)
def _execute_on_shard(self, query, bind_vars, shard_idx):
query, bind_vars = dbapi.prepare_query_bind_vars(query, bind_vars)
for x in xrange(self.max_attempts):
try:
conn = self.conns[shard_idx]
if conn is None:
conn = self._dial_shard(shard_idx)
if self.txn:
self.txn.stmts.append(query)
if conn not in self.txt.conns:
# Defer the begin until we actually issue a statement.
conn.begin()
self.txt.conns.append(conn)
return conn._execute(query, bind_vars)
except dbexceptions.OperationalError as e:
# Tear down regardless of the precise failure.
self.conns[shard_idx] = None
if isinstance(e, tablet3.TimeoutError):
# On any timeout let the error bubble up and just redial next time.
raise e
if isinstance(e, tablet3.RetryError):
# Give the tablet a moment to restart itself. This isn't
# strictly necessary since there is a significant chance you
# will end up talking to another host.
time.sleep(self.reconnect_delay)
raise dbexceptions.OperationalError('tablets unreachable',
self.keyspace_name, shard_idx,
self.db_type)
def stream_next(self):
try:
response = self.client.stream_next()
if response is None:
return None, None, None
update_stream_response = UpdateStreamResponse(response.reply)
except gorpc.GoRpcError as e:
raise dbexceptions.OperationalError(*e.args)
except:
logging.exception('gorpc low-level error')
raise
return update_stream_response.Coord, update_stream_response.Data, update_stream_response.Error
def stream_next(self):
try:
response = self.client.stream_next()
if response is None:
return None
return EventData(response.reply).__dict__
except gorpc.AppError as e:
raise dbexceptions.DatabaseError(*e.args)
except gorpc.GoRpcError as e:
raise dbexceptions.OperationalError(*e.args)
except:
logging.exception('gorpc low-level error')
raise
def stream_start(self, group_id):
try:
self.client.stream_call('UpdateStream.ServeUpdateStream',
{"GroupId": group_id})
response = self.client.stream_next()
if response is None:
return None
return EventData(response.reply).__dict__
except gorpc.GoRpcError as e:
raise dbexceptions.OperationalError(*e.args)
except:
logging.exception('gorpc low-level error')
raise
def stream_start(self, start_position):
req = {'StartPosition': start_position}
try:
self.client.stream_call('UpdateStream.ServeUpdateStream', req)
first_response = self.client.stream_next()
update_stream_response = UpdateStreamResponse(first_response.reply)
except gorpc.GoRpcError as e:
raise dbexceptions.OperationalError(*e.args)
except:
logging.exception('gorpc low-level error')
raise
return update_stream_response.Coord, update_stream_response.Data, update_stream_response.Error
def stream_update(self, position, timeout=3600.0):
"""Note this implementation doesn't honor the timeout."""
try:
self.client.stream_call('UpdateStream.ServeUpdateStream',
{'Position': position})
while True:
response = self.client.stream_next()
if response is None:
break
reply = response.reply
str_category = reply['Category']
if str_category == 'DML':
category = update_stream.StreamEvent.DML
elif str_category == 'DDL':
category = update_stream.StreamEvent.DDL
elif str_category == 'POS':
category = update_stream.StreamEvent.POS
else:
category = update_stream.StreamEvent.ERR
fields = []
rows = []
if reply['PrimaryKeyFields']:
conversions = []
for field in reply['PrimaryKeyFields']:
fields.append(field['Name'])
conversions.append(
field_types.conversions.get(field['Type']))
for pk_list in reply['PrimaryKeyValues']:
if not pk_list:
continue
row = tuple(_make_row(pk_list, conversions))
rows.append(row)
yield update_stream.StreamEvent(
category=category,
table_name=reply['TableName'],
fields=fields,
rows=rows,
sql=reply['Sql'],
timestamp=reply['Timestamp'],
transaction_id=reply['TransactionID'])
except gorpc.AppError as e:
raise dbexceptions.DatabaseError(*e.args)
except gorpc.GoRpcError as e:
raise dbexceptions.OperationalError(*e.args)
except:
raise
def _dial_shard(self, shard_idx):
shard_name = self.keyspace.shard_names[shard_idx]
name_path = os.path.join(keyspace.ZK_KEYSPACE_PATH, self.keyspace_name,
shard_name, self.db_type)
addrs = zkns_query.lookup_name(zkocc_client, name_path)
for addr in addrs:
tablet_conn = tablet3.TabletConnection(addr, self.keyspace_name,
shard_name, self.timeout,
self.user, self.password)
try:
tablet_conn.dial()
self.conns[shard_idx] = tablet_conn
return tablet_conn
except dbexceptions.OperationalError:
# FIXME(msolomon) Implement retry deadline.
pass
raise dbexceptions.OperationalError('no tablet available for shard',
name_path)
def stream_update(self,
keyspace,
shard,
tablet_type,
position='',
timestamp=0,
timeout=3600.0):
try:
target = query_pb2.Target(keyspace=keyspace,
shard=shard,
tablet_type=tablet_type)
req = query_pb2.UpdateStreamRequest(target=target,
position=position,
timestamp=timestamp)
for response in self.stub.UpdateStream(req, timeout):
yield response.event
except face.AbortionError as e:
# FIXME(alainjobart) These exceptions don't print well, so raise
# one that will. The real fix is to define a set of exceptions
# for this library and raise that, but it's more work.
raise dbexceptions.OperationalError(e.details, e)
def _begin(self, shard_idx):
for x in xrange(self.max_attempts):
try:
conn = self.conns[shard_idx]
if conn is None:
conn = self._dial_shard(shard_idx)
return conn.begin()
except dbexceptions.OperationalError as e:
# Tear down regardless of the precise failure.
self.conns[shard_idx] = None
if isinstance(e, tablet3.TimeoutError):
# On any timeout let the error bubble up and just redial next time.
raise e
if isinstance(e, tablet3.RetryError):
# Give the tablet a moment to restart itself. This isn't
# strictly necessary since there is a significant chance you
# will end up talking to another host.
time.sleep(self.reconnect_delay)
raise dbexceptions.OperationalError('tablets unreachable',
self.keyspace_name, shard_idx,
self.db_type)
