def connect(self): db_key = "%s.%s.%s" % (self.keyspace, self.shard, self.db_type) db_params_list = get_vt_connection_params_list( self.zkocc_client, self.keyspace, self.shard, self.db_type, self.timeout, self.encrypted, self.user, self.password, self.vtgate_protocol, self.vtgate_addrs) if not db_params_list: raise dbexceptions.OperationalError( "empty db params list - no db instance available for key %s" % db_key) db_exception = None host_addr = None # no retries here, since there is a higher level retry with reconnect. for params in db_params_list: try: db_params = params.copy() host_addr = db_params['addr'] if self.vtgate_protocol in vtclient_conn_classes: self.conn = vtclient_conn_classes[self.vtgate_protocol]( **db_params) else: raise dbexceptions.OperationalError( 'unknown vtgate protocol: %s' % self.vtgate_protocol) self.conn.dial() self.conn_db_params = db_params return self.conn except Exception as e: db_exception = e logging.warning('db connection failed: %s %s, %s', db_key, host_addr, e) raise dbexceptions.OperationalError('unable to create vt connection', db_key, host_addr, db_exception)
def connect(vtgate_addrs, timeout, user=None, password=None): """Return opened connection to vtgate.""" db_params_list = get_params_for_vtgate_conn(vtgate_addrs, timeout, user=user, password=password) if not db_params_list: raise dbexceptions.OperationalError( 'empty db params list - no db instance available for vtgate_addrs %s' % vtgate_addrs) db_exception = None host_addr = None for params in db_params_list: try: db_params = params.copy() host_addr = db_params['addr'] conn = VTGateConnection(**db_params) conn.dial() return conn except Exception as e: db_exception = e logging.warning('db connection failed: %s, %s', host_addr, e) raise dbexceptions.OperationalError('unable to create vt connection', host_addr, db_exception)
def _connect(self): db_key = "%s.%s.%s" % (self.keyspace, self.shard, self.db_type) db_params_list = get_vt_connection_params_list( self.topo_client, self.keyspace, self.shard, self.db_type, self.timeout, self.encrypted, self.user, self.password) if not db_params_list: # no valid end-points were found, re-read the keyspace self.resolve_topology() raise dbexceptions.OperationalError( "empty db params list - no db instance available for key %s" % db_key) db_exception = None host_addr = None # no retries here, since there is a higher level retry with reconnect. for params in db_params_list: try: db_params = params.copy() host_addr = db_params['addr'] self.conn = tablet.TabletConnection(**db_params) self.conn.dial() self.conn_db_params = db_params return self.conn except Exception as e: db_exception = e logging.warning('db connection failed: %s %s, %s', db_key, host_addr, e) # vttablet threw an Operational Error on connect, re-read the keyspace if isinstance(e, dbexceptions.OperationalError): self.resolve_topology() raise dbexceptions.OperationalError('unable to create vt connection', db_key, host_addr, db_exception)
def read_keyspace(zkocc_client, keyspace_name): try: data = zkocc_client.get_srv_keyspace('local', keyspace_name) if not data: raise dbexceptions.OperationalError('invalid empty keyspace', keyspace_name) return Keyspace(keyspace_name, data) except Exception as e: raise dbexceptions.OperationalError('invalid keyspace', keyspace_name, e)
def read_keyspace(zkocc_client, keyspace_name): keyspace_path = ZK_KEYSPACE_PATH + '/' + keyspace_name try: data = zkocc_client.get(keyspace_path)['Data'] if not data: raise dbexceptions.OperationalError('invalid empty keyspace', keyspace_path) return Keyspace(keyspace_name, json.loads(data)) except Exception as e: raise dbexceptions.OperationalError('invalid keyspace', keyspace_path, e)
def read_keyspace(topo_client, keyspace_name): try: data = topo_client.get_srv_keyspace('local', keyspace_name) if not data: raise dbexceptions.OperationalError('invalid empty keyspace', keyspace_name) return Keyspace(keyspace_name, keyrange_constants.srv_keyspace_proto3_to_old(data)) except dbexceptions.OperationalError as e: raise e except Exception as e: raise dbexceptions.OperationalError('invalid keyspace', keyspace_name, e)
def stream_update(self, position, timeout=3600.0): try: req = binlogdata_pb2.StreamUpdateRequest(position=position) it = self.stub.StreamUpdate(req, timeout) for response in it: stream_event = response.stream_event fields = [] rows = [] if stream_event.primary_key_fields: conversions = [] for field in stream_event.primary_key_fields: fields.append(field.name) conversions.append( field_types_proto3.conversions.get(field.type)) for r in stream_event.primary_key_values: row = tuple(_make_row(r, conversions)) rows.append(row) yield update_stream.StreamEvent( category=int(stream_event.category), table_name=stream_event.table_name, fields=fields, rows=rows, sql=stream_event.sql, timestamp=stream_event.timestamp, transaction_id=stream_event.transaction_id) except face.AbortionError as e: # FIXME(alainjobart) These exceptions don't print well, so raise # one that will. The real fix is to define a set of exceptions # for this library and raise that, but it's more work. raise dbexceptions.OperationalError(e.details, e)
def commit(self): try: if self.txn: err_conns = [] for i, conn in enumerate(self.txn.conns): if conn.is_closed(): err_conns.append(conn) if err_conns: raise dbexceptions.OperationalError( 'tablets offline', [str(x) for x in err_conn]) for i, conn in enumerate(self.txn.conns): try: conn.commit() except dbexceptions.DatabaseError as e: err_conns.append(conn) # If our first commit fails, just raise the error after rolling back # everything else. if i == 0: try: self.rollback() except dbexceptions.DatabaseError: pass raise e if err_conns and len(err_conns) != len(self.txn.conns): raise dbexceptions.PartialCommitError(err_conns) except dbexceptions.DatabaseError: # If a DatabaseError occurred, scan for dead connections and remove # them so they will be recreated. for i, conn in enumerate(self.conns): if conn.is_closed(): self.conns[i] = None finally: self.txn = None
def _execute_batch(self, query_list, bind_vars_list, shard_idx): new_query_list = [] new_bind_vars_list = [] for query, bind_vars in zip(query_list, bind_vars_list): query, bind_vars = dbapi.prepare_query_bind_vars(query, bind_vars) new_query_list.append(query) new_bind_vars_list.append(bind_vars) query_list = new_query_list bind_vars_list = new_bind_vars_list for x in xrange(self.max_attempts): try: conn = self.conns[shard_idx] if conn is None: conn = self._dial_shard(shard_idx) return conn._execute_batch(query_list, bind_vars_list) except dbexceptions.OperationalError as e: # Tear down regardless of the precise failure. self.conns[shard_idx] = None if isinstance(e, tablet3.TimeoutError): # On any timeout let the error bubble up and just redial next time. raise e if isinstance(e, tablet3.RetryError): # Give the tablet a moment to restart itself. This isn't # strictly necessary since there is a significant chance you # will end up talking to another host. time.sleep(self.reconnect_delay) raise dbexceptions.OperationalError('tablets unreachable', self.keyspace_name, shard_idx, self.db_type)
def _execute_on_shard(self, query, bind_vars, shard_idx): query, bind_vars = dbapi.prepare_query_bind_vars(query, bind_vars) for x in xrange(self.max_attempts): try: conn = self.conns[shard_idx] if conn is None: conn = self._dial_shard(shard_idx) if self.txn: self.txn.stmts.append(query) if conn not in self.txt.conns: # Defer the begin until we actually issue a statement. conn.begin() self.txt.conns.append(conn) return conn._execute(query, bind_vars) except dbexceptions.OperationalError as e: # Tear down regardless of the precise failure. self.conns[shard_idx] = None if isinstance(e, tablet3.TimeoutError): # On any timeout let the error bubble up and just redial next time. raise e if isinstance(e, tablet3.RetryError): # Give the tablet a moment to restart itself. This isn't # strictly necessary since there is a significant chance you # will end up talking to another host. time.sleep(self.reconnect_delay) raise dbexceptions.OperationalError('tablets unreachable', self.keyspace_name, shard_idx, self.db_type)
def stream_next(self): try: response = self.client.stream_next() if response is None: return None, None, None update_stream_response = UpdateStreamResponse(response.reply) except gorpc.GoRpcError as e: raise dbexceptions.OperationalError(*e.args) except: logging.exception('gorpc low-level error') raise return update_stream_response.Coord, update_stream_response.Data, update_stream_response.Error
def stream_next(self): try: response = self.client.stream_next() if response is None: return None return EventData(response.reply).__dict__ except gorpc.AppError as e: raise dbexceptions.DatabaseError(*e.args) except gorpc.GoRpcError as e: raise dbexceptions.OperationalError(*e.args) except: logging.exception('gorpc low-level error') raise
def stream_start(self, group_id): try: self.client.stream_call('UpdateStream.ServeUpdateStream', {"GroupId": group_id}) response = self.client.stream_next() if response is None: return None return EventData(response.reply).__dict__ except gorpc.GoRpcError as e: raise dbexceptions.OperationalError(*e.args) except: logging.exception('gorpc low-level error') raise
def stream_start(self, start_position): req = {'StartPosition': start_position} try: self.client.stream_call('UpdateStream.ServeUpdateStream', req) first_response = self.client.stream_next() update_stream_response = UpdateStreamResponse(first_response.reply) except gorpc.GoRpcError as e: raise dbexceptions.OperationalError(*e.args) except: logging.exception('gorpc low-level error') raise return update_stream_response.Coord, update_stream_response.Data, update_stream_response.Error
def stream_update(self, position, timeout=3600.0): """Note this implementation doesn't honor the timeout.""" try: self.client.stream_call('UpdateStream.ServeUpdateStream', {'Position': position}) while True: response = self.client.stream_next() if response is None: break reply = response.reply str_category = reply['Category'] if str_category == 'DML': category = update_stream.StreamEvent.DML elif str_category == 'DDL': category = update_stream.StreamEvent.DDL elif str_category == 'POS': category = update_stream.StreamEvent.POS else: category = update_stream.StreamEvent.ERR fields = [] rows = [] if reply['PrimaryKeyFields']: conversions = [] for field in reply['PrimaryKeyFields']: fields.append(field['Name']) conversions.append( field_types.conversions.get(field['Type'])) for pk_list in reply['PrimaryKeyValues']: if not pk_list: continue row = tuple(_make_row(pk_list, conversions)) rows.append(row) yield update_stream.StreamEvent( category=category, table_name=reply['TableName'], fields=fields, rows=rows, sql=reply['Sql'], timestamp=reply['Timestamp'], transaction_id=reply['TransactionID']) except gorpc.AppError as e: raise dbexceptions.DatabaseError(*e.args) except gorpc.GoRpcError as e: raise dbexceptions.OperationalError(*e.args) except: raise
def _dial_shard(self, shard_idx): shard_name = self.keyspace.shard_names[shard_idx] name_path = os.path.join(keyspace.ZK_KEYSPACE_PATH, self.keyspace_name, shard_name, self.db_type) addrs = zkns_query.lookup_name(zkocc_client, name_path) for addr in addrs: tablet_conn = tablet3.TabletConnection(addr, self.keyspace_name, shard_name, self.timeout, self.user, self.password) try: tablet_conn.dial() self.conns[shard_idx] = tablet_conn return tablet_conn except dbexceptions.OperationalError: # FIXME(msolomon) Implement retry deadline. pass raise dbexceptions.OperationalError('no tablet available for shard', name_path)
def stream_update(self, keyspace, shard, tablet_type, position='', timestamp=0, timeout=3600.0): try: target = query_pb2.Target(keyspace=keyspace, shard=shard, tablet_type=tablet_type) req = query_pb2.UpdateStreamRequest(target=target, position=position, timestamp=timestamp) for response in self.stub.UpdateStream(req, timeout): yield response.event except face.AbortionError as e: # FIXME(alainjobart) These exceptions don't print well, so raise # one that will. The real fix is to define a set of exceptions # for this library and raise that, but it's more work. raise dbexceptions.OperationalError(e.details, e)
def _begin(self, shard_idx): for x in xrange(self.max_attempts): try: conn = self.conns[shard_idx] if conn is None: conn = self._dial_shard(shard_idx) return conn.begin() except dbexceptions.OperationalError as e: # Tear down regardless of the precise failure. self.conns[shard_idx] = None if isinstance(e, tablet3.TimeoutError): # On any timeout let the error bubble up and just redial next time. raise e if isinstance(e, tablet3.RetryError): # Give the tablet a moment to restart itself. This isn't # strictly necessary since there is a significant chance you # will end up talking to another host. time.sleep(self.reconnect_delay) raise dbexceptions.OperationalError('tablets unreachable', self.keyspace_name, shard_idx, self.db_type)