def test__mutate_rows_request(self): from google.cloud.bigtable.row import DirectRow table = mock.Mock(name='table', spec=['name']) table.name = 'table' rows = [ DirectRow(row_key=b'row_key', table=table), DirectRow(row_key=b'row_key_2', table=table) ] rows[0].set_cell('cf1', b'c1', b'1') rows[1].set_cell('cf1', b'c1', b'2') result = self._call_fut('table', rows) expected_result = _mutate_rows_request_pb(table_name='table') entry1 = expected_result.entries.add() entry1.row_key = b'row_key' mutations1 = entry1.mutations.add() mutations1.set_cell.family_name = 'cf1' mutations1.set_cell.column_qualifier = b'c1' mutations1.set_cell.timestamp_micros = -1 mutations1.set_cell.value = b'1' entry2 = expected_result.entries.add() entry2.row_key = b'row_key_2' mutations2 = entry2.mutations.add() mutations2.set_cell.family_name = 'cf1' mutations2.set_cell.column_qualifier = b'c1' mutations2.set_cell.timestamp_micros = -1 mutations2.set_cell.value = b'2' self.assertEqual(result, expected_result)
def test_add_row_with_max_flush_count(self): table = _Table(self.TABLE_NAME) mutation_batcher = MutationsBatcher(table=table, flush_count=3) row_1 = DirectRow(row_key=b"row_key_1") row_2 = DirectRow(row_key=b"row_key_2") row_3 = DirectRow(row_key=b"row_key_3") mutation_batcher.mutate(row_1) mutation_batcher.mutate(row_2) mutation_batcher.mutate(row_3) self.assertEqual(table.mutation_calls, 1)
def test_mutation_batcher_mutate_w_max_flush_count(): table = _Table(TABLE_NAME) mutation_batcher = _make_mutation_batcher(table=table, flush_count=3) row_1 = DirectRow(row_key=b"row_key_1") row_2 = DirectRow(row_key=b"row_key_2") row_3 = DirectRow(row_key=b"row_key_3") mutation_batcher.mutate(row_1) mutation_batcher.mutate(row_2) mutation_batcher.mutate(row_3) assert table.mutation_calls == 1
def test_mutate_row_with_max_mutations(self): table = _Table(self.TABLE_NAME) mutation_batcher = MutationsBatcher(table=table) row = DirectRow(row_key=b'row_key') row.set_cell('cf1', b'c1', 1) row.set_cell('cf1', b'c2', 2) row.set_cell('cf1', b'c3', 3) mutation_batcher.mutate(row) mutation_batcher.flush() self.assertEqual(table.mutation_calls, 1)
def test_mutate_row_with_max_mutations(self): table = _Table(self.TABLE_NAME) mutation_batcher = MutationsBatcher(table=table) row = DirectRow(row_key=b"row_key") row.set_cell("cf1", b"c1", 1) row.set_cell("cf1", b"c2", 2) row.set_cell("cf1", b"c3", 3) mutation_batcher.mutate(row) mutation_batcher.flush() self.assertEqual(table.mutation_calls, 1)
def test__mutate_rows_too_many_mutations(self): from google.cloud.bigtable.row import DirectRow from google.cloud.bigtable.table import TooManyMutationsError table = mock.Mock(name='table', spec=['name']) table.name = 'table' rows = [DirectRow(row_key=b'row_key', table=table), DirectRow(row_key=b'row_key_2', table=table)] rows[0].set_cell('cf1', b'c1', 1) rows[0].set_cell('cf1', b'c1', 2) rows[1].set_cell('cf1', b'c1', 3) rows[1].set_cell('cf1', b'c1', 4) with self.assertRaises(TooManyMutationsError): self._call_fut('table', rows)
def test_do_mutate_retryable_rows(self): from google.cloud.bigtable.row import DirectRow from tests.unit._testing import _FakeStub # Setup: # - Mutate 2 rows. # Action: # - Initial attempt will mutate all 2 rows. # Expectation: # - Expect [success, non-retryable] client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client) table = self._make_table(self.TABLE_ID, instance) row_1 = DirectRow(row_key=b'row_key', table=table) row_1.set_cell('cf', b'col', b'value1') row_2 = DirectRow(row_key=b'row_key_2', table=table) row_2.set_cell('cf', b'col', b'value2') response = self._make_responses([self.SUCCESS, self.NON_RETRYABLE]) # Patch the stub used by the API method. client._data_stub = _FakeStub([response]) worker = self._make_worker(table._instance._client, table.name, [row_1, row_2]) statuses = worker._do_mutate_retryable_rows() result = [status.code for status in statuses] expected_result = [self.SUCCESS, self.NON_RETRYABLE] self.assertEqual(result, expected_result)
def test_do_mutate_retryable_rows_second_try_no_retryable(self): from google.cloud.bigtable.row import DirectRow from tests.unit._testing import _FakeStub # Setup: # - Mutate 2 rows. # - First try results: [success, non-retryable] # Action: # - Second try has no row to retry. # Expectation: # - After second try: [success, non-retryable] client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client) table = self._make_table(self.TABLE_ID, instance) row_1 = DirectRow(row_key=b'row_key', table=table) row_1.set_cell('cf', b'col', b'value1') row_2 = DirectRow(row_key=b'row_key_2', table=table) row_2.set_cell('cf', b'col', b'value2') worker = self._make_worker(table._instance._client, table.name, [row_1, row_2]) worker.responses_statuses = self._make_responses_statuses( [self.SUCCESS, self.NON_RETRYABLE]) statuses = worker._do_mutate_retryable_rows() result = [status.code for status in statuses] expected_result = [self.SUCCESS, self.NON_RETRYABLE] self.assertEqual(result, expected_result)
def test_mutate_row_with_max_mutations_failure(self): from google.cloud.bigtable.batcher import MaxMutationsError table = _Table(self.TABLE_NAME) mutation_batcher = MutationsBatcher(table=table) row = DirectRow(row_key=b'row_key') row.set_cell('cf1', b'c1', 1) row.set_cell('cf1', b'c2', 2) row.set_cell('cf1', b'c3', 3) row.set_cell('cf1', b'c4', 4) with self.assertRaises(MaxMutationsError): mutation_batcher.mutate(row)
def test_do_mutate_retryable_rows_mismatch_num_responses(self): from google.cloud.bigtable._generated.bigtable_pb2 import ( MutateRowsResponse) from google.cloud.bigtable.row import DirectRow from google.rpc.status_pb2 import Status from tests.unit._testing import _FakeStub client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client) table = self._make_table(self.TABLE_ID, instance) row_1 = DirectRow(row_key=b'row_key', table=table) row_1.set_cell('cf', b'col', b'value1') row_2 = DirectRow(row_key=b'row_key_2', table=table) row_2.set_cell('cf', b'col', b'value2') response = MutateRowsResponse(entries=[ MutateRowsResponse.Entry( index=0, status=Status(code=0), ), ], ) # Patch the stub used by the API method. client._data_stub = _FakeStub([response]) worker = self._make_worker(table._instance._client, table.name, [row_1, row_2]) with self.assertRaises(RuntimeError): worker._do_mutate_retryable_rows()
def test_mutate_row_with_max_mutations_failure(self): from google.cloud.bigtable.batcher import MaxMutationsError table = _Table(self.TABLE_NAME) mutation_batcher = MutationsBatcher(table=table) row = DirectRow(row_key=b"row_key") row.set_cell("cf1", b"c1", 1) row.set_cell("cf1", b"c2", 2) row.set_cell("cf1", b"c3", 3) row.set_cell("cf1", b"c4", 4) with self.assertRaises(MaxMutationsError): mutation_batcher.mutate(row)
def test_mutate_rows(self): from google.cloud.bigtable._generated.bigtable_pb2 import ( MutateRowsResponse) from google.cloud.bigtable.row import DirectRow from google.rpc.status_pb2 import Status from tests.unit._testing import _FakeStub client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client) table = self._make_one(self.TABLE_ID, instance) row_1 = DirectRow(row_key=b'row_key', table=table) row_1.set_cell('cf', b'col', b'value1') row_2 = DirectRow(row_key=b'row_key_2', table=table) row_2.set_cell('cf', b'col', b'value2') response = MutateRowsResponse(entries=[ MutateRowsResponse.Entry( index=0, status=Status(code=0), ), MutateRowsResponse.Entry( index=1, status=Status(code=1), ), ], ) # Patch the stub used by the API method. client._data_stub = _FakeStub([response]) statuses = table.mutate_rows([row_1, row_2]) result = [status.code for status in statuses] expected_result = [0, 1] self.assertEqual(result, expected_result)
def test_do_mutate_retryable_rows_second_try_no_retryable(self): from google.cloud.bigtable.row import DirectRow # Setup: # - Mutate 2 rows. # - First try results: [success, non-retryable] # Action: # - Second try has no row to retry. # Expectation: # - After second try: [success, non-retryable] channel = self._make_channel() client = self._make_client(project='project-id', channel=channel, admin=True) instance = client.instance(instance_id=self.INSTANCE_ID) table = self._make_table(self.TABLE_ID, instance) row_1 = DirectRow(row_key=b'row_key', table=table) row_1.set_cell('cf', b'col', b'value1') row_2 = DirectRow(row_key=b'row_key_2', table=table) row_2.set_cell('cf', b'col', b'value2') worker = self._make_worker(client, table.name, [row_1, row_2]) worker.responses_statuses = self._make_responses_statuses( [self.SUCCESS, self.NON_RETRYABLE]) statuses = worker._do_mutate_retryable_rows() result = [status.code for status in statuses] expected_result = [self.SUCCESS, self.NON_RETRYABLE] self.assertEqual(result, expected_result)
def test_do_mutate_retryable_rows(self): from google.cloud.bigtable.row import DirectRow # Setup: # - Mutate 2 rows. # Action: # - Initial attempt will mutate all 2 rows. # Expectation: # - Expect [success, non-retryable] channel = self._make_channel() client = self._make_client(project='project-id', channel=channel, admin=True) instance = client.instance(instance_id=self.INSTANCE_ID) table = self._make_table(self.TABLE_ID, instance) row_1 = DirectRow(row_key=b'row_key', table=table) row_1.set_cell('cf', b'col', b'value1') row_2 = DirectRow(row_key=b'row_key_2', table=table) row_2.set_cell('cf', b'col', b'value2') response = self._make_responses([self.SUCCESS, self.NON_RETRYABLE]) # Patch the stub used by the API method. bigtable_stub = client._table_data_client.bigtable_stub bigtable_stub.MutateRows.side_effect = [[response]] worker = self._make_worker(client, table.name, [row_1, row_2]) statuses = worker._do_mutate_retryable_rows() result = [status.code for status in statuses] expected_result = [self.SUCCESS, self.NON_RETRYABLE] self.assertEqual(result, expected_result)
def test_mutate_row(self): table = _Table(self.TABLE_NAME) mutation_batcher = MutationsBatcher(table=table) rows = [ DirectRow(row_key=b"row_key"), DirectRow(row_key=b"row_key_2"), DirectRow(row_key=b"row_key_3"), DirectRow(row_key=b"row_key_4"), ] mutation_batcher.mutate_rows(rows) mutation_batcher.flush() self.assertEqual(table.mutation_calls, 1)
def test_mutation_batcher_mutate_row(): table = _Table(TABLE_NAME) mutation_batcher = _make_mutation_batcher(table=table) rows = [ DirectRow(row_key=b"row_key"), DirectRow(row_key=b"row_key_2"), DirectRow(row_key=b"row_key_3"), DirectRow(row_key=b"row_key_4"), ] mutation_batcher.mutate_rows(rows) mutation_batcher.flush() assert table.mutation_calls == 1
def test_mutate_row_with_max_row_bytes(self): table = _Table(self.TABLE_NAME) mutation_batcher = MutationsBatcher(table=table, max_row_bytes=3 * 1024 * 1024) number_of_bytes = 1 * 1024 * 1024 max_value = b"1" * number_of_bytes row = DirectRow(row_key=b"row_key") row.set_cell("cf1", b"c1", max_value) row.set_cell("cf1", b"c2", max_value) row.set_cell("cf1", b"c3", max_value) mutation_batcher.mutate(row) self.assertEqual(table.mutation_calls, 1)
def row(self, row_key, filter_=None, append=False): """Factory to create a row associated with this table. .. warning:: At most one of ``filter_`` and ``append`` can be used in a :class:`Row`. :type row_key: bytes :param row_key: The key for the row being created. :type filter_: :class:`.RowFilter` :param filter_: (Optional) Filter to be used for conditional mutations. See :class:`.DirectRow` for more details. :type append: bool :param append: (Optional) Flag to determine if the row should be used for append mutations. :rtype: :class:`.DirectRow` :returns: A row owned by this table. :raises: :class:`ValueError <exceptions.ValueError>` if both ``filter_`` and ``append`` are used. """ if append and filter_ is not None: raise ValueError('At most one of filter_ and append can be set') if append: return AppendRow(row_key, self) elif filter_ is not None: return ConditionalRow(row_key, self, filter_=filter_) else: return DirectRow(row_key, self)
def test_mutate_row_with_max_row_bytes(self): table = _Table(self.TABLE_NAME) mutation_batcher = MutationsBatcher(table=table, max_row_bytes=3 * 1024 * 1024) number_of_bytes = 1 * 1024 * 1024 max_value = b'1' * number_of_bytes row = DirectRow(row_key=b'row_key') row.set_cell('cf1', b'c1', max_value) row.set_cell('cf1', b'c2', max_value) row.set_cell('cf1', b'c3', max_value) mutation_batcher.mutate(row) self.assertEqual(table.mutation_calls, 1)
def test_right_table_name(self): from google.cloud.bigtable.row import DirectRow table = mock.Mock(name='table', spec=['name']) table.name = 'table' row = DirectRow(row_key=b'row_key', table=table) result = self._call_fut('table', row) self.assertFalse(result)
def test_wrong_table_name(self): from google.cloud.bigtable.table import TableMismatchError from google.cloud.bigtable.row import DirectRow table = mock.Mock(name='table', spec=['name']) table.name = 'table' row = DirectRow(row_key=b'row_key', table=table) with self.assertRaises(TableMismatchError): self._call_fut('other_table', row)
def test_callable_retry_timeout(self): from google.api_core.retry import Retry from google.cloud.bigtable._generated.bigtable_pb2 import ( MutateRowsResponse) from google.cloud.bigtable.row import DirectRow from google.cloud.bigtable.table import DEFAULT_RETRY from google.rpc.status_pb2 import Status # Setup: # - Mutate 2 rows. # Action: # - Initial attempt will mutate all 2 rows. # Expectation: # - Both rows always return retryable errors. # - google.api_core.Retry should keep retrying. # - Check MutateRows is called multiple times. # - By the time deadline is reached, statuses should be # [retryable, retryable] client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client) table = self._make_table(self.TABLE_ID, instance) row_1 = DirectRow(row_key=b'row_key', table=table) row_1.set_cell('cf', b'col', b'value1') row_2 = DirectRow(row_key=b'row_key_2', table=table) row_2.set_cell('cf', b'col', b'value2') response = MutateRowsResponse(entries=[ MutateRowsResponse.Entry( index=0, status=Status(code=4), ), MutateRowsResponse.Entry( index=1, status=Status(code=4), ), ], ) # Patch the stub used by the API method. client._data_stub = mock.MagicMock() client._data_stub.MutateRows.return_value = [response] retry = DEFAULT_RETRY.with_delay(initial=0.1, maximum=0.2, multiplier=2.0).with_deadline(0.5) worker = self._make_worker(client, table.name, [row_1, row_2]) statuses = worker(retry=retry) result = [status.code for status in statuses] expected_result = [4, 4] self.assertTrue(client._data_stub.MutateRows.call_count > 1) self.assertEqual(result, expected_result)
def test_callable_retry(self): from google.cloud.bigtable.row import DirectRow from google.cloud.bigtable.table import DEFAULT_RETRY # Setup: # - Mutate 3 rows. # Action: # - Initial attempt will mutate all 3 rows. # Expectation: # - First attempt will result in one retryable error. # - Second attempt will result in success for the retry-ed row. # - Check MutateRows is called twice. # - State of responses_statuses should be # [success, success, non-retryable] client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client) table = self._make_table(self.TABLE_ID, instance) row_1 = DirectRow(row_key=b'row_key', table=table) row_1.set_cell('cf', b'col', b'value1') row_2 = DirectRow(row_key=b'row_key_2', table=table) row_2.set_cell('cf', b'col', b'value2') row_3 = DirectRow(row_key=b'row_key_3', table=table) row_3.set_cell('cf', b'col', b'value3') response_1 = self._make_responses([ self.SUCCESS, self.RETRYABLE_1, self.NON_RETRYABLE]) response_2 = self._make_responses([self.SUCCESS]) # Patch the stub used by the API method. client._data_stub = mock.MagicMock() client._data_stub.MutateRows.side_effect = [[response_1], [response_2]] retry = DEFAULT_RETRY.with_delay(initial=0.1) worker = self._make_worker(client, table.name, [row_1, row_2, row_3]) statuses = worker(retry=retry) result = [status.code for status in statuses] expected_result = [self.SUCCESS, self.SUCCESS, self.NON_RETRYABLE] client._data_stub.MutateRows.assert_has_calls([ mock.call(mock.ANY), mock.call(mock.ANY)]) self.assertEqual(client._data_stub.MutateRows.call_count, 2) self.assertEqual(result, expected_result)
def test_do_mutate_retryable_rows(self): from google.cloud.bigtable.row import DirectRow from tests.unit._testing import _FakeStub # Setup: # - Mutate 2 rows. # Action: # - Initial attempt will mutate all 2 rows. # Expectation: # - Expect [success, non-retryable] client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client) table = self._make_table(self.TABLE_ID, instance) row_1 = DirectRow(row_key=b'row_key', table=table) row_1.set_cell('cf', b'col', b'value1') row_2 = DirectRow(row_key=b'row_key_2', table=table) row_2.set_cell('cf', b'col', b'value2') response = self._make_responses([self.SUCCESS, self.NON_RETRYABLE]) # Patch the stub used by the API method. client._data_stub = _FakeStub([response]) worker = self._make_worker( table._instance._client, table.name, [row_1, row_2]) statuses = worker._do_mutate_retryable_rows() result = [status.code for status in statuses] expected_result = [self.SUCCESS, self.NON_RETRYABLE] self.assertEqual(result, expected_result)
def test_do_mutate_retryable_rows_second_try_no_retryable(self): from google.cloud.bigtable.row import DirectRow # Setup: # - Mutate 2 rows. # - First try results: [success, non-retryable] # Action: # - Second try has no row to retry. # Expectation: # - After second try: [success, non-retryable] client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client) table = self._make_table(self.TABLE_ID, instance) row_1 = DirectRow(row_key=b'row_key', table=table) row_1.set_cell('cf', b'col', b'value1') row_2 = DirectRow(row_key=b'row_key_2', table=table) row_2.set_cell('cf', b'col', b'value2') worker = self._make_worker( table._instance._client, table.name, [row_1, row_2]) worker.responses_statuses = self._make_responses_statuses( [self.SUCCESS, self.NON_RETRYABLE]) statuses = worker._do_mutate_retryable_rows() result = [status.code for status in statuses] expected_result = [self.SUCCESS, self.NON_RETRYABLE] self.assertEqual(result, expected_result)
def test_do_mutate_retryable_rows_retry(self): from google.cloud.bigtable.row import DirectRow from google.cloud.bigtable.table import _BigtableRetryableError from tests.unit._testing import _FakeStub # Setup: # - Mutate 3 rows. # Action: # - Initial attempt will mutate all 3 rows. # Expectation: # - Second row returns retryable error code, so expect a raise. # - State of responses_statuses should be # [success, retryable, non-retryable] client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client) table = self._make_table(self.TABLE_ID, instance) row_1 = DirectRow(row_key=b'row_key', table=table) row_1.set_cell('cf', b'col', b'value1') row_2 = DirectRow(row_key=b'row_key_2', table=table) row_2.set_cell('cf', b'col', b'value2') row_3 = DirectRow(row_key=b'row_key_3', table=table) row_3.set_cell('cf', b'col', b'value3') response = self._make_responses([ self.SUCCESS, self.RETRYABLE_1, self.NON_RETRYABLE]) # Patch the stub used by the API method. client._data_stub = _FakeStub([response]) worker = self._make_worker( table._instance._client, table.name, [row_1, row_2, row_3]) with self.assertRaises(_BigtableRetryableError): worker._do_mutate_retryable_rows() statuses = worker.responses_statuses result = [status.code for status in statuses] expected_result = [self.SUCCESS, self.RETRYABLE_1, self.NON_RETRYABLE] self.assertEqual(result, expected_result)
def test_callable_no_retry_strategy(self): from google.cloud.bigtable.row import DirectRow # Setup: # - Mutate 3 rows. # Action: # - Attempt to mutate the rows w/o any retry strategy. # Expectation: # - Since no retry, should return statuses as they come back. # - Even if there are retryable errors, no retry attempt is made. # - State of responses_statuses should be # [success, retryable, non-retryable] client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client) table = self._make_table(self.TABLE_ID, instance) row_1 = DirectRow(row_key=b'row_key', table=table) row_1.set_cell('cf', b'col', b'value1') row_2 = DirectRow(row_key=b'row_key_2', table=table) row_2.set_cell('cf', b'col', b'value2') row_3 = DirectRow(row_key=b'row_key_3', table=table) row_3.set_cell('cf', b'col', b'value3') response = self._make_responses([ self.SUCCESS, self.RETRYABLE_1, self.NON_RETRYABLE]) # Patch the stub used by the API method. client._data_stub = mock.MagicMock() client._data_stub.MutateRows.return_value = [response] worker = self._make_worker(client, table.name, [row_1, row_2, row_3]) statuses = worker(retry=None) result = [status.code for status in statuses] expected_result = [self.SUCCESS, self.RETRYABLE_1, self.NON_RETRYABLE] client._data_stub.MutateRows.assert_called_once() self.assertEqual(result, expected_result)
def test_do_mutate_retryable_rows(self): from google.cloud.bigtable._generated.bigtable_pb2 import ( MutateRowsResponse) from google.cloud.bigtable.row import DirectRow from google.rpc.status_pb2 import Status from tests.unit._testing import _FakeStub # Setup: # - Mutate 2 rows. # Action: # - Initial attempt will mutate all 2 rows. # Expectation: # - Expect [success, non-retryable] client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client) table = self._make_table(self.TABLE_ID, instance) row_1 = DirectRow(row_key=b'row_key', table=table) row_1.set_cell('cf', b'col', b'value1') row_2 = DirectRow(row_key=b'row_key_2', table=table) row_2.set_cell('cf', b'col', b'value2') response = MutateRowsResponse(entries=[ MutateRowsResponse.Entry( index=0, status=Status(code=0), ), MutateRowsResponse.Entry( index=1, status=Status(code=1), ), ], ) # Patch the stub used by the API method. client._data_stub = _FakeStub([response]) worker = self._make_worker(table._instance._client, table.name, [row_1, row_2]) statuses = worker._do_mutate_retryable_rows() result = [status.code for status in statuses] expected_result = [0, 1] self.assertEqual(result, expected_result)
def direct_row(self, row_key): """Create a :class:`~google.cloud.bigtable.row.DirectRow` associated with this table. For example: .. literalinclude:: snippets_table.py :start-after: [START bigtable_table_direct_row] :end-before: [END bigtable_table_direct_row] Args: row_key (bytes): The key for the row being created. Returns: A row owned by this table. """ return DirectRow(row_key, self)
def row(self, row_key, filter_=None, append=False): """Factory to create a row associated with this table. For example: .. literalinclude:: snippets_table.py :start-after: [START bigtable_table_row] :end-before: [END bigtable_table_row] .. warning:: At most one of ``filter_`` and ``append`` can be used in a :class:`~google.cloud.bigtable.row.Row`. :type row_key: bytes :param row_key: The key for the row being created. :type filter_: :class:`.RowFilter` :param filter_: (Optional) Filter to be used for conditional mutations. See :class:`.ConditionalRow` for more details. :type append: bool :param append: (Optional) Flag to determine if the row should be used for append mutations. :rtype: :class:`~google.cloud.bigtable.row.Row` :returns: A row owned by this table. :raises: :class:`ValueError <exceptions.ValueError>` if both ``filter_`` and ``append`` are used. """ warnings.warn( "This method will be deprecated in future versions. Please " "use Table.append_row(), Table.conditional_row() " "and Table.direct_row() methods instead.", PendingDeprecationWarning, stacklevel=2, ) if append and filter_ is not None: raise ValueError("At most one of filter_ and append can be set") if append: return AppendRow(row_key, self) elif filter_ is not None: return ConditionalRow(row_key, self, filter_=filter_) else: return DirectRow(row_key, self)
def test_callable_retry_timeout(self): from google.cloud.bigtable.row import DirectRow from google.cloud.bigtable.table import DEFAULT_RETRY # Setup: # - Mutate 2 rows. # Action: # - Initial attempt will mutate all 2 rows. # Expectation: # - Both rows always return retryable errors. # - google.api_core.Retry should keep retrying. # - Check MutateRows is called multiple times. # - By the time deadline is reached, statuses should be # [retryable, retryable] channel = self._make_channel() client = self._make_client(project='project-id', channel=channel, admin=True) instance = client.instance(instance_id=self.INSTANCE_ID) table = self._make_table(self.TABLE_ID, instance) row_1 = DirectRow(row_key=b'row_key', table=table) row_1.set_cell('cf', b'col', b'value1') row_2 = DirectRow(row_key=b'row_key_2', table=table) row_2.set_cell('cf', b'col', b'value2') response = self._make_responses([self.RETRYABLE_1, self.RETRYABLE_1]) # Patch the stub used by the API method. bigtable_stub = client._table_data_client.bigtable_stub bigtable_stub.MutateRows.return_value = [response] retry = DEFAULT_RETRY.with_delay( initial=0.1, maximum=0.2, multiplier=2.0).with_deadline(0.5) worker = self._make_worker(client, table.name, [row_1, row_2]) statuses = worker(retry=retry) result = [status.code for status in statuses] expected_result = [self.RETRYABLE_1, self.RETRYABLE_1] self.assertTrue( client._table_data_client.bigtable_stub.MutateRows.call_count > 1) self.assertEqual(result, expected_result)
def test_mutation_batcher_mutate_w_max_mutations(): table = _Table(TABLE_NAME) mutation_batcher = _make_mutation_batcher(table=table) row = DirectRow(row_key=b"row_key") row.set_cell("cf1", b"c1", 1) row.set_cell("cf1", b"c2", 2) row.set_cell("cf1", b"c3", 3) mutation_batcher.mutate(row) mutation_batcher.flush() assert table.mutation_calls == 1
def test_do_mutate_retryable_rows_mismatch_num_responses(self): from google.cloud.bigtable.row import DirectRow from tests.unit._testing import _FakeStub client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client) table = self._make_table(self.TABLE_ID, instance) row_1 = DirectRow(row_key=b'row_key', table=table) row_1.set_cell('cf', b'col', b'value1') row_2 = DirectRow(row_key=b'row_key_2', table=table) row_2.set_cell('cf', b'col', b'value2') response = self._make_responses([self.SUCCESS]) # Patch the stub used by the API method. client._data_stub = _FakeStub([response]) worker = self._make_worker(table._instance._client, table.name, [row_1, row_2]) with self.assertRaises(RuntimeError): worker._do_mutate_retryable_rows()
def test_do_mutate_retryable_rows_mismatch_num_responses(self): from google.cloud.bigtable.row import DirectRow channel = self._make_channel() client = self._make_client(project='project-id', channel=channel, admin=True) instance = client.instance(instance_id=self.INSTANCE_ID) table = self._make_table(self.TABLE_ID, instance) row_1 = DirectRow(row_key=b'row_key', table=table) row_1.set_cell('cf', b'col', b'value1') row_2 = DirectRow(row_key=b'row_key_2', table=table) row_2.set_cell('cf', b'col', b'value2') response = self._make_responses([self.SUCCESS]) # Patch the stub used by the API method. bigtable_stub = client._table_data_client.bigtable_stub bigtable_stub.MutateRows.side_effect = [[response]] worker = self._make_worker(client, table.name, [row_1, row_2]) with self.assertRaises(RuntimeError): worker._do_mutate_retryable_rows()
def test_mutation_batcher_mutate_w_max_row_bytes(): table = _Table(TABLE_NAME) mutation_batcher = _make_mutation_batcher(table=table, max_row_bytes=3 * 1024 * 1024) number_of_bytes = 1 * 1024 * 1024 max_value = b"1" * number_of_bytes row = DirectRow(row_key=b"row_key") row.set_cell("cf1", b"c1", max_value) row.set_cell("cf1", b"c2", max_value) row.set_cell("cf1", b"c3", max_value) mutation_batcher.mutate(row) assert table.mutation_calls == 1
def test_callable_retry_timeout(self): from google.cloud.bigtable.row import DirectRow from google.cloud.bigtable.table import DEFAULT_RETRY # Setup: # - Mutate 2 rows. # Action: # - Initial attempt will mutate all 2 rows. # Expectation: # - Both rows always return retryable errors. # - google.api_core.Retry should keep retrying. # - Check MutateRows is called multiple times. # - By the time deadline is reached, statuses should be # [retryable, retryable] client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client) table = self._make_table(self.TABLE_ID, instance) row_1 = DirectRow(row_key=b'row_key', table=table) row_1.set_cell('cf', b'col', b'value1') row_2 = DirectRow(row_key=b'row_key_2', table=table) row_2.set_cell('cf', b'col', b'value2') response = self._make_responses([self.RETRYABLE_1, self.RETRYABLE_1]) # Patch the stub used by the API method. client._data_stub = mock.MagicMock() client._data_stub.MutateRows.return_value = [response] retry = DEFAULT_RETRY.with_delay( initial=0.1, maximum=0.2, multiplier=2.0).with_deadline(0.5) worker = self._make_worker(client, table.name, [row_1, row_2]) statuses = worker(retry=retry) result = [status.code for status in statuses] expected_result = [self.RETRYABLE_1, self.RETRYABLE_1] self.assertTrue(client._data_stub.MutateRows.call_count > 1) self.assertEqual(result, expected_result)
def test_do_mutate_retryable_rows_mismatch_num_responses(self): from google.cloud.bigtable.row import DirectRow from tests.unit._testing import _FakeStub client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client) table = self._make_table(self.TABLE_ID, instance) row_1 = DirectRow(row_key=b'row_key', table=table) row_1.set_cell('cf', b'col', b'value1') row_2 = DirectRow(row_key=b'row_key_2', table=table) row_2.set_cell('cf', b'col', b'value2') response = self._make_responses([self.SUCCESS]) # Patch the stub used by the API method. client._data_stub = _FakeStub([response]) worker = self._make_worker( table._instance._client, table.name, [row_1, row_2]) with self.assertRaises(RuntimeError): worker._do_mutate_retryable_rows()
def test_do_mutate_retryable_rows_second_try(self): from google.cloud.bigtable.row import DirectRow from tests.unit._testing import _FakeStub # Setup: # - Mutate 4 rows. # - First try results: # [success, retryable, non-retryable, retryable] # Action: # - Second try should re-attempt the 'retryable' rows. # Expectation: # - After second try: # [success, non-retryable, non-retryable, success] client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client) table = self._make_table(self.TABLE_ID, instance) row_1 = DirectRow(row_key=b'row_key', table=table) row_1.set_cell('cf', b'col', b'value1') row_2 = DirectRow(row_key=b'row_key_2', table=table) row_2.set_cell('cf', b'col', b'value2') row_3 = DirectRow(row_key=b'row_key_3', table=table) row_3.set_cell('cf', b'col', b'value3') row_4 = DirectRow(row_key=b'row_key_4', table=table) row_4.set_cell('cf', b'col', b'value4') response = self._make_responses([self.NON_RETRYABLE, self.SUCCESS]) # Patch the stub used by the API method. client._data_stub = _FakeStub([response]) worker = self._make_worker( table._instance._client, table.name, [row_1, row_2, row_3, row_4]) worker.responses_statuses = self._make_responses_statuses([ self.SUCCESS, self.RETRYABLE_1, self.NON_RETRYABLE, self.RETRYABLE_2]) statuses = worker._do_mutate_retryable_rows() result = [status.code for status in statuses] expected_result = [self.SUCCESS, self.NON_RETRYABLE, self.NON_RETRYABLE, self.SUCCESS] self.assertEqual(result, expected_result)
def test_do_mutate_retryable_rows_second_retry(self): from google.cloud.bigtable.row import DirectRow from google.cloud.bigtable.table import _BigtableRetryableError from tests.unit._testing import _FakeStub # Setup: # - Mutate 4 rows. # - First try results: # [success, retryable, non-retryable, retryable] # Action: # - Second try should re-attempt the 'retryable' rows. # Expectation: # - After second try: # [success, success, non-retryable, retryable] # - One of the rows tried second time returns retryable error code, # so expect a raise. # - Exception contains response whose index should be '3' even though # only two rows were retried. client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client) table = self._make_table(self.TABLE_ID, instance) row_1 = DirectRow(row_key=b'row_key', table=table) row_1.set_cell('cf', b'col', b'value1') row_2 = DirectRow(row_key=b'row_key_2', table=table) row_2.set_cell('cf', b'col', b'value2') row_3 = DirectRow(row_key=b'row_key_3', table=table) row_3.set_cell('cf', b'col', b'value3') row_4 = DirectRow(row_key=b'row_key_4', table=table) row_4.set_cell('cf', b'col', b'value4') response = self._make_responses([self.SUCCESS, self.RETRYABLE_1]) # Patch the stub used by the API method. client._data_stub = _FakeStub([response]) worker = self._make_worker( table._instance._client, table.name, [row_1, row_2, row_3, row_4]) worker.responses_statuses = self._make_responses_statuses([ self.SUCCESS, self.RETRYABLE_1, self.NON_RETRYABLE, self.RETRYABLE_2]) with self.assertRaises(_BigtableRetryableError): worker._do_mutate_retryable_rows() statuses = worker.responses_statuses result = [status.code for status in statuses] expected_result = [self.SUCCESS, self.SUCCESS, self.NON_RETRYABLE, self.RETRYABLE_1] self.assertEqual(result, expected_result)