def read_row(self, row_key, filter_=None): """Read a single row from this table. :type row_key: bytes :param row_key: The key of the row to read from. :type filter_: :class:`.RowFilter` :param filter_: (Optional) The filter to apply to the contents of the row. If unset, returns the entire row. :rtype: :class:`.PartialRowData`, :data:`NoneType <types.NoneType>` :returns: The contents of the row if any chunks were returned in the response, otherwise :data:`None`. :raises: :class:`ValueError <exceptions.ValueError>` if a commit row chunk is never encountered. """ request_pb = _create_row_request(self.name, row_key=row_key, filter_=filter_) client = self._cluster._client response_iterator = client._data_stub.ReadRows(request_pb, client.timeout_seconds) # We expect an iterator of `data_messages_pb2.ReadRowsResponse` result = PartialRowData(row_key) for read_rows_response in response_iterator: result.update_from_read_rows(read_rows_response) # Make sure the result actually contains data. if not result._chunks_encountered: return None # Make sure the result was committed by the back-end. if not result.committed: raise ValueError('The row remains partial / is not committed.') return result
def read_row(self, row_key, filter_=None): """Read a single row from this table. :type row_key: bytes :param row_key: The key of the row to read from. :type filter_: :class:`.RowFilter` :param filter_: (Optional) The filter to apply to the contents of the row. If unset, returns the entire row. :rtype: :class:`.PartialRowData`, :data:`NoneType <types.NoneType>` :returns: The contents of the row if any chunks were returned in the response, otherwise :data:`None`. :raises: :class:`ValueError <exceptions.ValueError>` if a commit row chunk is never encountered. """ request_pb = _create_row_request(self.name, row_key=row_key, filter_=filter_) client = self._cluster._client response_iterator = client._data_stub.ReadRows(request_pb, client.timeout_seconds) # We expect an iterator of `data_messages_pb2.ReadRowsResponse` result = PartialRowData(row_key) for read_rows_response in response_iterator: result.update_from_read_rows(read_rows_response) # Make sure the result actually contains data. if not result._chunks_encountered: return None # Make sure the result was committed by the back-end. if not result.committed: raise ValueError("The row remains partial / is not committed.") return result
def _read_row_helper(self, chunks): from gcloud._testing import _Monkey from gcloud.bigtable._generated import ( bigtable_service_messages_pb2 as messages_pb2) from gcloud.bigtable._testing import _FakeStub from gcloud.bigtable.row_data import PartialRowData from gcloud.bigtable import table as MUT project_id = 'project-id' zone = 'zone' cluster_id = 'cluster-id' table_id = 'table-id' timeout_seconds = 596 client = _Client(timeout_seconds=timeout_seconds) cluster_name = ('projects/' + project_id + '/zones/' + zone + '/clusters/' + cluster_id) cluster = _Cluster(cluster_name, client=client) table = self._makeOne(table_id, cluster) # Create request_pb request_pb = object() # Returned by our mock. mock_created = [] def mock_create_row_request(table_name, row_key, filter_): mock_created.append((table_name, row_key, filter_)) return request_pb # Create response_iterator row_key = b'row-key' response_pb = messages_pb2.ReadRowsResponse(row_key=row_key, chunks=chunks) response_iterator = [response_pb] # Patch the stub used by the API method. client._data_stub = stub = _FakeStub(response_iterator) # Create expected_result. if chunks: expected_result = PartialRowData(row_key) expected_result._committed = True expected_result._chunks_encountered = True else: expected_result = None # Perform the method and check the result. filter_obj = object() with _Monkey(MUT, _create_row_request=mock_create_row_request): result = table.read_row(row_key, filter_=filter_obj) self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'ReadRows', (request_pb, timeout_seconds), {}, )]) self.assertEqual(mock_created, [(table.name, row_key, filter_obj)])
def test_row_with_results(self): from gcloud._testing import _Monkey from gcloud.bigtable.happybase import table as MUT from gcloud.bigtable.row_data import PartialRowData row_key = 'row-key' name = 'table-name' connection = None table = self._makeOne(name, connection) table._low_level_table = _MockLowLevelTable() partial_row = PartialRowData(row_key) table._low_level_table.read_row_result = partial_row # Set-up mocks. fake_filter = object() mock_filters = [] def mock_filter_chain_helper(**kwargs): mock_filters.append(kwargs) return fake_filter fake_pair = object() mock_cells = [] def mock_cells_to_pairs(*args, **kwargs): mock_cells.append((args, kwargs)) return [fake_pair] col_fam = u'cf1' qual = b'qual' fake_cells = object() partial_row._cells = {col_fam: {qual: fake_cells}} include_timestamp = object() with _Monkey(MUT, _filter_chain_helper=mock_filter_chain_helper, _cells_to_pairs=mock_cells_to_pairs): result = table.row(row_key, include_timestamp=include_timestamp) # The results come from _cells_to_pairs. expected_result = {col_fam.encode('ascii') + b':' + qual: fake_pair} self.assertEqual(result, expected_result) read_row_args = (row_key,) read_row_kwargs = {'filter_': fake_filter} self.assertEqual(table._low_level_table.read_row_calls, [ (read_row_args, read_row_kwargs), ]) expected_kwargs = { 'filters': [], 'versions': 1, 'timestamp': None, } self.assertEqual(mock_filters, [expected_kwargs]) to_pairs_kwargs = {'include_timestamp': include_timestamp} self.assertEqual(mock_cells, [((fake_cells,), to_pairs_kwargs)])
def test_without_timestamp(self): from gcloud.bigtable.row_data import Cell from gcloud.bigtable.row_data import PartialRowData row_data = PartialRowData(b"row-key") val1 = b"hi-im-bytes" val2 = b"bi-im-hytes" row_data._cells[u"fam1"] = {b"col1": [Cell(val1, None)], b"col2": [Cell(val2, None)]} result = self._callFUT(row_data) expected_result = {b"fam1:col1": val1, b"fam1:col2": val2} self.assertEqual(result, expected_result)
def test_cells_with_results(self): from gcloud._testing import _Monkey from gcloud.bigtable.happybase import table as MUT from gcloud.bigtable.row_data import PartialRowData row_key = "row-key" name = "table-name" connection = None table = self._makeOne(name, connection) table._low_level_table = _MockLowLevelTable() partial_row = PartialRowData(row_key) table._low_level_table.read_row_result = partial_row # These are all passed to mocks. versions = object() timestamp = object() include_timestamp = object() # Set-up mocks. fake_filter = object() mock_filters = [] def mock_filter_chain_helper(**kwargs): mock_filters.append(kwargs) return fake_filter fake_result = object() mock_cells = [] def mock_cells_to_pairs(*args, **kwargs): mock_cells.append((args, kwargs)) return fake_result col_fam = "cf1" qual = "qual" fake_cells = object() partial_row._cells = {col_fam: {qual: fake_cells}} column = col_fam + ":" + qual with _Monkey(MUT, _filter_chain_helper=mock_filter_chain_helper, _cells_to_pairs=mock_cells_to_pairs): result = table.cells( row_key, column, versions=versions, timestamp=timestamp, include_timestamp=include_timestamp ) self.assertEqual(result, fake_result) read_row_args = (row_key,) read_row_kwargs = {"filter_": fake_filter} self.assertEqual(table._low_level_table.read_row_calls, [(read_row_args, read_row_kwargs)]) filter_kwargs = {"column": column, "versions": versions, "timestamp": timestamp} self.assertEqual(mock_filters, [filter_kwargs]) to_pairs_kwargs = {"include_timestamp": include_timestamp} self.assertEqual(mock_cells, [((fake_cells,), to_pairs_kwargs)])
def test_with_timestamp(self): from gcloud._helpers import _datetime_from_microseconds from gcloud.bigtable.row_data import Cell from gcloud.bigtable.row_data import PartialRowData row_data = PartialRowData(b"row-key") val1 = b"hi-im-bytes" ts1_millis = 1221934570148 ts1 = _datetime_from_microseconds(ts1_millis * 1000) val2 = b"bi-im-hytes" ts2_millis = 1331934880000 ts2 = _datetime_from_microseconds(ts2_millis * 1000) row_data._cells[u"fam1"] = {b"col1": [Cell(val1, ts1)], b"col2": [Cell(val2, ts2)]} result = self._callFUT(row_data, include_timestamp=True) expected_result = {b"fam1:col1": (val1, ts1_millis), b"fam1:col2": (val2, ts2_millis)} self.assertEqual(result, expected_result)
def test_without_timestamp(self): from gcloud.bigtable.row_data import Cell from gcloud.bigtable.row_data import PartialRowData row_data = PartialRowData(b'row-key') val1 = b'hi-im-bytes' val2 = b'bi-im-hytes' row_data._cells[u'fam1'] = { b'col1': [Cell(val1, None)], b'col2': [Cell(val2, None)], } result = self._callFUT(row_data) expected_result = { b'fam1:col1': val1, b'fam1:col2': val2, } self.assertEqual(result, expected_result)
def test_read_rows(self): row = self._table.row(ROW_KEY) row_alt = self._table.row(ROW_KEY_ALT) self.rows_to_delete.extend([row, row_alt]) cell1, cell2, cell3, cell4 = self._write_to_row(row, row_alt, row, row_alt) row.commit() row_alt.commit() rows_data = self._table.read_rows() self.assertEqual(rows_data.rows, {}) rows_data.consume_all() # NOTE: We should refrain from editing protected data on instances. # Instead we should make the values public or provide factories # for constructing objects with them. row_data = PartialRowData(ROW_KEY) row_data._chunks_encountered = True row_data._committed = True row_data._cells = { COLUMN_FAMILY_ID1: { COL_NAME1: [cell1], COL_NAME2: [cell3], }, } row_alt_data = PartialRowData(ROW_KEY_ALT) row_alt_data._chunks_encountered = True row_alt_data._committed = True row_alt_data._cells = { COLUMN_FAMILY_ID1: { COL_NAME1: [cell2], }, COLUMN_FAMILY_ID2: { COL_NAME3: [cell4], }, } expected_rows = { ROW_KEY: row_data, ROW_KEY_ALT: row_alt_data, } self.assertEqual(rows_data.rows, expected_rows)
def test_consume_next(self): from gcloud.bigtable._generated import ( bigtable_service_messages_pb2 as messages_pb2) from gcloud.bigtable.row_data import PartialRowData row_key = b'row-key' value_pb = messages_pb2.ReadRowsResponse(row_key=row_key) response_iterator = _MockCancellableIterator(value_pb) partial_rows_data = self._makeOne(response_iterator) self.assertEqual(partial_rows_data.rows, {}) partial_rows_data.consume_next() expected_rows = {row_key: PartialRowData(row_key)} self.assertEqual(partial_rows_data.rows, expected_rows)
def test_consume_next_row_exists(self): from gcloud.bigtable._generated import ( bigtable_service_messages_pb2 as messages_pb2) from gcloud.bigtable.row_data import PartialRowData row_key = b'row-key' chunk = messages_pb2.ReadRowsResponse.Chunk(commit_row=True) value_pb = messages_pb2.ReadRowsResponse(row_key=row_key, chunks=[chunk]) response_iterator = _MockCancellableIterator(value_pb) partial_rows_data = self._makeOne(response_iterator) existing_values = PartialRowData(row_key) partial_rows_data._rows[row_key] = existing_values self.assertFalse(existing_values.committed) partial_rows_data.consume_next() self.assertTrue(existing_values.committed) self.assertEqual(existing_values.cells, {})
def test_read_row_complete(self): from gcloud.bigtable.row_data import Cell from gcloud.bigtable.row_data import PartialRowData chunk = _ReadRowsResponseCellChunkPB( row_key=self.ROW_KEY, family_name=self.FAMILY_NAME, qualifier=self.QUALIFIER, timestamp_micros=self.TIMESTAMP_MICROS, value=self.VALUE, commit_row=True, ) chunks = [chunk] expected_result = PartialRowData(row_key=self.ROW_KEY) family = expected_result._cells.setdefault(self.FAMILY_NAME, {}) column = family.setdefault(self.QUALIFIER, []) column.append(Cell.from_pb(chunk)) self._read_row_helper(chunks, expected_result)
def test_rows_with_results(self): from gcloud._testing import _Monkey from gcloud.bigtable.happybase import table as MUT from gcloud.bigtable.row_data import PartialRowData row_key1 = 'row-key1' row_key2 = 'row-key2' rows = [row_key1, row_key2] name = 'table-name' connection = None table = self._makeOne(name, connection) table._low_level_table = _MockLowLevelTable() row1 = PartialRowData(row_key1) # Return row1 but not row2 rr_result = _MockPartialRowsData(rows={row_key1: row1}) table._low_level_table.read_rows_result = rr_result self.assertEqual(rr_result.consume_all_calls, 0) # Set-up mocks. fake_rows_filter = object() mock_rows = [] def mock_row_keys_filter_helper(*args): mock_rows.append(args) return fake_rows_filter fake_filter = object() mock_filters = [] def mock_filter_chain_helper(**kwargs): mock_filters.append(kwargs) return fake_filter fake_pair = object() mock_cells = [] def mock_cells_to_pairs(*args, **kwargs): mock_cells.append((args, kwargs)) return [fake_pair] col_fam = u'cf1' qual = b'qual' fake_cells = object() row1._cells = {col_fam: {qual: fake_cells}} include_timestamp = object() with _Monkey(MUT, _row_keys_filter_helper=mock_row_keys_filter_helper, _filter_chain_helper=mock_filter_chain_helper, _cells_to_pairs=mock_cells_to_pairs): result = table.rows(rows, include_timestamp=include_timestamp) # read_rows_result == PartialRowsData with row_key1 expected_result = {col_fam.encode('ascii') + b':' + qual: fake_pair} self.assertEqual(result, [(row_key1, expected_result)]) read_rows_args = () read_rows_kwargs = {'filter_': fake_filter} self.assertEqual(table._low_level_table.read_rows_calls, [ (read_rows_args, read_rows_kwargs), ]) self.assertEqual(rr_result.consume_all_calls, 1) self.assertEqual(mock_rows, [(rows,)]) expected_kwargs = { 'filters': [fake_rows_filter], 'versions': 1, 'timestamp': None, } self.assertEqual(mock_filters, [expected_kwargs]) to_pairs_kwargs = {'include_timestamp': include_timestamp} self.assertEqual(mock_cells, [((fake_cells,), to_pairs_kwargs)])