def append_cell_value(self, column_family_id, column, value): """Appends a value to an existing cell. .. note:: This method adds a read-modify rule protobuf to the accumulated read-modify rules on this :class:`Row`, but does not make an API request. To actually send an API request (with the rules) to the Google Cloud Bigtable API, call :meth:`commit_modifications`. :type column_family_id: str :param column_family_id: The column family that contains the column. Must be of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. :type column: bytes :param column: The column within the column family where the cell is located. :type value: bytes :param value: The value to append to the existing value in the cell. If the targeted cell is unset, it will be treated as containing the empty string. """ column = _to_bytes(column) value = _to_bytes(value) rule_pb = data_pb2.ReadModifyWriteRule(family_name=column_family_id, column_qualifier=column, append_value=value) self._rule_pb_list.append(rule_pb)
def to_pb(self): """Converts the row filter to a protobuf. First converts to a :class:`.data_v2_pb2.ColumnRange` and then uses it in the ``column_range_filter`` field. :rtype: :class:`.data_v2_pb2.RowFilter` :returns: The converted current object. """ column_range_kwargs = {'family_name': self.column_family_id} if self.start_column is not None: if self.inclusive_start: key = 'start_qualifier_closed' else: key = 'start_qualifier_open' column_range_kwargs[key] = _to_bytes(self.start_column) if self.end_column is not None: if self.inclusive_end: key = 'end_qualifier_closed' else: key = 'end_qualifier_open' column_range_kwargs[key] = _to_bytes(self.end_column) column_range = data_v2_pb2.ColumnRange(**column_range_kwargs) return data_v2_pb2.RowFilter(column_range_filter=column_range)
def to_pb(self): """Converts the row filter to a protobuf. First converts to a :class:`.data_v2_pb2.ValueRange` and then uses it to create a row filter protobuf. :rtype: :class:`.data_v2_pb2.RowFilter` :returns: The converted current object. """ value_range_kwargs = {} if self.start_value is not None: if self.inclusive_start: key = 'start_value_closed' else: key = 'start_value_open' value_range_kwargs[key] = _to_bytes(self.start_value) if self.end_value is not None: if self.inclusive_end: key = 'end_value_closed' else: key = 'end_value_open' value_range_kwargs[key] = _to_bytes(self.end_value) value_range = data_v2_pb2.ValueRange(**value_range_kwargs) return data_v2_pb2.RowFilter(value_range_filter=value_range)
def _create_row_request(table_name, row_key=None, start_key=None, end_key=None, filter_=None, limit=None): """Creates a request to read rows in a table. :type table_name: str :param table_name: The name of the table to read from. :type row_key: bytes :param row_key: (Optional) The key of a specific row to read from. :type start_key: bytes :param start_key: (Optional) The beginning of a range of row keys to read from. The range will include ``start_key``. If left empty, will be interpreted as the empty string. :type end_key: bytes :param end_key: (Optional) The end of a range of row keys to read from. The range will not include ``end_key``. If left empty, will be interpreted as an infinite string. :type filter_: :class:`.RowFilter` :param filter_: (Optional) The filter to apply to the contents of the specified row(s). If unset, reads the entire table. :type limit: int :param limit: (Optional) The read will terminate after committing to N rows' worth of results. The default (zero) is to return all results. :rtype: :class:`data_messages_v2_pb2.ReadRowsRequest` :returns: The ``ReadRowsRequest`` protobuf corresponding to the inputs. :raises: :class:`ValueError <exceptions.ValueError>` if both ``row_key`` and one of ``start_key`` and ``end_key`` are set """ request_kwargs = {'table_name': table_name} if (row_key is not None and (start_key is not None or end_key is not None)): raise ValueError('Row key and row range cannot be ' 'set simultaneously') range_kwargs = {} if start_key is not None or end_key is not None: if start_key is not None: range_kwargs['start_key_closed'] = _to_bytes(start_key) if end_key is not None: range_kwargs['end_key_open'] = _to_bytes(end_key) if filter_ is not None: request_kwargs['filter'] = filter_.to_pb() if limit is not None: request_kwargs['rows_limit'] = limit message = data_messages_v2_pb2.ReadRowsRequest(**request_kwargs) if row_key is not None: message.rows.row_keys.append(_to_bytes(row_key)) if range_kwargs: message.rows.row_ranges.add(**range_kwargs) return message
def set_cell(self, column_family_id, column, value, timestamp=None, state=None): """Sets a value in this row. The cell is determined by the ``row_key`` of the :class:`Row` and the ``column``. The ``column`` must be in an existing :class:`.column_family.ColumnFamily` (as determined by ``column_family_id``). .. note:: This method adds a mutation to the accumulated mutations on this :class:`Row`, but does not make an API request. To actually send an API request (with the mutations) to the Google Cloud Bigtable API, call :meth:`commit`. :type column_family_id: str :param column_family_id: The column family that contains the column. Must be of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. :type column: bytes :param column: The column within the column family where the cell is located. :type value: bytes or :class:`int` :param value: The value to set in the cell. If an integer is used, will be interpreted as a 64-bit big-endian signed integer (8 bytes). :type timestamp: :class:`datetime.datetime` :param timestamp: (Optional) The timestamp of the operation. :type state: bool :param state: (Optional) The state that the mutation should be applied in. Unset if the mutation is not conditional, otherwise :data:`True` or :data:`False`. """ column = _to_bytes(column) if isinstance(value, six.integer_types): value = _PACK_I64(value) value = _to_bytes(value) if timestamp is None: # Use -1 for current Bigtable server time. timestamp_micros = -1 else: timestamp_micros = _microseconds_from_datetime(timestamp) # Truncate to millisecond granularity. timestamp_micros -= (timestamp_micros % 1000) mutation_val = data_pb2.Mutation.SetCell( family_name=column_family_id, column_qualifier=column, timestamp_micros=timestamp_micros, value=value, ) mutation_pb = data_pb2.Mutation(set_cell=mutation_val) self._get_mutations(state).append(mutation_pb)
def _set_cell(self, column_family_id, column, value, timestamp=None, state=None): """Helper for :meth:`set_cell` Adds a mutation to set the value in a specific cell. ``state`` is unused by :class:`DirectRow` but is used by subclasses. :type column_family_id: str :param column_family_id: The column family that contains the column. Must be of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. :type column: bytes :param column: The column within the column family where the cell is located. :type value: bytes or :class:`int` :param value: The value to set in the cell. If an integer is used, will be interpreted as a 64-bit big-endian signed integer (8 bytes). :type timestamp: :class:`datetime.datetime` :param timestamp: (Optional) The timestamp of the operation. :type state: bool :param state: (Optional) The state that is passed along to :meth:`_get_mutations`. """ column = _to_bytes(column) if isinstance(value, six.integer_types): value = _PACK_I64(value) value = _to_bytes(value) if timestamp is None: # Use -1 for current Bigtable server time. timestamp_micros = -1 else: timestamp_micros = _microseconds_from_datetime(timestamp) # Truncate to millisecond granularity. timestamp_micros -= (timestamp_micros % 1000) mutation_val = data_v2_pb2.Mutation.SetCell( family_name=column_family_id, column_qualifier=column, timestamp_micros=timestamp_micros, value=value, ) mutation_pb = data_v2_pb2.Mutation(set_cell=mutation_val) self._get_mutations(state).append(mutation_pb)
def _create_test_helper(self, initial_split_keys): from gcloud._helpers import _to_bytes from gcloud.bigtable._testing import _FakeStub client = _Client(timeout_seconds=self.TIMEOUT_SECONDS) instance = _Instance(self.INSTANCE_NAME, client=client) table = self._makeOne(self.TABLE_ID, instance) # Create request_pb splits_pb = [ _CreateTableRequestSplitPB(key=_to_bytes(key)) for key in initial_split_keys or ()] request_pb = _CreateTableRequestPB( initial_splits=splits_pb, parent=self.INSTANCE_NAME, table_id=self.TABLE_ID, ) # Create response_pb response_pb = _TablePB() # Patch the stub used by the API method. client._table_stub = stub = _FakeStub(response_pb) # Create expected_result. expected_result = None # create() has no return value. # Perform the method and check the result. result = table.create(initial_split_keys=initial_split_keys) self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'CreateTable', (request_pb, self.TIMEOUT_SECONDS), {}, )])
def increment_cell_value(self, column_family_id, column, int_value): """Increments a value in an existing cell. Assumes the value in the cell is stored as a 64 bit integer serialized to bytes. .. note:: This method adds a read-modify rule protobuf to the accumulated read-modify rules on this :class:`Row`, but does not make an API request. To actually send an API request (with the rules) to the Google Cloud Bigtable API, call :meth:`commit_modifications`. :type column_family_id: str :param column_family_id: The column family that contains the column. Must be of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. :type column: bytes :param column: The column within the column family where the cell is located. :type int_value: int :param int_value: The value to increment the existing value in the cell by. If the targeted cell is unset, it will be treated as containing a zero. Otherwise, the targeted cell must contain an 8-byte value (interpreted as a 64-bit big-endian signed integer), or the entire request will fail. """ column = _to_bytes(column) rule_pb = data_pb2.ReadModifyWriteRule(family_name=column_family_id, column_qualifier=column, increment_amount=int_value) self._rule_pb_list.append(rule_pb)
class TestVisionRequest(unittest.TestCase): _IMAGE_CONTENT = _to_bytes('/9j/4QNURXhpZgAASUkq') def _getTargetClass(self): from gcloud.vision.client import VisionRequest return VisionRequest def _makeOne(self, *args, **kw): return self._getTargetClass()(*args, **kw) def test_make_vision_request(self): from gcloud.vision.feature import Feature, FeatureTypes feature = Feature(feature_type=FeatureTypes.FACE_DETECTION, max_results=3) vision_request = self._makeOne(self._IMAGE_CONTENT, feature) self.assertEqual(self._IMAGE_CONTENT, vision_request.image) self.assertEqual(FeatureTypes.FACE_DETECTION, vision_request.features[0].feature_type) vision_request = self._makeOne(self._IMAGE_CONTENT, [feature]) self.assertEqual(self._IMAGE_CONTENT, vision_request.image) self.assertEqual(FeatureTypes.FACE_DETECTION, vision_request.features[0].feature_type) with self.assertRaises(TypeError): self._makeOne(self._IMAGE_CONTENT, 'nonsensefeature')
def to_dict(self): """Convert the cells to a dictionary. This is intended to be used with HappyBase, so the column family and column qualiers are combined (with ``:``). :rtype: dict :returns: Dictionary containing all the data in the cells of this row. """ result = {} for column_family_id, columns in six.iteritems(self._cells): for column_qual, cells in six.iteritems(columns): key = (_to_bytes(column_family_id) + b':' + _to_bytes(column_qual)) result[key] = cells return result
def _string_successor(str_val): """Increment and truncate a byte string. Determines shortest string that sorts after the given string when compared using regular string comparison semantics. Modeled after implementation in ``gcloud-golang``. Increments the last byte that is smaller than ``0xFF``, and drops everything after it. If the string only contains ``0xFF`` bytes, ``''`` is returned. :type str_val: str :param str_val: String to increment. :rtype: str :returns: The next string in lexical order after ``str_val``. """ str_val = _to_bytes(str_val, encoding='latin-1') if str_val == b'': return str_val index = len(str_val) - 1 while index >= 0: if six.indexbytes(str_val, index) != 0xff: break index -= 1 if index == -1: return b'' return str_val[:index] + _next_char(str_val, index)
def _message_pb_from_mapping(message): """Helper for :meth:`_PublisherAPI.topic_publish`. Performs "impedance matching" between the protobuf attrs and the keys expected in the JSON API. """ return PubsubMessage(data=_to_bytes(message['data']), attributes=message['attributes'])
def _delete_cells(self, column_family_id, columns, time_range=None, state=None): """Helper for :meth:`delete_cell` and :meth:`delete_cells`. ``state`` is unused by :class:`DirectRow` but is used by subclasses. :type column_family_id: str :param column_family_id: The column family that contains the column or columns with cells being deleted. Must be of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. :type columns: :class:`list` of :class:`str` / :func:`unicode <unicode>`, or :class:`object` :param columns: The columns within the column family that will have cells deleted. If :attr:`ALL_COLUMNS` is used then the entire column family will be deleted from the row. :type time_range: :class:`TimestampRange` :param time_range: (Optional) The range of time within which cells should be deleted. :type state: bool :param state: (Optional) The state that is passed along to :meth:`_get_mutations`. """ mutations_list = self._get_mutations(state) if columns is self.ALL_COLUMNS: mutation_val = data_v2_pb2.Mutation.DeleteFromFamily( family_name=column_family_id, ) mutation_pb = data_v2_pb2.Mutation(delete_from_family=mutation_val) mutations_list.append(mutation_pb) else: delete_kwargs = {} if time_range is not None: delete_kwargs['time_range'] = time_range.to_pb() to_append = [] for column in columns: column = _to_bytes(column) # time_range will never change if present, but the rest of # delete_kwargs will delete_kwargs.update( family_name=column_family_id, column_qualifier=column, ) mutation_val = data_v2_pb2.Mutation.DeleteFromColumn( **delete_kwargs) mutation_pb = data_v2_pb2.Mutation( delete_from_column=mutation_val) to_append.append(mutation_pb) # We don't add the mutations until all columns have been # processed without error. mutations_list.extend(to_append)
def __init__(self, image_source, client): self.client = client self._content = None self._source = None if _bytes_to_unicode(image_source).startswith('gs://'): self._source = image_source else: self._content = b64encode(_to_bytes(image_source))
def __init__(self, row_key, table, filter_=None): self._row_key = _to_bytes(row_key) self._table = table self._filter = filter_ self._rule_pb_list = [] if self._filter is None: self._pb_mutations = [] self._true_pb_mutations = None self._false_pb_mutations = None else: self._pb_mutations = None self._true_pb_mutations = [] self._false_pb_mutations = []
class TestClient(unittest.TestCase): PROJECT = 'PROJECT' IMAGE_SOURCE = 'gs://some/image.jpg' IMAGE_CONTENT = _to_bytes('/9j/4QNURXhpZgAASUkq') B64_IMAGE_CONTENT = base64.b64encode(IMAGE_CONTENT) def _getTargetClass(self): from gcloud.vision.client import Client return Client def _makeOne(self, *args, **kw): return self._getTargetClass()(*args, **kw) def test_ctor(self): creds = _Credentials() client = self._makeOne(project=self.PROJECT, credentials=creds) self.assertEqual(client.project, self.PROJECT) self.assertTrue('annotate' in dir(client)) def test_face_annotation(self): from gcloud.vision._fixtures import FACE_DETECTION_RESPONSE as RETURNED REQUEST = { "requests": [{ "image": { "content": self.B64_IMAGE_CONTENT }, "features": [{ "maxResults": 3, "type": "FACE_DETECTION" }] }] } credentials = _Credentials() client = self._makeOne(project=self.PROJECT, credentials=credentials) client.connection = _Connection(RETURNED) from gcloud.vision.feature import Feature, FeatureTypes features = [ Feature(feature_type=FeatureTypes.FACE_DETECTION, max_results=3) ] response = client.annotate(self.IMAGE_CONTENT, features) self.assertEqual(REQUEST, client.connection._requested[0]['data']) self.assertTrue('faceAnnotations' in response)
def _next_char(str_val, index): """Gets the next character based on a position in a string. :type str_val: str :param str_val: A string containing the character to update. :type index: int :param index: An integer index in ``str_val``. :rtype: str :returns: The next character after the character at ``index`` in ``str_val``. """ ord_val = six.indexbytes(str_val, index) return _to_bytes(chr(ord_val + 1), encoding='latin-1')
def _set_encryption_headers(key, headers): """Builds customer encyrption key headers :type key: str or bytes :param key: 32 byte key to build request key and hash. :type headers: dict :param headers: dict of HTTP headers being sent in request. """ key = _to_bytes(key) sha256_key = hashlib.sha256(key).digest() key_hash = base64.b64encode(sha256_key).rstrip() encoded_key = base64.b64encode(key).rstrip() headers['X-Goog-Encryption-Algorithm'] = 'AES256' headers['X-Goog-Encryption-Key'] = _bytes_to_unicode(encoded_key) headers['X-Goog-Encryption-Key-Sha256'] = _bytes_to_unicode(key_hash)
def _create_test_helper(self, initial_split_keys, column_families=()): from gcloud._helpers import _to_bytes from gcloud.bigtable._testing import _FakeStub client = _Client() instance = _Instance(self.INSTANCE_NAME, client=client) table = self._makeOne(self.TABLE_ID, instance) # Create request_pb splits_pb = [ _CreateTableRequestSplitPB(key=_to_bytes(key)) for key in initial_split_keys or () ] table_pb = None if column_families: table_pb = _TablePB() for cf in column_families: cf_pb = table_pb.column_families[cf.column_family_id] if cf.gc_rule is not None: cf_pb.gc_rule.MergeFrom(cf.gc_rule.to_pb()) request_pb = _CreateTableRequestPB( initial_splits=splits_pb, parent=self.INSTANCE_NAME, table_id=self.TABLE_ID, table=table_pb, ) # Create response_pb response_pb = _TablePB() # Patch the stub used by the API method. client._table_stub = stub = _FakeStub(response_pb) # Create expected_result. expected_result = None # create() has no return value. # Perform the method and check the result. result = table.create(initial_split_keys=initial_split_keys, column_families=column_families) self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'CreateTable', (request_pb, ), {}, )])
def _create_test_helper(self, initial_split_keys, column_families=()): from gcloud._helpers import _to_bytes from gcloud.bigtable._testing import _FakeStub client = _Client(timeout_seconds=self.TIMEOUT_SECONDS) instance = _Instance(self.INSTANCE_NAME, client=client) table = self._makeOne(self.TABLE_ID, instance) # Create request_pb splits_pb = [ _CreateTableRequestSplitPB(key=_to_bytes(key)) for key in initial_split_keys or ()] table_pb = None if column_families: table_pb = _TablePB() for cf in column_families: cf_pb = table_pb.column_families[cf.column_family_id] if cf.gc_rule is not None: cf_pb.gc_rule.MergeFrom(cf.gc_rule.to_pb()) request_pb = _CreateTableRequestPB( initial_splits=splits_pb, parent=self.INSTANCE_NAME, table_id=self.TABLE_ID, table=table_pb, ) # Create response_pb response_pb = _TablePB() # Patch the stub used by the API method. client._table_stub = stub = _FakeStub(response_pb) # Create expected_result. expected_result = None # create() has no return value. # Perform the method and check the result. result = table.create(initial_split_keys=initial_split_keys, column_families=column_families) self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'CreateTable', (request_pb, self.TIMEOUT_SECONDS), {}, )])
class TestVisionImage(unittest.TestCase): _IMAGE_SOURCE = 'gs://some/image.jpg' _IMAGE_CONTENT = _to_bytes('/9j/4QNURXhpZgAASUkq') _B64_IMAGE_CONTENT = base64.b64encode(_IMAGE_CONTENT) _CLIENT_MOCK = {'source': ''} def _getTargetClass(self): from gcloud.vision.image import Image return Image def _makeOne(self, *args, **kw): return self._getTargetClass()(*args, **kw) def test_image_source_type_content(self): image = self._makeOne(self._IMAGE_CONTENT, self._CLIENT_MOCK) _AS_DICT = {'content': self._B64_IMAGE_CONTENT} self.assertEqual(self._B64_IMAGE_CONTENT, image.content) self.assertEqual(None, image.source) self.assertEqual(_AS_DICT, image.as_dict()) def test_image_source_type_gcloud_storage(self): image = self._makeOne(self._IMAGE_SOURCE, self._CLIENT_MOCK) _AS_DICT = {'source': {'gcs_image_uri': self._IMAGE_SOURCE}} self.assertEqual(self._IMAGE_SOURCE, image.source) self.assertEqual(None, image.content) self.assertEqual(_AS_DICT, image.as_dict()) def test_cannot_set_both_source_and_content(self): image = self._makeOne(self._IMAGE_CONTENT, self._CLIENT_MOCK) self.assertEqual(self._B64_IMAGE_CONTENT, image.content) with self.assertRaises(AttributeError): image.source = self._IMAGE_SOURCE image = self._makeOne(self._IMAGE_SOURCE, self._CLIENT_MOCK) self.assertEqual(self._IMAGE_SOURCE, image.source) with self.assertRaises(AttributeError): image.content = self._IMAGE_CONTENT
def _configure_multipart_request(self, http_request): """Helper for 'configure_request': set up multipart request.""" # This is a multipart/related upload. msg_root = mime_multipart.MIMEMultipart('related') # msg_root should not write out its own headers setattr(msg_root, '_write_headers', lambda self: None) # attach the body as one part msg = mime_nonmultipart.MIMENonMultipart( *http_request.headers['content-type'].split('/')) msg.set_payload(http_request.body) msg_root.attach(msg) # attach the media as the second part msg = mime_nonmultipart.MIMENonMultipart(*self.mime_type.split('/')) msg['Content-Transfer-Encoding'] = 'binary' msg.set_payload(self.stream.read()) msg_root.attach(msg) # NOTE: generate multipart message as bytes, not text stream = six.BytesIO() if six.PY3: # pragma: NO COVER Python3 generator_class = email_generator.BytesGenerator else: generator_class = email_generator.Generator generator = generator_class(stream, mangle_from_=False) generator.flatten(msg_root, unixfrom=False) http_request.body = stream.getvalue() multipart_boundary = msg_root.get_boundary() http_request.headers['content-type'] = ( 'multipart/related; boundary="%s"' % multipart_boundary) boundary_bytes = _to_bytes(multipart_boundary) body_components = http_request.body.split(boundary_bytes) headers, _, _ = body_components[-2].partition(b'\n\n') body_components[-2] = b'\n\n'.join([headers, b'<media body>\n\n--']) http_request.loggable_body = boundary_bytes.join(body_components)
def _message_pb_from_dict(message): """Helper for :meth:`_PublisherAPI.topic_publish`.""" return PubsubMessage(data=_to_bytes(message['data']), attributes=message['attributes'])
def delete_cells(self, column_family_id, columns, time_range=None, state=None): """Deletes cells in this row. .. note:: This method adds a mutation to the accumulated mutations on this :class:`Row`, but does not make an API request. To actually send an API request (with the mutations) to the Google Cloud Bigtable API, call :meth:`commit`. :type column_family_id: str :param column_family_id: The column family that contains the column or columns with cells being deleted. Must be of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. :type columns: :class:`list` of :class:`str` / :func:`unicode <unicode>`, or :class:`object` :param columns: The columns within the column family that will have cells deleted. If :attr:`Row.ALL_COLUMNS` is used then the entire column family will be deleted from the row. :type time_range: :class:`TimestampRange` :param time_range: (Optional) The range of time within which cells should be deleted. :type state: bool :param state: (Optional) The state that the mutation should be applied in. Unset if the mutation is not conditional, otherwise :data:`True` or :data:`False`. """ mutations_list = self._get_mutations(state) if columns is self.ALL_COLUMNS: mutation_val = data_pb2.Mutation.DeleteFromFamily( family_name=column_family_id, ) mutation_pb = data_pb2.Mutation(delete_from_family=mutation_val) mutations_list.append(mutation_pb) else: delete_kwargs = {} if time_range is not None: delete_kwargs['time_range'] = time_range.to_pb() to_append = [] for column in columns: column = _to_bytes(column) # time_range will never change if present, but the rest of # delete_kwargs will delete_kwargs.update( family_name=column_family_id, column_qualifier=column, ) mutation_val = data_pb2.Mutation.DeleteFromColumn( **delete_kwargs) mutation_pb = data_pb2.Mutation( delete_from_column=mutation_val) to_append.append(mutation_pb) # We don't add the mutations until all columns have been # processed without error. mutations_list.extend(to_append)
def _callFUT(self, *args, **kwargs): from gcloud._helpers import _to_bytes return _to_bytes(*args, **kwargs)
def translate(self, values, target_language=None, format_=None, source_language=None, customization_ids=()): """Translate a string or list of strings. See: https://cloud.google.com/translate/v2/\ translating-text-with-rest :type values: str or list :param values: String or list of strings to translate. :type target_language: str :param target_language: The language to translate results into. This is required by the API and defaults to the target language of the current instance. :type format_: str :param format_: (Optional) One of ``text`` or ``html``, to specify if the input text is plain text or HTML. :type source_language: str :param source_language: (Optional) The language of the text to be translated. :type customization_ids: str or list :param customization_ids: (Optional) ID or list of customization IDs for translation. Sets the ``cid`` parameter in the query. :rtype: str or list list :returns: A list of dictionaries for each queried value. Each dictionary typically contains three keys (though not all will be present in all cases) * ``detectedSourceLanguage``: The detected language (as an ISO 639-1 language code) of the text. * ``translatedText``: The translation of the text into the target language. * ``input``: The corresponding input value. If only a single value is passed, then only a single dictionary will be returned. :raises: :class:`ValueError <exceptions.ValueError>` if the number of values and translations differ. """ single_value = False if isinstance(values, six.string_types): single_value = True values = [values] if target_language is None: target_language = self.target_language if isinstance(customization_ids, six.string_types): customization_ids = [customization_ids] query_params = [('key', self.api_key), ('target', target_language)] query_params.extend(('q', _to_bytes(value, 'utf-8')) for value in values) query_params.extend(('cid', cid) for cid in customization_ids) if format_ is not None: query_params.append(('format', format_)) if source_language is not None: query_params.append(('source', source_language)) response = self.connection.api_request( method='GET', path='', query_params=query_params) translations = response.get('data', {}).get('translations', ()) if len(values) != len(translations): raise ValueError('Expected iterations to have same length', values, translations) for value, translation in six.moves.zip(values, translations): translation['input'] = value if single_value: return translations[0] else: return translations
def __init__(self, row_key, table, filter_=None): self._row_key = _to_bytes(row_key) self._table = table self._filter = filter_
def detect_language(self, values): """Detect the language of a string or list of strings. See: https://cloud.google.com/translate/v2/\ detecting-language-with-rest :type values: str or list :param values: String or list of strings that will have language detected. :rtype: str or list :returns: A list of dictionaries for each queried value. Each dictionary typically contains three keys * ``confidence``: The confidence in language detection, a float between 0 and 1. * ``input``: The corresponding input value. * ``language``: The detected language (as an ISO 639-1 language code). though the key ``confidence`` may not always be present. If only a single value is passed, then only a single dictionary will be returned. :raises: :class:`ValueError <exceptions.ValueError>` if the number of detections is not equal to the number of values. :class:`ValueError <exceptions.ValueError>` if a value produces a list of detections with 0 or multiple results in it. """ single_value = False if isinstance(values, six.string_types): single_value = True values = [values] query_params = [('key', self.api_key)] query_params.extend( ('q', _to_bytes(value, 'utf-8')) for value in values) response = self.connection.api_request(method='GET', path='/detect', query_params=query_params) detections = response.get('data', {}).get('detections', ()) if len(values) != len(detections): raise ValueError('Expected same number of values and detections', values, detections) for index, value in enumerate(values): # Empirically, even clearly ambiguous text like "no" only returns # a single detection, so we replace the list of detections with # the single detection contained. if len(detections[index]) == 1: detections[index] = detections[index][0] else: message = ('Expected a single detection per value, API ' 'returned %d') % (len(detections[index]), ) raise ValueError(message, value, detections[index]) detections[index]['input'] = value # The ``isReliable`` field is deprecated. detections[index].pop('isReliable', None) if single_value: return detections[0] else: return detections
def __init__(self, regex): self.regex = _to_bytes(regex)
def _create_row_request(table_name, row_key=None, start_key=None, end_key=None, filter_=None, allow_row_interleaving=None, limit=None): """Creates a request to read rows in a table. :type table_name: str :param table_name: The name of the table to read from. :type row_key: bytes :param row_key: (Optional) The key of a specific row to read from. :type start_key: bytes :param start_key: (Optional) The beginning of a range of row keys to read from. The range will include ``start_key``. If left empty, will be interpreted as the empty string. :type end_key: bytes :param end_key: (Optional) The end of a range of row keys to read from. The range will not include ``end_key``. If left empty, will be interpreted as an infinite string. :type filter_: :class:`.RowFilter` :param filter_: (Optional) The filter to apply to the contents of the specified row(s). If unset, reads the entire table. :type allow_row_interleaving: bool :param allow_row_interleaving: (Optional) By default, rows are read sequentially, producing results which are guaranteed to arrive in increasing row order. Setting ``allow_row_interleaving`` to :data:`True` allows multiple rows to be interleaved in the response stream, which increases throughput but breaks this guarantee, and may force the client to use more memory to buffer partially-received rows. :type limit: int :param limit: (Optional) The read will terminate after committing to N rows' worth of results. The default (zero) is to return all results. Note that if ``allow_row_interleaving`` is set to :data:`True`, partial results may be returned for more than N rows. However, only N ``commit_row`` chunks will be sent. :rtype: :class:`data_messages_pb2.ReadRowsRequest` :returns: The ``ReadRowsRequest`` protobuf corresponding to the inputs. :raises: :class:`ValueError <exceptions.ValueError>` if both ``row_key`` and one of ``start_key`` and ``end_key`` are set """ request_kwargs = {'table_name': table_name} if (row_key is not None and (start_key is not None or end_key is not None)): raise ValueError('Row key and row range cannot be ' 'set simultaneously') if row_key is not None: request_kwargs['row_key'] = _to_bytes(row_key) if start_key is not None or end_key is not None: range_kwargs = {} if start_key is not None: range_kwargs['start_key'] = _to_bytes(start_key) if end_key is not None: range_kwargs['end_key'] = _to_bytes(end_key) row_range = data_pb2.RowRange(**range_kwargs) request_kwargs['row_range'] = row_range if filter_ is not None: request_kwargs['filter'] = filter_.to_pb() if allow_row_interleaving is not None: request_kwargs['allow_row_interleaving'] = allow_row_interleaving if limit is not None: request_kwargs['num_rows_limit'] = limit return data_messages_pb2.ReadRowsRequest(**request_kwargs)
def detect_language(self, values): """Detect the language of a string or list of strings. See: https://cloud.google.com/translate/v2/\ detecting-language-with-rest :type values: str or list :param values: String or list of strings that will have language detected. :rtype: str or list :returns: A list of dictionaries for each queried value. Each dictionary typically contains three keys * ``confidence``: The confidence in language detection, a float between 0 and 1. * ``input``: The corresponding input value. * ``language``: The detected language (as an ISO 639-1 language code). though the key ``confidence`` may not always be present. If only a single value is passed, then only a single dictionary will be returned. :raises: :class:`ValueError <exceptions.ValueError>` if the number of detections is not equal to the number of values. :class:`ValueError <exceptions.ValueError>` if a value produces a list of detections with 0 or multiple results in it. """ single_value = False if isinstance(values, six.string_types): single_value = True values = [values] query_params = [('key', self.api_key)] query_params.extend(('q', _to_bytes(value, 'utf-8')) for value in values) response = self.connection.api_request( method='GET', path='/detect', query_params=query_params) detections = response.get('data', {}).get('detections', ()) if len(values) != len(detections): raise ValueError('Expected same number of values and detections', values, detections) for index, value in enumerate(values): # Empirically, even clearly ambiguous text like "no" only returns # a single detection, so we replace the list of detections with # the single detection contained. if len(detections[index]) == 1: detections[index] = detections[index][0] else: message = ('Expected a single detection per value, API ' 'returned %d') % (len(detections[index]),) raise ValueError(message, value, detections[index]) detections[index]['input'] = value # The ``isReliable`` field is deprecated. detections[index].pop('isReliable', None) if single_value: return detections[0] else: return detections
def __init__(self, row_key, table): self._row_key = _to_bytes(row_key) self._table = table
def _create_row_request( table_name, row_key=None, start_key=None, end_key=None, filter_=None, allow_row_interleaving=None, limit=None ): """Creates a request to read rows in a table. :type table_name: str :param table_name: The name of the table to read from. :type row_key: bytes :param row_key: (Optional) The key of a specific row to read from. :type start_key: bytes :param start_key: (Optional) The beginning of a range of row keys to read from. The range will include ``start_key``. If left empty, will be interpreted as the empty string. :type end_key: bytes :param end_key: (Optional) The end of a range of row keys to read from. The range will not include ``end_key``. If left empty, will be interpreted as an infinite string. :type filter_: :class:`.RowFilter` :param filter_: (Optional) The filter to apply to the contents of the specified row(s). If unset, reads the entire table. :type allow_row_interleaving: bool :param allow_row_interleaving: (Optional) By default, rows are read sequentially, producing results which are guaranteed to arrive in increasing row order. Setting ``allow_row_interleaving`` to :data:`True` allows multiple rows to be interleaved in the response stream, which increases throughput but breaks this guarantee, and may force the client to use more memory to buffer partially-received rows. :type limit: int :param limit: (Optional) The read will terminate after committing to N rows' worth of results. The default (zero) is to return all results. Note that if ``allow_row_interleaving`` is set to :data:`True`, partial results may be returned for more than N rows. However, only N ``commit_row`` chunks will be sent. :rtype: :class:`data_messages_pb2.ReadRowsRequest` :returns: The ``ReadRowsRequest`` protobuf corresponding to the inputs. :raises: :class:`ValueError <exceptions.ValueError>` if both ``row_key`` and one of ``start_key`` and ``end_key`` are set """ request_kwargs = {"table_name": table_name} if row_key is not None and (start_key is not None or end_key is not None): raise ValueError("Row key and row range cannot be " "set simultaneously") if row_key is not None: request_kwargs["row_key"] = _to_bytes(row_key) if start_key is not None or end_key is not None: range_kwargs = {} if start_key is not None: range_kwargs["start_key"] = _to_bytes(start_key) if end_key is not None: range_kwargs["end_key"] = _to_bytes(end_key) row_range = data_pb2.RowRange(**range_kwargs) request_kwargs["row_range"] = row_range if filter_ is not None: request_kwargs["filter"] = filter_.to_pb() if allow_row_interleaving is not None: request_kwargs["allow_row_interleaving"] = allow_row_interleaving if limit is not None: request_kwargs["num_rows_limit"] = limit return data_messages_pb2.ReadRowsRequest(**request_kwargs)
def translate(self, values, target_language=None, format_=None, source_language=None, customization_ids=()): """Translate a string or list of strings. See: https://cloud.google.com/translate/v2/\ translating-text-with-rest :type values: str or list :param values: String or list of strings to translate. :type target_language: str :param target_language: The language to translate results into. This is required by the API and defaults to the target language of the current instance. :type format_: str :param format_: (Optional) One of ``text`` or ``html``, to specify if the input text is plain text or HTML. :type source_language: str :param source_language: (Optional) The language of the text to be translated. :type customization_ids: str or list :param customization_ids: (Optional) ID or list of customization IDs for translation. Sets the ``cid`` parameter in the query. :rtype: str or list list :returns: A list of dictionaries for each queried value. Each dictionary typically contains three keys (though not all will be present in all cases) * ``detectedSourceLanguage``: The detected language (as an ISO 639-1 language code) of the text. * ``translatedText``: The translation of the text into the target language. * ``input``: The corresponding input value. If only a single value is passed, then only a single dictionary will be returned. :raises: :class:`ValueError <exceptions.ValueError>` if the number of values and translations differ. """ single_value = False if isinstance(values, six.string_types): single_value = True values = [values] if target_language is None: target_language = self.target_language if isinstance(customization_ids, six.string_types): customization_ids = [customization_ids] query_params = [('key', self.api_key), ('target', target_language)] query_params.extend( ('q', _to_bytes(value, 'utf-8')) for value in values) query_params.extend(('cid', cid) for cid in customization_ids) if format_ is not None: query_params.append(('format', format_)) if source_language is not None: query_params.append(('source', source_language)) response = self.connection.api_request(method='GET', path='', query_params=query_params) translations = response.get('data', {}).get('translations', ()) if len(values) != len(translations): raise ValueError('Expected iterations to have same length', values, translations) for value, translation in six.moves.zip(values, translations): translation['input'] = value if single_value: return translations[0] else: return translations