def _migrate_audio_value(value): audio_value = value.audio data = [[audio_value.encoded_audio_string, b'']] # empty label tensor_proto = tf.make_tensor_proto(data) summary_metadata = audio_metadata.create_summary_metadata( display_name=value.metadata.display_name or value.tag, description=value.metadata.summary_description, encoding=audio_metadata.Encoding.Value('WAV')) return tf.Summary.Value(tag=value.tag, metadata=summary_metadata, tensor=tensor_proto)
def test_audio(self): audio = tf.reshape(tf.linspace(0.0, 100.0, 4 * 10 * 2), (4, 10, 2)) old_op = tf.summary.audio('k488', audio, 44100) old_value = self._value_from_op(old_op) assert old_value.HasField('audio'), old_value new_value = data_compat.migrate_value(old_value) self.assertEqual('k488/audio/0', new_value.tag) expected_metadata = audio_metadata.create_summary_metadata( display_name='k488/audio/0', description='', encoding=audio_metadata.Encoding.Value('WAV')) self.assertEqual(expected_metadata, new_value.metadata) self.assertTrue(new_value.HasField('tensor')) data = tf.make_ndarray(new_value.tensor) self.assertEqual((1, 2), data.shape) self.assertEqual(tf.compat.as_bytes(old_value.audio.encoded_audio_string), data[0][0]) self.assertEqual(b'', data[0][1]) # empty label
def test_audio(self): audio = tf.reshape(tf.linspace(0.0, 100.0, 4 * 10 * 2), (4, 10, 2)) old_op = tf.summary.audio('k488', audio, 44100) old_value = self._value_from_op(old_op) assert old_value.HasField('audio'), old_value new_value = data_compat.migrate_value(old_value) self.assertEqual('k488/audio/0', new_value.tag) expected_metadata = audio_metadata.create_summary_metadata( display_name='k488/audio/0', description='', encoding=audio_metadata.Encoding.Value('WAV')) self.assertEqual(expected_metadata, new_value.metadata) self.assertTrue(new_value.HasField('tensor')) data = tf.make_ndarray(new_value.tensor) self.assertEqual((1, 2), data.shape) self.assertEqual( tf.compat.as_bytes(old_value.audio.encoded_audio_string), data[0][0]) self.assertEqual(b'', data[0][1]) # empty label
def test_audio(self): with tf.compat.v1.Graph().as_default(): audio = tf.reshape(tf.linspace(0.0, 100.0, 4 * 10 * 2), (4, 10, 2)) old_op = tf.compat.v1.summary.audio("k488", audio, 44100) old_value = self._value_from_op(old_op) assert old_value.HasField("audio"), old_value new_value = data_compat.migrate_value(old_value) self.assertEqual("k488/audio/0", new_value.tag) expected_metadata = audio_metadata.create_summary_metadata( display_name="k488/audio/0", description="", encoding=audio_metadata.Encoding.Value("WAV"), ) self.assertEqual(expected_metadata, new_value.metadata) self.assertTrue(new_value.HasField("tensor")) data = tensor_util.make_ndarray(new_value.tensor) self.assertEqual((1, 2), data.shape) self.assertEqual( tf.compat.as_bytes(old_value.audio.encoded_audio_string), data[0][0]) self.assertEqual(b"", data[0][1]) # empty label
def test_audio(self): with tf.compat.v1.Graph().as_default(): audio = tf.reshape(tf.linspace(0.0, 100.0, 4 * 10 * 2), (4, 10, 2)) old_op = tf.compat.v1.summary.audio("k488", audio, 44100) old_value = self._value_from_op(old_op) assert old_value.HasField("audio"), old_value new_value = data_compat.migrate_value(old_value) self.assertEqual("k488/audio/0", new_value.tag) expected_metadata = audio_metadata.create_summary_metadata( display_name="k488/audio/0", description="", encoding=audio_metadata.Encoding.Value("WAV"), converted_to_tensor=True, ) # Check serialized submessages... plugin_content = audio_metadata.parse_plugin_metadata( new_value.metadata.plugin_data.content ) expected_content = audio_metadata.parse_plugin_metadata( expected_metadata.plugin_data.content ) self.assertEqual(plugin_content, expected_content) # ...then check full metadata except plugin content, since # serialized forms need not be identical. new_value.metadata.plugin_data.content = ( expected_metadata.plugin_data.content ) self.assertEqual(expected_metadata, new_value.metadata) self.assertTrue(new_value.HasField("tensor")) data = tensor_util.make_ndarray(new_value.tensor) self.assertEqual((1, 2), data.shape) self.assertEqual( tf.compat.as_bytes(old_value.audio.encoded_audio_string), data[0][0] ) self.assertEqual(b"", data[0][1]) # empty label
def op(name, audio, sample_rate, labels=None, max_outputs=3, encoding=None, display_name=None, description=None, collections=None): """Create an audio summary op for use in a TensorFlow graph. Arguments: name: A unique name for the generated summary node. audio: A `Tensor` representing audio data with shape `[k, t, c]`, where `k` is the number of audio clips, `t` is the number of frames, and `c` is the number of channels. Elements should be floating-point values in `[-1.0, 1.0]`. Any of the dimensions may be statically unknown (i.e., `None`). sample_rate: An `int` or rank-0 `int32` `Tensor` that represents the sample rate, in Hz. Must be positive. labels: Optional `string` `Tensor`, a vector whose length is the first dimension of `audio`, where `labels[i]` contains arbitrary textual information about `audio[i]`. (For instance, this could be some text that a TTS system was supposed to produce.) Markdown is supported. Contents should be UTF-8. max_outputs: Optional `int` or rank-0 integer `Tensor`. At most this many audio clips will be emitted at each step. When more than `max_outputs` many clips are provided, the first `max_outputs` many clips will be used and the rest silently discarded. encoding: A constant `str` (not string tensor) indicating the desired encoding. You can choose any format you like, as long as it's "wav". Please see the "API compatibility note" below. display_name: Optional name for this summary in TensorBoard, as a constant `str`. Defaults to `name`. description: Optional long-form description for this summary, as a constant `str`. Markdown is supported. Defaults to empty. collections: Optional list of graph collections keys. The new summary op is added to these collections. Defaults to `[Graph Keys.SUMMARIES]`. Returns: A TensorFlow summary op. API compatibility note: The default value of the `encoding` argument is _not_ guaranteed to remain unchanged across TensorBoard versions. In the future, we will by default encode as FLAC instead of as WAV. If the specific format is important to you, please provide a file format explicitly. """ if display_name is None: display_name = name if encoding is None: encoding = 'wav' if encoding == 'wav': encoding = metadata.Encoding.Value('WAV') encoder = functools.partial(tf.contrib.ffmpeg.encode_audio, samples_per_second=sample_rate, file_format='wav') else: raise ValueError('Unknown encoding: %r' % encoding) with tf.name_scope(name), \ tf.control_dependencies([tf.assert_rank(audio, 3)]): limited_audio = audio[:max_outputs] encoded_audio = tf.map_fn(encoder, limited_audio, dtype=tf.string, name='encode_each_audio') if labels is None: limited_labels = tf.tile([''], tf.shape(limited_audio)[:1]) else: limited_labels = labels[:max_outputs] tensor = tf.transpose(tf.stack([encoded_audio, limited_labels])) summary_metadata = metadata.create_summary_metadata( display_name=display_name, description=description, encoding=encoding) return tf.summary.tensor_summary(name='audio_summary', tensor=tensor, collections=collections, summary_metadata=summary_metadata)
def pb(name, audio, sample_rate, labels=None, max_outputs=3, encoding=None, display_name=None, description=None): """Create an audio summary protobuf. This behaves as if you were to create an `op` with the same arguments (wrapped with constant tensors where appropriate) and then execute that summary op in a TensorFlow session. Arguments: name: A unique name for the generated summary node. audio: An `np.array` representing audio data with shape `[k, t, c]`, where `k` is the number of audio clips, `t` is the number of frames, and `c` is the number of channels. Elements should be floating-point values in `[-1.0, 1.0]`. sample_rate: An `int` that represents the sample rate, in Hz. Must be positive. labels: Optional list (or rank-1 `np.array`) of textstrings or UTF-8 bytestrings whose length is the first dimension of `audio`, where `labels[i]` contains arbitrary textual information about `audio[i]`. (For instance, this could be some text that a TTS system was supposed to produce.) Markdown is supported. max_outputs: Optional `int`. At most this many audio clips will be emitted. When more than `max_outputs` many clips are provided, the first `max_outputs` many clips will be used and the rest silently discarded. encoding: A constant `str` indicating the desired encoding. You can choose any format you like, as long as it's "wav". Please see the "API compatibility note" below. display_name: Optional name for this summary in TensorBoard, as a `str`. Defaults to `name`. description: Optional long-form description for this summary, as a `str`. Markdown is supported. Defaults to empty. Returns: A `tf.Summary` protobuf object. API compatibility note: The default value of the `encoding` argument is _not_ guaranteed to remain unchanged across TensorBoard versions. In the future, we will by default encode as FLAC instead of as WAV. If the specific format is important to you, please provide a file format explicitly. """ audio = np.array(audio) if audio.ndim != 3: raise ValueError('Shape %r must have rank 3' % (audio.shape,)) if encoding is None: encoding = 'wav' if encoding == 'wav': encoding = metadata.Encoding.Value('WAV') encoder = functools.partial(encoder_util.encode_wav, samples_per_second=sample_rate) else: raise ValueError('Unknown encoding: %r' % encoding) limited_audio = audio[:max_outputs] if labels is None: limited_labels = [b''] * len(limited_audio) else: limited_labels = [tf.compat.as_bytes(label) for label in labels[:max_outputs]] encoded_audio = [encoder(a) for a in limited_audio] content = np.array([encoded_audio, limited_labels]).transpose() tensor = tf.compat.v1.make_tensor_proto(content, dtype=tf.string) if display_name is None: display_name = name summary_metadata = metadata.create_summary_metadata( display_name=display_name, description=description, encoding=encoding) tf_summary_metadata = tf.SummaryMetadata.FromString( summary_metadata.SerializeToString()) summary = tf.Summary() summary.value.add(tag='%s/audio_summary' % name, metadata=tf_summary_metadata, tensor=tensor) return summary
def op( name, audio, sample_rate, labels=None, max_outputs=3, encoding=None, display_name=None, description=None, collections=None, ): """Create a legacy audio summary op for use in a TensorFlow graph. Arguments: name: A unique name for the generated summary node. audio: A `Tensor` representing audio data with shape `[k, t, c]`, where `k` is the number of audio clips, `t` is the number of frames, and `c` is the number of channels. Elements should be floating-point values in `[-1.0, 1.0]`. Any of the dimensions may be statically unknown (i.e., `None`). sample_rate: An `int` or rank-0 `int32` `Tensor` that represents the sample rate, in Hz. Must be positive. labels: Deprecated. Do not set. max_outputs: Optional `int` or rank-0 integer `Tensor`. At most this many audio clips will be emitted at each step. When more than `max_outputs` many clips are provided, the first `max_outputs` many clips will be used and the rest silently discarded. encoding: A constant `str` (not string tensor) indicating the desired encoding. You can choose any format you like, as long as it's "wav". Please see the "API compatibility note" below. display_name: Optional name for this summary in TensorBoard, as a constant `str`. Defaults to `name`. description: Optional long-form description for this summary, as a constant `str`. Markdown is supported. Defaults to empty. collections: Optional list of graph collections keys. The new summary op is added to these collections. Defaults to `[Graph Keys.SUMMARIES]`. Returns: A TensorFlow summary op. API compatibility note: The default value of the `encoding` argument is _not_ guaranteed to remain unchanged across TensorBoard versions. In the future, we will by default encode as FLAC instead of as WAV. If the specific format is important to you, please provide a file format explicitly. """ if labels is not None: warnings.warn(_LABELS_WARNING) # TODO(nickfelt): remove on-demand imports once dep situation is fixed. import tensorflow.compat.v1 as tf if display_name is None: display_name = name if encoding is None: encoding = "wav" if encoding == "wav": encoding = metadata.Encoding.Value("WAV") encoder = functools.partial(tf.audio.encode_wav, sample_rate=sample_rate) else: raise ValueError("Unknown encoding: %r" % encoding) with tf.name_scope(name), tf.control_dependencies( [tf.assert_rank(audio, 3)]): limited_audio = audio[:max_outputs] encoded_audio = tf.map_fn(encoder, limited_audio, dtype=tf.string, name="encode_each_audio") if labels is None: limited_labels = tf.tile([""], tf.shape(input=limited_audio)[:1]) else: limited_labels = labels[:max_outputs] tensor = tf.transpose(a=tf.stack([encoded_audio, limited_labels])) summary_metadata = metadata.create_summary_metadata( display_name=display_name, description=description, encoding=encoding, ) return tf.summary.tensor_summary( name="audio_summary", tensor=tensor, collections=collections, summary_metadata=summary_metadata, )
def audio(name, data, sample_rate, step, max_outputs=3, encoding=None, description=None): """Write an audio summary. Arguments: name: A name for this summary. The summary tag used for TensorBoard will be this name prefixed by any active name scopes. data: A `Tensor` representing audio data with shape `[k, t, c]`, where `k` is the number of audio clips, `t` is the number of frames, and `c` is the number of channels. Elements should be floating-point values in `[-1.0, 1.0]`. Any of the dimensions may be statically unknown (i.e., `None`). sample_rate: An `int` or rank-0 `int32` `Tensor` that represents the sample rate, in Hz. Must be positive. step: Required `int64`-castable monotonic step value. max_outputs: Optional `int` or rank-0 integer `Tensor`. At most this many audio clips will be emitted at each step. When more than `max_outputs` many clips are provided, the first `max_outputs` many clips will be used and the rest silently discarded. encoding: Optional constant `str` for the desired encoding. Only "wav" is currently supported, but this is not guaranteed to remain the default, so if you want "wav" in particular, set this explicitly. description: Optional long-form description for this summary, as a constant `str`. Markdown is supported. Defaults to empty. Returns: True on success, or false if no summary was emitted because no default summary writer was available. """ # TODO(nickfelt): get encode_wav() exported in the public API. from tensorflow.python.ops import gen_audio_ops if encoding is None: encoding = 'wav' if encoding != 'wav': raise ValueError('Unknown encoding: %r' % encoding) summary_metadata = metadata.create_summary_metadata( display_name=None, description=description, encoding=metadata.Encoding.Value('WAV')) inputs = [data, sample_rate, max_outputs, step] with tf.summary.summary_scope(name, 'audio_summary', values=inputs) as (tag, _): tf.debugging.assert_rank(data, 3) tf.debugging.assert_non_negative(max_outputs) limited_audio = data[:max_outputs] encode_fn = functools.partial(gen_audio_ops.encode_wav, sample_rate=sample_rate) encoded_audio = tf.map_fn(encode_fn, limited_audio, dtype=tf.string, name='encode_each_audio') # Workaround for map_fn returning float dtype for an empty elems input. encoded_audio = tf.cond( tf.shape(input=encoded_audio)[0] > 0, lambda: encoded_audio, lambda: tf.constant([], tf.string)) limited_labels = tf.tile([''], tf.shape(input=limited_audio)[:1]) tensor = tf.transpose(a=tf.stack([encoded_audio, limited_labels])) return tf.summary.write(tag=tag, tensor=tensor, step=step, metadata=summary_metadata)
def op(name, audio, sample_rate, labels=None, max_outputs=3, encoding=None, display_name=None, description=None, collections=None): """Create an audio summary op for use in a TensorFlow graph. Arguments: name: A unique name for the generated summary node. audio: A `Tensor` representing audio data with shape `[k, t, c]`, where `k` is the number of audio clips, `t` is the number of frames, and `c` is the number of channels. Elements should be floating-point values in `[-1.0, 1.0]`. Any of the dimensions may be statically unknown (i.e., `None`). sample_rate: An `int` or rank-0 `int32` `Tensor` that represents the sample rate, in Hz. Must be positive. labels: Optional `string` `Tensor`, a vector whose length is the first dimension of `audio`, where `labels[i]` contains arbitrary textual information about `audio[i]`. (For instance, this could be some text that a TTS system was supposed to produce.) Markdown is supported. Contents should be UTF-8. max_outputs: Optional `int` or rank-0 integer `Tensor`. At most this many audio clips will be emitted at each step. When more than `max_outputs` many clips are provided, the first `max_outputs` many clips will be used and the rest silently discarded. encoding: A constant `str` (not string tensor) indicating the desired encoding. You can choose any format you like, as long as it's "wav". Please see the "API compatibility note" below. display_name: Optional name for this summary in TensorBoard, as a constant `str`. Defaults to `name`. description: Optional long-form description for this summary, as a constant `str`. Markdown is supported. Defaults to empty. collections: Optional list of graph collections keys. The new summary op is added to these collections. Defaults to `[Graph Keys.SUMMARIES]`. Returns: A TensorFlow summary op. API compatibility note: The default value of the `encoding` argument is _not_ guaranteed to remain unchanged across TensorBoard versions. In the future, we will by default encode as FLAC instead of as WAV. If the specific format is important to you, please provide a file format explicitly. """ if display_name is None: display_name = name if encoding is None: encoding = 'wav' if encoding == 'wav': encoding = metadata.Encoding.Value('WAV') encoder = functools.partial(tf.contrib.ffmpeg.encode_audio, samples_per_second=sample_rate, file_format='wav') else: raise ValueError('Unknown encoding: %r' % encoding) with tf.name_scope(name), \ tf.control_dependencies([tf.assert_rank(audio, 3)]): limited_audio = audio[:max_outputs] encoded_audio = tf.map_fn(encoder, limited_audio, dtype=tf.string, name='encode_each_audio') if labels is None: limited_labels = tf.tile([''], tf.shape(limited_audio)[:1]) else: limited_labels = labels[:max_outputs] tensor = tf.transpose(tf.stack([encoded_audio, limited_labels])) summary_metadata = metadata.create_summary_metadata( display_name=display_name, description=description, encoding=encoding) return tf.summary.tensor_summary(name='audio_summary', tensor=tensor, collections=collections, summary_metadata=summary_metadata)
def pb(name, audio, sample_rate, labels=None, max_outputs=3, encoding=None, display_name=None, description=None): """Create an audio summary protobuf. This behaves as if you were to create an `op` with the same arguments (wrapped with constant tensors where appropriate) and then execute that summary op in a TensorFlow session. Arguments: name: A unique name for the generated summary node. audio: An `np.array` representing audio data with shape `[k, t, c]`, where `k` is the number of audio clips, `t` is the number of frames, and `c` is the number of channels. Elements should be floating-point values in `[-1.0, 1.0]`. sample_rate: An `int` that represents the sample rate, in Hz. Must be positive. labels: Optional list (or rank-1 `np.array`) of textstrings or UTF-8 bytestrings whose length is the first dimension of `audio`, where `labels[i]` contains arbitrary textual information about `audio[i]`. (For instance, this could be some text that a TTS system was supposed to produce.) Markdown is supported. max_outputs: Optional `int`. At most this many audio clips will be emitted. When more than `max_outputs` many clips are provided, the first `max_outputs` many clips will be used and the rest silently discarded. encoding: A constant `str` indicating the desired encoding. You can choose any format you like, as long as it's "wav". Please see the "API compatibility note" below. display_name: Optional name for this summary in TensorBoard, as a `str`. Defaults to `name`. description: Optional long-form description for this summary, as a `str`. Markdown is supported. Defaults to empty. Returns: A `tf.Summary` protobuf object. API compatibility note: The default value of the `encoding` argument is _not_ guaranteed to remain unchanged across TensorBoard versions. In the future, we will by default encode as FLAC instead of as WAV. If the specific format is important to you, please provide a file format explicitly. """ audio = np.array(audio) if audio.ndim != 3: raise ValueError('Shape %r must have rank 3' % (audio.shape,)) if encoding is None: encoding = 'wav' if encoding == 'wav': encoding = metadata.Encoding.Value('WAV') encoder = functools.partial(util.encode_wav, samples_per_second=sample_rate) else: raise ValueError('Unknown encoding: %r' % encoding) limited_audio = audio[:max_outputs] if labels is None: limited_labels = [b''] * len(limited_audio) else: limited_labels = [tf.compat.as_bytes(label) for label in labels[:max_outputs]] encoded_audio = [encoder(a) for a in limited_audio] content = np.array([encoded_audio, limited_labels]).transpose() tensor = tf.make_tensor_proto(content, dtype=tf.string) if display_name is None: display_name = name summary_metadata = metadata.create_summary_metadata( display_name=display_name, description=description, encoding=encoding) summary = tf.Summary() summary.value.add(tag='%s/audio_summary' % name, metadata=summary_metadata, tensor=tensor) return summary
def audio( name, data, sample_rate, step=None, max_outputs=3, encoding=None, description=None, ): """Write an audio summary. Arguments: name: A name for this summary. The summary tag used for TensorBoard will be this name prefixed by any active name scopes. data: A `Tensor` representing audio data with shape `[k, t, c]`, where `k` is the number of audio clips, `t` is the number of frames, and `c` is the number of channels. Elements should be floating-point values in `[-1.0, 1.0]`. Any of the dimensions may be statically unknown (i.e., `None`). sample_rate: An `int` or rank-0 `int32` `Tensor` that represents the sample rate, in Hz. Must be positive. step: Explicit `int64`-castable monotonic step value for this summary. If omitted, this defaults to `tf.summary.experimental.get_step()`, which must not be None. max_outputs: Optional `int` or rank-0 integer `Tensor`. At most this many audio clips will be emitted at each step. When more than `max_outputs` many clips are provided, the first `max_outputs` many clips will be used and the rest silently discarded. encoding: Optional constant `str` for the desired encoding. Only "wav" is currently supported, but this is not guaranteed to remain the default, so if you want "wav" in particular, set this explicitly. description: Optional long-form description for this summary, as a constant `str`. Markdown is supported. Defaults to empty. Returns: True on success, or false if no summary was emitted because no default summary writer was available. Raises: ValueError: if a default writer exists, but no step was provided and `tf.summary.experimental.get_step()` is None. """ audio_ops = getattr(tf, "audio", None) if audio_ops is None: # Fallback for older versions of TF without tf.audio. from tensorflow.python.ops import gen_audio_ops as audio_ops if encoding is None: encoding = "wav" if encoding != "wav": raise ValueError("Unknown encoding: %r" % encoding) summary_metadata = metadata.create_summary_metadata( display_name=None, description=description, encoding=metadata.Encoding.Value("WAV"), ) inputs = [data, sample_rate, max_outputs, step] # TODO(https://github.com/tensorflow/tensorboard/issues/2109): remove fallback summary_scope = ( getattr(tf.summary.experimental, "summary_scope", None) or tf.summary.summary_scope ) with summary_scope(name, "audio_summary", values=inputs) as (tag, _): # Defer audio encoding preprocessing by passing it as a callable to write(), # wrapped in a LazyTensorCreator for backwards compatibility, so that we # only do this work when summaries are actually written. @lazy_tensor_creator.LazyTensorCreator def lazy_tensor(): tf.debugging.assert_rank(data, 3) tf.debugging.assert_non_negative(max_outputs) limited_audio = data[:max_outputs] encode_fn = functools.partial( audio_ops.encode_wav, sample_rate=sample_rate ) encoded_audio = tf.map_fn( encode_fn, limited_audio, dtype=tf.string, name="encode_each_audio", ) # Workaround for map_fn returning float dtype for an empty elems input. encoded_audio = tf.cond( tf.shape(input=encoded_audio)[0] > 0, lambda: encoded_audio, lambda: tf.constant([], tf.string), ) limited_labels = tf.tile([""], tf.shape(input=limited_audio)[:1]) return tf.transpose(a=tf.stack([encoded_audio, limited_labels])) # To ensure that audio encoding logic is only executed when summaries # are written, we pass callable to `tensor` parameter. return tf.summary.write( tag=tag, tensor=lazy_tensor, step=step, metadata=summary_metadata )