def audio(tag, audio_array, sample_rate, step, walltime):
"""Package data to one audio.
Args:
tag (string): Data identifier
audio_array (numpy.ndarray or list): audio represented by a numpy.array
sample_rate (int): Sample rate of audio
step (int): Step of audio
walltime (int): Wall time of audio
Return:
Package with format of record_pb2.Record
"""
import io
import wave
fio = io.BytesIO()
wave_writer = wave.open(fio, 'wb')
wave_writer.setnchannels(1)
wave_writer.setsampwidth(2)
wave_writer.setframerate(sample_rate)
wave_writer.writeframes(audio_array)
wave_writer.close()
audio_string = fio.getvalue()
fio.close()
audio_data = Record.Audio(sample_rate=sample_rate,
num_channels=1,
length_frames=len(audio_array),
encoded_audio_string=audio_string,
content_type='audio/wav')
return Record(values=[
Record.Value(id=step, tag=tag, timestamp=walltime, audio=audio_data)
])
def roc_curve_raw(tag, tp, fp, tn, fn, tpr, fpr, step, walltime):
"""Package raw data to one roc_curve.
Args:
tag (string): Data identifier
tp (list): True Positive.
fp (list): False Positive.
tn (list): True Negative.
fn (list): False Negative.
tpr (list): true positive rate:
fpr (list): false positive rate.
step (int): Step of roc_curve
walltime (int): Wall time of roc_curve
num_thresholds (int): Number of thresholds used to draw the curve.
weights (float): Multiple of data to display on the curve.
Return:
Package with format of record_pb2.Record
"""
"""
if isinstance(tp, np.ndarray):
tp = tp.astype(int).tolist()
if isinstance(fp, np.ndarray):
fp = fp.astype(int).tolist()
if isinstance(tn, np.ndarray):
tn = tn.astype(int).tolist()
if isinstance(fn, np.ndarray):
fn = fn.astype(int).tolist()
if isinstance(tpr, np.ndarray):
tpr = tpr.astype(int).tolist()
if isinstance(fpr, np.ndarray):
fpr = fpr.astype(int).tolist()
"""
roc_curve = Record.ROC_Curve(TP=tp, FP=fp, TN=tn, FN=fn, tpr=tpr, fpr=fpr)
return Record(values=[
Record.Value(id=step, tag=tag, timestamp=walltime, roc_curve=roc_curve)
])
def embedding(tag, labels, hot_vectors, step, labels_meta=None, walltime=None):
"""Package data to one embedding.
Args:
tag (string): Data identifier
labels (list): A list of labels.
hot_vectors (np.array or list): A matrix which each row is
feature of labels.
step (int): Step of embeddings.
walltime (int): Wall time of embeddings.
Return:
Package with format of record_pb2.Record
"""
embeddings = Record.Embeddings()
if labels_meta:
embeddings.label_meta.extend(labels_meta)
if isinstance(labels[0], list):
temp = []
for index in range(len(labels[0])):
temp.append([label[index] for label in labels])
labels = temp
for label, hot_vector in zip(labels, hot_vectors):
if not isinstance(label, list):
label = [label]
embeddings.embeddings.append(
Record.Embedding(label=label, vectors=hot_vector))
return Record(values=[
Record.Value(
id=step, tag=tag, timestamp=walltime, embeddings=embeddings)
])
def text(tag, text_string, step, walltime=None):
"""Package data to one image.
Args:
tag (string): Data identifier
text_string (string): Value of text
step (int): Step of text
walltime (int): Wall time of text
Return:
Package with format of record_pb2.Record
"""
_text = Record.Text(encoded_text_string=text_string)
return Record(values=[
Record.Value(id=step, tag=tag, timestamp=walltime, text=_text)
])
def scalar(tag, value, step, walltime=None):
"""Package data to one scalar.
Args:
tag (string): Data identifier
value (float): Value of scalar
step (int): Step of scalar
walltime (int): Wall time of scalar
Return:
Package with format of record_pb2.Record
"""
value = float(value)
return Record(values=[
Record.Value(id=step, tag=tag, timestamp=walltime, value=value)
])
def histogram(tag, hist, bin_edges, step, walltime):
"""Package data to one histogram.
Args:
tag (string): Data identifier
hist (np.ndarray or list): The values of the histogram
bin_edges (np.ndarray or list): The bin edges
step (int): Step of histogram
walltime (int): Wall time of histogram
Return:
Package with format of record_pb2.Record
"""
histogram = Record.Histogram(hist=hist, bin_edges=bin_edges)
return Record(values=[
Record.Value(id=step, tag=tag, timestamp=walltime, histogram=histogram)
])
def image(tag, image_array, step, walltime=None):
"""Package data to one image.
Args:
tag (string): Data identifier
image_array (numpy.ndarray): Value of iamge
step (int): Step of image
walltime (int): Wall time of image
Return:
Package with format of record_pb2.Record
"""
image_bytes = imgarray2bytes(image_array)
image = Record.Image(encoded_image_string=image_bytes)
return Record(values=[
Record.Value(id=step, tag=tag, timestamp=walltime, image=image)
])
def pr_curve_raw(tag, tp, fp, tn, fn, precision, recall, step, walltime):
"""Package raw data to one pr_curve.
Args:
tag (string): Data identifier
tp (list): True Positive.
fp (list): False Positive.
tn (list): True Negative.
fn (list): False Negative.
precision (list): The fraction of retrieved documents that are relevant
to the query:
recall (list): The fraction of the relevant documents that are
successfully retrieved.
step (int): Step of pr_curve
walltime (int): Wall time of pr_curve
num_thresholds (int): Number of thresholds used to draw the curve.
weights (float): Multiple of data to display on the curve.
Return:
Package with format of record_pb2.Record
"""
"""
if isinstance(tp, np.ndarray):
tp = tp.astype(int).tolist()
if isinstance(fp, np.ndarray):
fp = fp.astype(int).tolist()
if isinstance(tn, np.ndarray):
tn = tn.astype(int).tolist()
if isinstance(fn, np.ndarray):
fn = fn.astype(int).tolist()
if isinstance(precision, np.ndarray):
precision = precision.astype(int).tolist()
if isinstance(recall, np.ndarray):
recall = recall.astype(int).tolist()
"""
prcurve = Record.PRCurve(TP=tp,
FP=fp,
TN=tn,
FN=fn,
precision=precision,
recall=recall)
return Record(values=[
Record.Value(
id=step, tag=tag, timestamp=walltime, pr_curve=prcurve)
])
def audio(tag, audio_array, sample_rate, step, walltime):
"""Package data to one audio.
Args:
tag (string): Data identifier
audio_array (numpy.ndarray or list): audio represented by a numpy.array
sample_rate (int): Sample rate of audio
step (int): Step of audio
walltime (int): Wall time of audio
Return:
Package with format of record_pb2.Record
"""
audio_array = audio_array.squeeze()
if abs(audio_array).max() > 1:
print('warning: audio amplitude out of range, auto clipped.')
audio_array = audio_array.clip(-1, 1)
assert (audio_array.ndim == 1), 'input tensor should be 1 dimensional.'
audio_array = [int(32767.0 * x) for x in audio_array]
import io
import wave
import struct
fio = io.BytesIO()
wave_writer = wave.open(fio, 'wb')
wave_writer.setnchannels(1)
wave_writer.setsampwidth(2)
wave_writer.setframerate(sample_rate)
audio_enc = b''
audio_enc += struct.pack("<" + "h" * len(audio_array), *audio_array)
wave_writer.writeframes(audio_enc)
wave_writer.close()
audio_string = fio.getvalue()
fio.close()
audio_data = Record.Audio(
sample_rate=sample_rate,
num_channels=1,
length_frames=len(audio_array),
encoded_audio_string=audio_string,
content_type='audio/wav')
return Record(values=[
Record.Value(id=step, tag=tag, timestamp=walltime, audio=audio_data)
])
def meta_data(tag='meta_data_tag', display_name="", step=0, walltime=None):
"""Package data to one meta_data.
Meta data is info for one record file, include `display_name` etc.
Args:
tag (string): Data identifier
display_name (string): Replace
step (int): Step of scalar
walltime (int): Wall time of scalar
Return:
Package with format of record_pb2.Record
"""
meta = Record.MetaData(display_name=display_name)
return Record(values=[
Record.Value(id=step, tag=tag, timestamp=walltime, meta_data=meta)
])
def hparam(name, hparam_dict, metric_list, walltime):
"""Package data to one histogram.
Args:
name (str): Name of hparam.
hparam_dict (dictionary): Each key-value pair in the dictionary is the
name of the hyper parameter and it's corresponding value. The type of the value
can be one of `bool`, `string`, `float`, `int`, or `None`.
metric_list (list): Name of all metrics.
walltime (int): Wall time of hparam.
Return:
Package with format of record_pb2.Record
"""
hm = Record.HParam()
hm.name = name
for k, v in hparam_dict.items():
if v is None:
continue
hparamInfo = Record.HParam.HparamInfo()
hparamInfo.name = k
if isinstance(v, int):
hparamInfo.int_value = v
hm.hparamInfos.append(hparamInfo)
elif isinstance(v, float):
hparamInfo.float_value = v
hm.hparamInfos.append(hparamInfo)
elif isinstance(v, str):
hparamInfo.string_value = v
hm.hparamInfos.append(hparamInfo)
else:
print("The value of %s must be int, float or str, not %s" %
(k, str(type(v))))
for metric in metric_list:
metricInfo = Record.HParam.HparamInfo()
metricInfo.name = metric
metricInfo.float_value = 0
hm.metricInfos.append(metricInfo)
return Record(values=[
Record.Value(id=1, tag="hparam", timestamp=walltime, hparam=hm)
])
def image(tag, image_array, step, walltime=None, dataformats="HWC"):
"""Package data to one image.
Args:
tag (string): Data identifier
image_array (np.ndarray): Value of image
step (int): Step of image
walltime (int): Wall time of image
dataformats (string): Format of image
Return:
Package with format of record_pb2.Record
"""
image_array = denormalization(image_array)
image_array = convert_to_HWC(image_array, dataformats)
image_bytes = imgarray2bytes(image_array)
image = Record.Image(encoded_image_string=image_bytes)
return Record(values=[
Record.Value(id=step, tag=tag, timestamp=walltime, image=image)
])
def embedding(tag, labels, hot_vectors, step, walltime=None):
"""Package data to one embedding.
Args:
tag (string): Data identifier
labels (numpy.array or list): A list of labels.
hot_vectors (numpy.array or list): A matrix which each row is
feature of labels.
step (int): Step of embeddings.
walltime (int): Wall time of embeddings.
Return:
Package with format of record_pb2.Record
"""
embeddings = Record.Embeddings()
for index in range(len(hot_vectors)):
embeddings.embeddings.append(
Record.Embedding(label=labels[index], vectors=hot_vectors[index]))
return Record(values=[
Record.Value(
id=step, tag=tag, timestamp=walltime, embeddings=embeddings)
])