def get_scalar(log_reader, run, tag):
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("scalar").get_items(
run, decode_tag(tag))
results = [[s2ms(item.timestamp), item.id, item.value] for item in records]
return results
def get_individual_audio(log_reader, run, tag, step_index):
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("audio").get_items(
run, decode_tag(tag))
result = records[step_index].audio.encoded_audio_string
return result
def parse_from_bin(self, record_bin):
"""Register to self._tags by component type.
Args:
record_bin: Binary data from vdl log file.
"""
record = record_pb2.Record()
record.ParseFromString(record_bin)
value = record.values[0]
tag = decode_tag(value.tag)
path = bfile.join(self.reader.dir, tag)
if path not in self._tags.keys():
value_type = value.WhichOneof("one_value")
if "value" == value_type:
component = "scalar"
elif "image" == value_type:
component = "image"
elif "embeddings" == value_type:
component = "embeddings"
elif "audio" == value_type:
component = "audio"
else:
raise TypeError("Invalid value type `%s`." % value_type)
self._tags[path] = component
return self._tags[path], self.reader.dir, tag, value
def get_pr_curve_step(log_reader, run, tag=None):
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
tag = get_pr_curve_tags(log_reader)[run][0] if tag is None else tag
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("pr_curve").get_items(
run, decode_tag(tag))
results = [[item.timestamp, item.id] for item in records]
return results
def get_pr_curve_step(log_reader, run, tag=None):
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
run2tag = get_pr_curve_tags(log_reader)
tag = run2tag['tags'][run2tag['runs'].index(run)][0]
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("pr_curve").get_items(
run, decode_tag(tag))
results = [[s2ms(item.timestamp), item.id] for item in records]
return results
def get_audio_tag_steps(log_reader, run, tag):
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("audio").get_items(
run, decode_tag(tag))
result = [{
"step": item.id,
"wallTime": item.timestamp
} for item in records]
return result
def get_audio_tag_steps(log_reader, run, tag):
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("audio").get_items(
run, decode_tag(tag))
result = [{
"step": item.id,
"wallTime": s2ms(item.timestamp)
} for item in records]
return result
def get_hparam_metric(log_reader, run, tag):
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("scalar").get_items(
run, decode_tag(tag))
results = [[
s2ms(item.timestamp), item.id,
transfer_abnomal_scalar_value(item.value)
] for item in records]
return results
def get_embedding_tensors(log_reader, name):
run = EMBEDDING_NAME[name]['run']
tag = EMBEDDING_NAME[name]['tag']
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("embeddings").get_items(
run, decode_tag(tag))
vectors = []
for item in records[0].embeddings.embeddings:
vectors.append(item.vectors)
vectors = np.array(vectors).flatten().astype(np.float32).tobytes()
return vectors
def get_scalar_data(log_reader, run, tag, type='tsv'):
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
log_reader.load_new_data()
result = log_reader.get_log_data('scalar', run, decode_tag(tag))
delimeter = '\t' if 'tsv' == type else ','
with io.StringIO() as fp:
csv_writer = csv.writer(fp, delimiter=delimeter)
csv_writer.writerow(['id', 'tag', 'timestamp', 'value'])
csv_writer.writerows(result)
result = fp.getvalue()
return result
def get_embeddings_list(log_reader):
run2tag = get_logs(log_reader, 'embeddings')
for run, _tags in zip(run2tag['runs'], run2tag['tags']):
for tag in _tags:
name = path = os.path.join(run, tag)
if name in EMBEDDING_NAME:
return embedding_names
EMBEDDING_NAME.update({name: {'run': run, 'tag': tag}})
records = log_reader.data_manager.get_reservoir("embeddings").get_items(
run, decode_tag(tag))
row_len = len(records[0].embeddings.embeddings)
col_len = len(records[0].embeddings.embeddings[0].vectors)
shape = [row_len, col_len]
embedding_names.append({'name': name, 'shape': shape, 'path': path})
return embedding_names
def get_embedding_labels(log_reader, name):
run = EMBEDDING_NAME[name]['run']
tag = EMBEDDING_NAME[name]['tag']
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("embeddings").get_items(
run, decode_tag(tag))
labels = []
for item in records[0].embeddings.embeddings:
labels.append([item.label])
with io.StringIO() as fp:
csv_writer = csv.writer(fp, delimiter='\t')
csv_writer.writerows(labels)
labels = fp.getvalue()
# labels = "\n".join(str(i) for i in labels)
return labels
def get_histogram(log_reader, run, tag):
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("histogram").get_items(
run, decode_tag(tag))
results = []
for item in records:
histogram = item.histogram
hist = histogram.hist
bin_edges = histogram.bin_edges
histogram_data = []
for index in range(len(hist)):
histogram_data.append([bin_edges[index], bin_edges[index+1], hist[index]])
results.append([s2ms(item.timestamp), item.id, histogram_data])
return results
def get_roc_curve(log_reader, run, tag):
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("roc_curve").get_items(
run, decode_tag(tag))
results = []
for item in records:
roc_curve = item.roc_curve
length = len(roc_curve.tpr)
num_thresholds = [float(v) / length for v in range(1, length + 1)]
results.append([
s2ms(item.timestamp), item.id,
list(roc_curve.tpr),
list(roc_curve.fpr),
list(roc_curve.TP),
list(roc_curve.FP),
list(roc_curve.TN),
list(roc_curve.FN), num_thresholds
])
return results
def get_pr_curve(log_reader, run, tag):
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("pr_curve").get_items(
run, decode_tag(tag))
results = []
for item in records:
pr_curve = item.pr_curve
length = len(pr_curve.precision)
num_thresholds = [float(v) / length for v in range(1, length + 1)]
results.append([
item.timestamp, item.id,
list(pr_curve.precision),
list(pr_curve.recall),
list(pr_curve.TP),
list(pr_curve.FP),
list(pr_curve.TN),
list(pr_curve.FN), num_thresholds
])
return results
def get_hparam_list(log_reader):
run2tag = get_logs(log_reader, 'hyper_parameters')
runs = run2tag['runs']
results = []
records_list = []
for run in runs:
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir(
"hyper_parameters").get_items(run, decode_tag('hparam'))
records_list.append([records, run])
records_list.sort(key=lambda x: x[0][0].timestamp)
for records, run in records_list:
hparams = {}
for hparamInfo in records[0].hparam.hparamInfos:
hparam_type = hparamInfo.WhichOneof("type")
if "float_value" == hparam_type:
hparams[hparamInfo.name] = hparamInfo.float_value
elif "string_value" == hparam_type:
hparams[hparamInfo.name] = hparamInfo.string_value
elif "int_value" == hparam_type:
hparams[hparamInfo.name] = hparamInfo.int_value
else:
raise TypeError("Invalid hparams param value type `%s`." %
hparam_type)
metrics = {}
for metricInfo in records[0].hparam.metricInfos:
try:
metrics_data = get_hparam_metric(log_reader, run,
metricInfo.name)
metrics[metricInfo.name] = metrics_data[-1][-1]
except:
logger.error(
'Missing data of metrics! Please make sure use add_scalar to log metrics data.'
)
metrics[metricInfo.name] = None
results.append({'name': run, 'hparams': hparams, 'metrics': metrics})
return results
def get_embeddings(log_reader, run, tag, reduction, dimension=2):
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("embeddings").get_items(
run, decode_tag(tag))
labels = []
vectors = []
for item in records[0].embeddings.embeddings:
labels.append(item.label)
vectors.append(item.vectors)
vectors = np.array(vectors)
if reduction == 'tsne':
import visualdl.server.tsne as tsne
low_dim_embs = tsne.tsne(
vectors, dimension, initial_dims=50, perplexity=30.0)
elif reduction == 'pca':
low_dim_embs = simple_pca(vectors, dimension)
return {"embedding": low_dim_embs.tolist(), "labels": labels}
def get_individual_text(log_reader, run, tag, step_index):
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("text").get_items(
run, decode_tag(tag))
return records[step_index].text.encoded_text_string
def get_scalar(log_reader, run, tag):
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("scalar").get_items(
run, decode_tag(tag))
results = [[item.timestamp, item.id, item.value] for item in records]
return results
def get_individual_audio(log_reader, run, tag, step_index):
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("audio").get_items(
run, decode_tag(tag))
result = records[step_index].audio.encoded_image_string
return result
def get_data(self, component, tag):
return self._log_data[component][decode_tag(tag)]
def get_hparam_indicator(log_reader):
run2tag = get_logs(log_reader, 'hyper_parameters')
runs = run2tag['runs']
hparams = {}
metrics = {}
records_list = []
for run in runs:
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir(
"hyper_parameters").get_items(run, decode_tag('hparam'))
records_list.append([records, run])
records_list.sort(key=lambda x: x[0][0].timestamp)
runs = [run for r, run in records_list]
for records, run in records_list:
for hparamInfo in records[0].hparam.hparamInfos:
type = hparamInfo.WhichOneof("type")
if "float_value" == type:
if hparamInfo.name not in hparams.keys():
hparams[hparamInfo.name] = {
'name': hparamInfo.name,
'type': 'continuous',
'values': [hparamInfo.float_value]
}
elif hparamInfo.float_value not in hparams[
hparamInfo.name]['values']:
hparams[hparamInfo.name]['values'].append(
hparamInfo.float_value)
elif "string_value" == type:
if hparamInfo.name not in hparams.keys():
hparams[hparamInfo.name] = {
'name': hparamInfo.name,
'type': 'string',
'values': [hparamInfo.string_value]
}
elif hparamInfo.string_value not in hparams[
hparamInfo.name]['values']:
hparams[hparamInfo.name]['values'].append(
hparamInfo.string_value)
elif "int_value" == type:
if hparamInfo.name not in hparams.keys():
hparams[hparamInfo.name] = {
'name': hparamInfo.name,
'type': 'numeric',
'values': [hparamInfo.int_value]
}
elif hparamInfo.int_value not in hparams[
hparamInfo.name]['values']:
hparams[hparamInfo.name]['values'].append(
hparamInfo.int_value)
else:
raise TypeError("Invalid hparams param value type `%s`." %
type)
for metricInfo in records[0].hparam.metricInfos:
metrics[metricInfo.name] = {
'name': metricInfo.name,
'type': 'continuous',
'values': []
}
for run in runs:
try:
metrics_data = get_hparam_metric(log_reader, run,
metricInfo.name)
metrics[metricInfo.name]['values'].append(
metrics_data[-1][-1])
break
except:
logger.error(
'Missing data of metrics! Please make sure use add_scalar to log metrics data.'
)
if len(metrics[metricInfo.name]['values']) == 0:
metrics.pop(metricInfo.name)
else:
metrics[metricInfo.name].pop('values')
results = {
'hparams': [value for key, value in hparams.items()],
'metrics': [value for key, value in metrics.items()]
}
return results
