def __init__(self, logdir, max_queue_size=10, flush_secs=120,
filename_suffix='', filename=''):
self._logdir = logdir
if not bfile.exists(logdir):
bfile.makedirs(logdir)
if filename:
if 'vdlrecords' in filename:
self._file_name = bfile.join(logdir, filename)
if bfile.exists(self._file_name):
print(
'`{}` is exists, VisualDL will add logs to it.'.format(
self._file_name))
else:
fn = "vdlrecords.%010d.log%s" % (time.time(), filename_suffix)
self._file_name = bfile.join(logdir, fn)
print(
'Since the log filename should contain `vdlrecords`, the filename is invalid and `{}` will replace `{}`'.format(
fn, filename))
else:
self._file_name = bfile.join(logdir, "vdlrecords.%010d.log%s" % (
time.time(), filename_suffix))
self._general_file_writer = bfile.BFile(self._file_name, "wb")
self._async_writer = _AsyncWriter(RecordWriter(
self._general_file_writer), max_queue_size, flush_secs)
# TODO(shenyuhan) Maybe file_version in future.
# _record = record_pb2.Record()
# self.add_record(_record)
self.flush()
def get_vdl_log_file(logdirs):
"""Get logs.
Every dir(means `run` in vdl) has only one log(meads `actual log file`).
Returns:
walks: A dict like {"exp1": "vdlrecords.1587375595.log",
"exp2": "vdlrecords.1587375685.log"}
"""
walks = {}
for logdir in logdirs:
for root, dirs, files in bfile.walk(logdir):
walks.update({root: files})
walks_temp = {}
for run, tags in walks.items():
tags_temp = [
tag for tag in tags
if is_VDLRecord_file(path=bfile.join(run, tag),
check=False)
]
tags_temp.sort(reverse=True)
if len(tags_temp) > 0:
walks_temp.update({run: tags_temp[0]})
return walks_temp
def parse_from_bin(self, record_bin):
"""Register to self._tags by component type.
Args:
record_bin: Binary data from vdl log file.
"""
record = record_pb2.Record()
record.ParseFromString(record_bin)
value = record.values[0]
tag = decode_tag(value.tag)
path = bfile.join(self.reader.dir, tag)
if path not in self._tags.keys():
value_type = value.WhichOneof("one_value")
if "value" == value_type:
component = "scalar"
elif "image" == value_type:
component = "image"
elif "embeddings" == value_type:
component = "embeddings"
elif "audio" == value_type:
component = "audio"
else:
raise TypeError("Invalid value type `%s`." % value_type)
self._tags[path] = component
return self._tags[path], self.reader.dir, tag, value
def register_readers(self, update=False):
"""Register all readers for all vdl log files.
Args:
update: Need update if `update` is True.
"""
self.logs(update)
for dir, path in self.walks.items():
filepath = bfile.join(dir, path)
self.register_reader(filepath, dir)
def get_log_data(self, component, run, tag):
if (run in self._log_datas.keys() and
component in self._log_datas[run].keys() and
tag in self._log_datas[run][component].keys()):
return self._log_datas[run][component][tag]
else:
file_path = bfile.join(run, self.walks[run])
reader = self._get_file_reader(file_path=file_path, update=False)
remain = self.get_remain(reader=reader)
data = self.read_log_data(remain=remain, update=False)[component][tag]
data = self.parsing_from_proto(component, data)
self._log_datas[run][component][tag] = data
return data
def __init__(self,
logdir,
max_queue_size=10,
flush_secs=120,
filename_suffix=''):
self._logdir = logdir
if not bfile.exists(logdir):
bfile.makedirs(logdir)
self._file_name = bfile.join(
logdir, "vdlrecords.%010d.log%s" % (time.time(), filename_suffix))
self._general_file_writer = bfile.BFile(self._file_name, "wb")
self._async_writer = _AsyncWriter(
RecordWriter(self._general_file_writer), max_queue_size,
flush_secs)
# TODO(shenyuhan) Maybe file_version in future.
# _record = record_pb2.Record()
# self.add_record(_record)
self.flush()
def get_log_reader(self, dir, log):
"""Get log reader for every vdl log file.
Get instance of class RecordReader base on BFile. Note that each
`log` may contain multi `tag`.
Args:
dir: Dir name of log.
log: Vdl log file name.
"""
if self.walks is None:
self.logs()
if self.walks.get(dir, None) != log:
raise FileNotFoundError("Can't find file %s.", (dir + "/" + log))
filepath = bfile.join(dir, log)
if filepath not in self.readers.keys():
self.register_reader(filepath, dir)
self.reader = self.readers[filepath]
return self.reader
