def __init__(self, logdir, max_queue_size=10, flush_secs=120, filename_suffix='', filename=''): self._logdir = logdir if not bfile.exists(logdir): bfile.makedirs(logdir) if filename: if 'vdlrecords' in filename: self._file_name = bfile.join(logdir, filename) if bfile.exists(self._file_name): print( '`{}` is exists, VisualDL will add logs to it.'.format( self._file_name)) else: fn = "vdlrecords.%010d.log%s" % (time.time(), filename_suffix) self._file_name = bfile.join(logdir, fn) print( 'Since the log filename should contain `vdlrecords`, the filename is invalid and `{}` will replace `{}`'.format( fn, filename)) else: self._file_name = bfile.join(logdir, "vdlrecords.%010d.log%s" % ( time.time(), filename_suffix)) self._general_file_writer = bfile.BFile(self._file_name, "wb") self._async_writer = _AsyncWriter(RecordWriter( self._general_file_writer), max_queue_size, flush_secs) # TODO(shenyuhan) Maybe file_version in future. # _record = record_pb2.Record() # self.add_record(_record) self.flush()
def get_vdl_log_file(logdirs): """Get logs. Every dir(means `run` in vdl) has only one log(meads `actual log file`). Returns: walks: A dict like {"exp1": "vdlrecords.1587375595.log", "exp2": "vdlrecords.1587375685.log"} """ walks = {} for logdir in logdirs: for root, dirs, files in bfile.walk(logdir): walks.update({root: files}) walks_temp = {} for run, tags in walks.items(): tags_temp = [ tag for tag in tags if is_VDLRecord_file(path=bfile.join(run, tag), check=False) ] tags_temp.sort(reverse=True) if len(tags_temp) > 0: walks_temp.update({run: tags_temp[0]}) return walks_temp
def parse_from_bin(self, record_bin): """Register to self._tags by component type. Args: record_bin: Binary data from vdl log file. """ record = record_pb2.Record() record.ParseFromString(record_bin) value = record.values[0] tag = decode_tag(value.tag) path = bfile.join(self.reader.dir, tag) if path not in self._tags.keys(): value_type = value.WhichOneof("one_value") if "value" == value_type: component = "scalar" elif "image" == value_type: component = "image" elif "embeddings" == value_type: component = "embeddings" elif "audio" == value_type: component = "audio" else: raise TypeError("Invalid value type `%s`." % value_type) self._tags[path] = component return self._tags[path], self.reader.dir, tag, value
def register_readers(self, update=False): """Register all readers for all vdl log files. Args: update: Need update if `update` is True. """ self.logs(update) for dir, path in self.walks.items(): filepath = bfile.join(dir, path) self.register_reader(filepath, dir)
def get_log_data(self, component, run, tag): if (run in self._log_datas.keys() and component in self._log_datas[run].keys() and tag in self._log_datas[run][component].keys()): return self._log_datas[run][component][tag] else: file_path = bfile.join(run, self.walks[run]) reader = self._get_file_reader(file_path=file_path, update=False) remain = self.get_remain(reader=reader) data = self.read_log_data(remain=remain, update=False)[component][tag] data = self.parsing_from_proto(component, data) self._log_datas[run][component][tag] = data return data
def __init__(self, logdir, max_queue_size=10, flush_secs=120, filename_suffix=''): self._logdir = logdir if not bfile.exists(logdir): bfile.makedirs(logdir) self._file_name = bfile.join( logdir, "vdlrecords.%010d.log%s" % (time.time(), filename_suffix)) self._general_file_writer = bfile.BFile(self._file_name, "wb") self._async_writer = _AsyncWriter( RecordWriter(self._general_file_writer), max_queue_size, flush_secs) # TODO(shenyuhan) Maybe file_version in future. # _record = record_pb2.Record() # self.add_record(_record) self.flush()
def get_log_reader(self, dir, log): """Get log reader for every vdl log file. Get instance of class RecordReader base on BFile. Note that each `log` may contain multi `tag`. Args: dir: Dir name of log. log: Vdl log file name. """ if self.walks is None: self.logs() if self.walks.get(dir, None) != log: raise FileNotFoundError("Can't find file %s.", (dir + "/" + log)) filepath = bfile.join(dir, log) if filepath not in self.readers.keys(): self.register_reader(filepath, dir) self.reader = self.readers[filepath] return self.reader