def refresh(self): import os rows, columns = os.popen('stty size', 'r').read().split() tabulars = self.tabulars[-(int(rows) - 3):] sys.stdout.write("\x1b[2J\x1b[H") sys.stdout.write(tabulate(tabulars, self.headers)) sys.stdout.write("\n")
def dump_tabular(self, *args, **kwargs): """ Dump existing tabular data to disk :param args: :param kwargs: :return: """ wh = kwargs.pop("write_header", None) if len(self._tabular) > 0: if self._log_tabular_only: self.table_printer.print_tabular(self._tabular) else: for line in tabulate(self._tabular).split('\n'): self.log(line, *args, **kwargs) tabular_dict = dict(self._tabular) # Also write to the csv files # This assumes that the keys in each iteration won't change! for tabular_fd in list(self._tabular_fds.values()): writer = csv.DictWriter(tabular_fd, fieldnames=list(tabular_dict.keys())) if wh or (wh is None and tabular_fd not in self._tabular_header_written): writer.writeheader() self._tabular_header_written.add(tabular_fd) writer.writerow(tabular_dict) tabular_fd.flush() del self._tabular[:]
def dump_tabular(self, *args, **kwargs): self.epoch += 1 wh = kwargs.pop("write_header", None) if len(self._tabular) > 0: if self._log_tabular_only: self.table_printer.print_tabular(self._tabular) else: for line in tabulate(self._tabular).split('\n'): self.log(line, *args, **kwargs) tabular_dict = dict(self._tabular) # Only saves keys in first iteration to CSV! # (But every key is printed out in text) if self._tabular_keys is None: self._tabular_keys = list(sorted(tabular_dict.keys())) # Write to the csv files for tabular_fd in list(self._tabular_fds.values()): writer = csv.DictWriter( tabular_fd, fieldnames=self._tabular_keys, extrasaction="ignore", ) if wh or (wh is None and tabular_fd not in self._tabular_header_written): writer.writeheader() self._tabular_header_written.add(tabular_fd) writer.writerow(tabular_dict) tabular_fd.flush() del self._tabular[:]
def dump_tabular(self, *args, **kwargs): wh = kwargs.pop("write_header", None) if len(self._tabular) > 0: if self._log_tabular_only: self.table_printer.print_tabular(self._tabular) pass else: for line in tabulate(self._tabular).split('\n'): self.log(line, *args, **kwargs) pass tabular_dict = dict(self._tabular) # write to tensorboard if self._tensorboard_writer is not None: epoch = tabular_dict['Epoch'] for name, value in tabular_dict.items(): if name != 'Epoch': self._tensorboard_writer.add_scalar( name, float(value), epoch) # Also write to the csv files # This assumes that the keys in each iteration won't change! for tabular_fd in list(self._tabular_fds.values()): writer = csv.DictWriter(tabular_fd, fieldnames=list(tabular_dict.keys())) if wh or (wh is None and tabular_fd not in self._tabular_header_written): writer.writeheader() self._tabular_header_written.add(tabular_fd) writer.writerow(tabular_dict) tabular_fd.flush() del self._tabular[:]
def dump_tabular(*args, **kwargs): wh = kwargs.pop("write_header", None) if len(_tabular) > 0: if _log_tabular_only: table_printer.print_tabular(_tabular) else: for line in tabulate(_tabular).split('\n'): log(line, *args, **kwargs) tabular_dict = dict(_tabular) # Also write to the csv files # This assumes that the keys in each iteration won't change! for tabular_fd in list(_tabular_fds.values()): writer = csv.DictWriter(tabular_fd, fieldnames=list(tabular_dict.keys())) if wh or (wh is None and tabular_fd not in _tabular_header_written): writer.writeheader() _tabular_header_written.add(tabular_fd) writer.writerow(tabular_dict) tabular_fd.flush() del _tabular[:]
def dump_tabular(self, *args, **kwargs): wh = kwargs.pop("write_header", None) if len(self._tabular) > 0: if self._log_tabular_only: #TODO: what pass # self.table_printer.print_tabular(self._tabular) else: for line in tabulate(self._tabular).split('\n'): self.log(line, *args, **kwargs) tabular_dict = dict(self._tabular) # Also write to the csv files for filename, tabular_fd in list(self._tabular_fds.items()): # Only saves keys in first iteration to CSV! # (But every key is printed out in text) itr0_keys = self._tabular_keys.get(filename) if itr0_keys is None: itr0_keys = list(sorted(tabular_dict.keys())) self._tabular_keys[filename] = itr0_keys else: prev_keys = set(itr0_keys) curr_keys = set(tabular_dict.keys()) if curr_keys != prev_keys: print("Warning: CSV key mismatch") print("extra keys in 0th iter", prev_keys - curr_keys) print("extra keys in cur iter", curr_keys - prev_keys) writer = csv.DictWriter(tabular_fd, fieldnames=itr0_keys, extrasaction="ignore",) if wh or ( wh is None and tabular_fd not in self._tabular_header_written): writer.writeheader() self._tabular_header_written.add(tabular_fd) writer.writerow(tabular_dict) tabular_fd.flush() try: del self._tabular[:] except: print("FOOBAR log fail")
def dump_tabular(self, *args, **kwargs): # print('in dump_tabular') wh = kwargs.pop("write_header", None) if len(self._tabular) > 0: if self._log_tabular_only: self.table_printer.print_tabular(self._tabular) else: file_name = kwargs.pop('file_name', None) file_name2 = kwargs.pop('file_name2', None) d1 = {} d2 = {} try: d2 = copy.deepcopy(torch.load(file_name2)) except: pass for item in self._tabular: d1[item[0]] = item[1] for line in tabulate(self._tabular).split('\n'): self.log(line, *args, **kwargs, file_name=file_name) if file_name2: d1_copy = copy.deepcopy(d1) d2[d1_copy['Epoch']] = d1_copy d2_copy = copy.deepcopy(d2) torch.save(d2_copy, file_name2) tabular_dict = dict(self._tabular) # Also write to the csv files # This assumes that the keys in each iteration won't change! for tabular_fd in list(self._tabular_fds.values()): writer = csv.DictWriter(tabular_fd, fieldnames=list(tabular_dict.keys())) if wh or (wh is None and tabular_fd not in self._tabular_header_written): writer.writeheader() self._tabular_header_written.add(tabular_fd) writer.writerow(tabular_dict) tabular_fd.flush() del self._tabular[:]
def dump_tabular(self, *args, **kwargs): self.epoch += 1 wh = kwargs.pop("write_header", None) # if len(self._tabular_fds_to_reopen) > 0: # self._tabular_fds = {} # for k, fd in self._tabular_fds_to_reopen.items(): # new_fd = open(fd.name, 'a') # self._tabular_fds[k] = new_fd # if fd in self._tabular_header_written: # self._tabular_header_written.remove(fd) # self._tabular_header_written.add(new_fd) if len(self._tabular) > 0: if self._log_tabular_only: self.table_printer.print_tabular(self._tabular) else: for line in tabulate(self._tabular).split('\n'): self.log(line, *args, **kwargs) tabular_dict = dict(self._tabular) # Only saves keys in first iteration to CSV! # (But every key is printed out in text) if self._tabular_keys is None: self._tabular_keys = list(sorted(tabular_dict.keys())) # Write to the csv files for tabular_fd in list(self._tabular_fds.values()): writer = csv.DictWriter(tabular_fd, fieldnames=self._tabular_keys, extrasaction="ignore",) if wh or ( wh is None and tabular_fd not in self._tabular_header_written): writer.writeheader() self._tabular_header_written.add(tabular_fd) writer.writerow(tabular_dict) tabular_fd.flush() if self.reopen_files_on_flush: # self._tabular_fds_to_reopen = {} new_tabular_fds = {} for k, fd in self._tabular_fds.items(): # import shutil # from pathlib import Path # base_path = Path(fd.name) # copy_path = str( # base_path.parent / '{}{}'.format( # base_path.stem + '_copy', # base_path.suffix # ) # ) # shutil.copy(fd.name, copy_path) new_fd = reopen(fd) new_tabular_fds[k] = new_fd if fd in self._tabular_header_written: self._tabular_header_written.remove(fd) self._tabular_header_written.add(new_fd) # fd.close() # self._tabular_fds_to_reopen[k] = fd self._tabular_fds = new_tabular_fds # self._tabular_fds_to_reopen = {} del self._tabular[:]