def refresh(self):
import os
rows, columns = os.popen('stty size', 'r').read().split()
tabulars = self.tabulars[-(int(rows) - 3):]
sys.stdout.write("\x1b[2J\x1b[H")
sys.stdout.write(tabulate(tabulars, self.headers))
sys.stdout.write("\n")
def dump_tabular(self, *args, **kwargs):
"""
Dump existing tabular data to disk
:param args:
:param kwargs:
:return:
"""
wh = kwargs.pop("write_header", None)
if len(self._tabular) > 0:
if self._log_tabular_only:
self.table_printer.print_tabular(self._tabular)
else:
for line in tabulate(self._tabular).split('\n'):
self.log(line, *args, **kwargs)
tabular_dict = dict(self._tabular)
# Also write to the csv files
# This assumes that the keys in each iteration won't change!
for tabular_fd in list(self._tabular_fds.values()):
writer = csv.DictWriter(tabular_fd,
fieldnames=list(tabular_dict.keys()))
if wh or (wh is None
and tabular_fd not in self._tabular_header_written):
writer.writeheader()
self._tabular_header_written.add(tabular_fd)
writer.writerow(tabular_dict)
tabular_fd.flush()
del self._tabular[:]
def dump_tabular(self, *args, **kwargs):
self.epoch += 1
wh = kwargs.pop("write_header", None)
if len(self._tabular) > 0:
if self._log_tabular_only:
self.table_printer.print_tabular(self._tabular)
else:
for line in tabulate(self._tabular).split('\n'):
self.log(line, *args, **kwargs)
tabular_dict = dict(self._tabular)
# Only saves keys in first iteration to CSV!
# (But every key is printed out in text)
if self._tabular_keys is None:
self._tabular_keys = list(sorted(tabular_dict.keys()))
# Write to the csv files
for tabular_fd in list(self._tabular_fds.values()):
writer = csv.DictWriter(
tabular_fd,
fieldnames=self._tabular_keys,
extrasaction="ignore",
)
if wh or (wh is None
and tabular_fd not in self._tabular_header_written):
writer.writeheader()
self._tabular_header_written.add(tabular_fd)
writer.writerow(tabular_dict)
tabular_fd.flush()
del self._tabular[:]
def dump_tabular(self, *args, **kwargs):
wh = kwargs.pop("write_header", None)
if len(self._tabular) > 0:
if self._log_tabular_only:
self.table_printer.print_tabular(self._tabular)
pass
else:
for line in tabulate(self._tabular).split('\n'):
self.log(line, *args, **kwargs)
pass
tabular_dict = dict(self._tabular)
# write to tensorboard
if self._tensorboard_writer is not None:
epoch = tabular_dict['Epoch']
for name, value in tabular_dict.items():
if name != 'Epoch':
self._tensorboard_writer.add_scalar(
name, float(value), epoch)
# Also write to the csv files
# This assumes that the keys in each iteration won't change!
for tabular_fd in list(self._tabular_fds.values()):
writer = csv.DictWriter(tabular_fd,
fieldnames=list(tabular_dict.keys()))
if wh or (wh is None
and tabular_fd not in self._tabular_header_written):
writer.writeheader()
self._tabular_header_written.add(tabular_fd)
writer.writerow(tabular_dict)
tabular_fd.flush()
del self._tabular[:]
def dump_tabular(*args, **kwargs):
wh = kwargs.pop("write_header", None)
if len(_tabular) > 0:
if _log_tabular_only:
table_printer.print_tabular(_tabular)
else:
for line in tabulate(_tabular).split('\n'):
log(line, *args, **kwargs)
tabular_dict = dict(_tabular)
# Also write to the csv files
# This assumes that the keys in each iteration won't change!
for tabular_fd in list(_tabular_fds.values()):
writer = csv.DictWriter(tabular_fd,
fieldnames=list(tabular_dict.keys()))
if wh or (wh is None and tabular_fd not in _tabular_header_written):
writer.writeheader()
_tabular_header_written.add(tabular_fd)
writer.writerow(tabular_dict)
tabular_fd.flush()
del _tabular[:]
def dump_tabular(self, *args, **kwargs):
wh = kwargs.pop("write_header", None)
if len(self._tabular) > 0:
if self._log_tabular_only:
#TODO: what
pass
# self.table_printer.print_tabular(self._tabular)
else:
for line in tabulate(self._tabular).split('\n'):
self.log(line, *args, **kwargs)
tabular_dict = dict(self._tabular)
# Also write to the csv files
for filename, tabular_fd in list(self._tabular_fds.items()):
# Only saves keys in first iteration to CSV!
# (But every key is printed out in text)
itr0_keys = self._tabular_keys.get(filename)
if itr0_keys is None:
itr0_keys = list(sorted(tabular_dict.keys()))
self._tabular_keys[filename] = itr0_keys
else:
prev_keys = set(itr0_keys)
curr_keys = set(tabular_dict.keys())
if curr_keys != prev_keys:
print("Warning: CSV key mismatch")
print("extra keys in 0th iter", prev_keys - curr_keys)
print("extra keys in cur iter", curr_keys - prev_keys)
writer = csv.DictWriter(tabular_fd,
fieldnames=itr0_keys,
extrasaction="ignore",)
if wh or (
wh is None and tabular_fd not in self._tabular_header_written):
writer.writeheader()
self._tabular_header_written.add(tabular_fd)
writer.writerow(tabular_dict)
tabular_fd.flush()
try:
del self._tabular[:]
except:
print("FOOBAR log fail")
def dump_tabular(self, *args, **kwargs):
# print('in dump_tabular')
wh = kwargs.pop("write_header", None)
if len(self._tabular) > 0:
if self._log_tabular_only:
self.table_printer.print_tabular(self._tabular)
else:
file_name = kwargs.pop('file_name', None)
file_name2 = kwargs.pop('file_name2', None)
d1 = {}
d2 = {}
try:
d2 = copy.deepcopy(torch.load(file_name2))
except:
pass
for item in self._tabular:
d1[item[0]] = item[1]
for line in tabulate(self._tabular).split('\n'):
self.log(line, *args, **kwargs, file_name=file_name)
if file_name2:
d1_copy = copy.deepcopy(d1)
d2[d1_copy['Epoch']] = d1_copy
d2_copy = copy.deepcopy(d2)
torch.save(d2_copy, file_name2)
tabular_dict = dict(self._tabular)
# Also write to the csv files
# This assumes that the keys in each iteration won't change!
for tabular_fd in list(self._tabular_fds.values()):
writer = csv.DictWriter(tabular_fd,
fieldnames=list(tabular_dict.keys()))
if wh or (wh is None
and tabular_fd not in self._tabular_header_written):
writer.writeheader()
self._tabular_header_written.add(tabular_fd)
writer.writerow(tabular_dict)
tabular_fd.flush()
del self._tabular[:]
def dump_tabular(self, *args, **kwargs):
self.epoch += 1
wh = kwargs.pop("write_header", None)
# if len(self._tabular_fds_to_reopen) > 0:
# self._tabular_fds = {}
# for k, fd in self._tabular_fds_to_reopen.items():
# new_fd = open(fd.name, 'a')
# self._tabular_fds[k] = new_fd
# if fd in self._tabular_header_written:
# self._tabular_header_written.remove(fd)
# self._tabular_header_written.add(new_fd)
if len(self._tabular) > 0:
if self._log_tabular_only:
self.table_printer.print_tabular(self._tabular)
else:
for line in tabulate(self._tabular).split('\n'):
self.log(line, *args, **kwargs)
tabular_dict = dict(self._tabular)
# Only saves keys in first iteration to CSV!
# (But every key is printed out in text)
if self._tabular_keys is None:
self._tabular_keys = list(sorted(tabular_dict.keys()))
# Write to the csv files
for tabular_fd in list(self._tabular_fds.values()):
writer = csv.DictWriter(tabular_fd,
fieldnames=self._tabular_keys,
extrasaction="ignore",)
if wh or (
wh is None and tabular_fd not in self._tabular_header_written):
writer.writeheader()
self._tabular_header_written.add(tabular_fd)
writer.writerow(tabular_dict)
tabular_fd.flush()
if self.reopen_files_on_flush:
# self._tabular_fds_to_reopen = {}
new_tabular_fds = {}
for k, fd in self._tabular_fds.items():
# import shutil
# from pathlib import Path
# base_path = Path(fd.name)
# copy_path = str(
# base_path.parent / '{}{}'.format(
# base_path.stem + '_copy',
# base_path.suffix
# )
# )
# shutil.copy(fd.name, copy_path)
new_fd = reopen(fd)
new_tabular_fds[k] = new_fd
if fd in self._tabular_header_written:
self._tabular_header_written.remove(fd)
self._tabular_header_written.add(new_fd)
# fd.close()
# self._tabular_fds_to_reopen[k] = fd
self._tabular_fds = new_tabular_fds
# self._tabular_fds_to_reopen = {}
del self._tabular[:]
