def _save_images(self, data: Data):
for key in self.inputs:
if key in data:
imgs = data[key]
im_path = os.path.join(self.save_dir,
"{}_{}_epoch_{}.png".format(key, self.system.mode, self.system.epoch_idx))
if isinstance(imgs, ImgData):
f = imgs.paint_figure()
plt.savefig(im_path, dpi=self.dpi, bbox_inches="tight")
plt.close(f)
print("FastEstimator-ImageSaver: saved image to {}".format(im_path))
elif isinstance(imgs, Summary):
visualize_logs([imgs], save_path=im_path, dpi=self.dpi, verbose=False)
print("FastEstimator-ImageSaver: saved image to {}".format(im_path))
elif isinstance(imgs, (list, tuple)) and all([isinstance(img, Summary) for img in imgs]):
visualize_logs(imgs, save_path=im_path, dpi=self.dpi, verbose=False)
print("FastEstimator-ImageSaver: saved image to {}".format(im_path))
else:
for idx, img in enumerate(imgs):
f = show_image(img, title=key)
im_path = os.path.join(
self.save_dir,
"{}_{}_epoch_{}_elem_{}.png".format(key, self.system.mode, self.system.epoch_idx, idx))
plt.savefig(im_path, dpi=self.dpi, bbox_inches="tight")
plt.close(f)
print("FastEstimator-ImageSaver: saved image to {}".format(im_path))
def _document_training_graphs(self) -> None:
"""Add training graphs to the traceability document.
"""
with self.doc.create(Section("Training Graphs")):
log_path = os.path.join(self.resource_dir,
f'{self.report_name}_logs.png')
visualize_logs(experiments=[self.system.summary],
save_path=log_path,
verbose=False,
ignore_metrics={'num_device', 'logging_interval'})
with self.doc.create(Figure(position='h!')) as plot:
plot.add_image(
os.path.relpath(log_path, start=self.save_dir),
width=NoEscape(
r'1.0\textwidth,height=0.95\textheight,keepaspectratio'
))
for idx, graph in enumerate(self.system.custom_graphs.values()):
graph_path = os.path.join(
self.resource_dir,
f'{self.report_name}_custom_graph_{idx}.png')
visualize_logs(experiments=graph,
save_path=graph_path,
verbose=False)
with self.doc.create(Figure(position='h!')) as plot:
plot.add_image(
os.path.relpath(graph_path, start=self.save_dir),
width=NoEscape(
r'1.0\textwidth,height=0.95\textheight,keepaspectratio'
))
def parse_log_files(file_paths: List[str],
log_extension: Optional[str] = '.txt',
smooth_factor: float = 0,
save: bool = False,
save_path: Optional[str] = None,
ignore_metrics: Optional[Set[str]] = None,
include_metrics: Optional[Set[str]] = None,
share_legend: bool = True,
pretty_names: bool = False,
group_by: Optional[str] = None) -> None:
"""Parse one or more log files for graphing.
This function which will iterate through the given log file paths, parse them to extract metrics, remove any
metrics which are blacklisted, and then pass the necessary information on the graphing function.
Args:
file_paths: A list of paths to various log files.
log_extension: The extension of the log files.
smooth_factor: A non-negative float representing the magnitude of gaussian smoothing to apply (zero for none).
save: Whether to save (True) or display (False) the generated graph.
save_path: Where to save the image if save is true. Defaults to dir_path if not provided.
ignore_metrics: Any metrics within the log files which will not be visualized.
include_metrics: A whitelist of metric keys (None whitelists all keys).
share_legend: Whether to have one legend across all graphs (True) or one legend per graph (False).
pretty_names: Whether to modify the metric names in graph titles (True) or leave them alone (False).
group_by: Combine multiple log files by a regex to visualize their mean+-stddev. For example, to group together
files like [a_1.txt, a_2.txt] vs [b_1.txt, b_2.txt] you can use: r'(.*)_[\d]+\.txt'.
Raises:
AssertionError: If no log files are provided.
ValueError: If a log file does not match the `group_by` regex pattern.
"""
if file_paths is None or len(file_paths) < 1:
raise AssertionError("must provide at least one log file")
if save and save_path is None:
save_path = file_paths[0]
groups = defaultdict(list) # {group_name: [experiment(s)]}
for path in file_paths:
experiment = parse_log_file(path, log_extension)
try:
key = (re.findall(
group_by,
os.path.split(path)[1]))[0] if group_by else experiment.name
except IndexError:
raise ValueError(
f"The log {os.path.split(path)[1]} did not match the given regex pattern: {group_by}"
)
groups[key].append(experiment)
experiments = [
average_summaries(name, exps) for name, exps in groups.items()
]
visualize_logs(experiments,
save_path=save_path,
smooth_factor=smooth_factor,
share_legend=share_legend,
pretty_names=pretty_names,
ignore_metrics=ignore_metrics,
include_metrics=include_metrics)
def _vis_logs(self, args: Dict[str, Any], unknown: List[str]) -> None:
"""A method to collect and visualize a given set of logs from the database.
Args:
args: The CLI arguments provided by the user.
unknown: Any CLI arguments not matching known inputs.
"""
if len(unknown) > 0:
print("unrecognized arguments: ", str.join(", ", unknown))
return
save_dir = args['save_dir']
if args['save'] and save_dir is None:
save_dir = os.path.join(str(Path.home()), 'fastestimator_data',
'logs.png')
group_indices = [x for y in args['groups'].values() for x in y]
pks = {
idx: self.response[idx - 1]['pk']
for idx in set(args['indices'] + group_indices)
}
if len(pks) == 0:
return
with closing(self.db.cursor()) as cursor:
cursor.execute(
"SELECT H.pk, H.experiment, L.log "
"FROM logs L LEFT JOIN history H ON L.fk = H.pk "
"WHERE L.fk IN ({seq})".format(seq=','.join(['?'] * len(pks))),
list(pks.values()))
logs = cursor.fetchall()
logs = {elem['pk']: elem for elem in logs}
failures = 0
for idx, pk in pks.items():
if pk not in logs:
print(f"No logs found for Index {idx}")
failures += 1
if failures:
return
groups = defaultdict(list) # {group_name: [experiment(s)]}
for idx, pk in pks.items():
log = logs[pk]
experiment = parse_log_iter(
log['log'].split('\n'),
Summary(
str(idx) if log['experiment'] is None else
f"{log['experiment']} ({idx})"))
if idx in args['indices']:
groups[experiment.name].append(experiment)
for group_name, group_indices in args['groups'].items():
if idx in group_indices:
groups[group_name].append(experiment)
experiments = [
average_summaries(name, exps) for name, exps in groups.items()
]
visualize_logs(experiments,
save_path=save_dir,
smooth_factor=args['smooth'],
share_legend=args['share_legend'],
pretty_names=args['pretty_names'],
ignore_metrics=args['ignore'],
include_metrics=args['include'])
def parse_log_files(file_paths: List[str],
log_extension: Optional[str] = '.txt',
smooth_factor: float = 0,
save: bool = False,
save_path: Optional[str] = None,
ignore_metrics: Optional[Set[str]] = None,
share_legend: bool = True,
pretty_names: bool = False) -> None:
"""Parse one or more log files for graphing.
This function which will iterate through the given log file paths, parse them to extract metrics, remove any
metrics which are blacklisted, and then pass the necessary information on the graphing function.
Args:
file_paths: A list of paths to various log files.
log_extension: The extension of the log files.
smooth_factor: A non-negative float representing the magnitude of gaussian smoothing to apply (zero for none).
save: Whether to save (True) or display (False) the generated graph.
save_path: Where to save the image if save is true. Defaults to dir_path if not provided.
ignore_metrics: Any metrics within the log files which will not be visualized.
share_legend: Whether to have one legend across all graphs (True) or one legend per graph (False).
pretty_names: Whether to modify the metric names in graph titles (True) or leave them alone (False).
Raises:
AssertionError: If no log files are provided.
"""
if file_paths is None or len(file_paths) < 1:
raise AssertionError("must provide at least one log file")
if save and save_path is None:
save_path = file_paths[0]
experiments = []
for file_path in file_paths:
experiments.append(parse_log_file(file_path, log_extension))
visualize_logs(experiments,
save_path=save_path,
smooth_factor=smooth_factor,
share_legend=share_legend,
pretty_names=pretty_names,
ignore_metrics=ignore_metrics)
def _display_images(self, data: Data) -> None:
"""A method to render images to the screen.
Args:
data: Data possibly containing images to render.
"""
for key in self.inputs:
if key in data:
imgs = data[key]
if isinstance(imgs, ImgData):
fig = imgs.paint_numpy(dpi=96)
plt.imshow(fig[0])
plt.axis('off')
plt.tight_layout()
plt.show()
elif isinstance(imgs, Summary):
visualize_logs([imgs])
elif isinstance(imgs, (list, tuple)) and all(
[isinstance(img, Summary) for img in imgs]):
visualize_logs(imgs)
else:
for idx, img in enumerate(imgs):
show_image(img, title="{}_{}".format(key, idx))
plt.show()
