def test_chart_pr_curves(self):
recall_axis = np.array([1, 2, 3, 4, 5, 6])
curve_info = {
1: {
'recalls': recall_axis,
'precisions': 2 * recall_axis,
'avg_prec': 0.6,
'best_op_pt': {
'threshold': 0.6,
'f1': 0.82,
'rec': 2,
'prec': 4,
}
},
2: {
'recalls': recall_axis,
'precisions': 0.5 * recall_axis,
'avg_prec': 0.8,
}
}
num_classes, fig = \
ClassificationPlotter.chart_pr_curves(curve_info)
self.assertEqual(num_classes, 2)
self.assertEqual(len(fig.axes), 1)
ax = fig.axes[0]
self.assertEqual(ax.get_xlabel(), 'recall')
self.assertEqual(ax.get_ylabel(), 'precision')
# Allow the fig to show
# before asking user to
# check it (the pause())
fig.show()
plt.pause(0.001)
fig_ok = FileUtils.user_confirm(f"Fig should have 2 lines, one point, and a legend\n" +\
f"Looks OK? (Y/n")
if not fig_ok:
self.fail("PR curve was not correct")
def create_csv_writer(self, raw_data_dir):
'''
Create a csv_writer that will fill a csv
file during training/validation as follows:
epoch train_preds train_labels val_preds val_labels
Cols after the integer 'epoch' col will each be
an array of ints:
train_preds train_lbls val_preds val_lbls
2,"[2,5,1,2,3]","[2,6,1,2,1]","[1,2]", "[1,3]"
If raw_data_dir is provided as a str, it is
taken as the directory where csv file with predictions
and labels are to be written. The dir is created if necessary.
If the arg is instead set to True, a dir 'runs_raw_results' is
created under this script's directory if it does not
exist. Then a subdirectory is created for this run,
using the hparam settings to build a file name. The dir
is created if needed. Result ex.:
<script_dir>
runs_raw_results
Run_lr_0.001_br_32
run_2021_05_ ... _lr_0.001_br_32.csv
Then file name is created, again from the run
hparam settings. If this file exists, user is asked whether
to remove or append. The inst var self.csv_writer is
initialized to:
o None if csv file exists, but is not to
be overwritten nor appended-to
o A filed descriptor for a file open for either
'write' or 'append.
:param raw_data_dir: If simply True, create dir and file names
from hparams, and create as needed. If a string, it is
assumed to be the directory where a .csv file is to be
created. If None, self.csv_writer is set to None.
:type raw_data_dir: {None | True | str|
:return: CSV writer ready for action. Set either to
write a fresh file, or append to an existing file.
Unless file exists, and user decided not to overwrite
:rtype: {None | csv.writer}
'''
# Ensure the csv file root dir exists if
# we'll do a csv dir and run-file below it:
if type(raw_data_dir) == str:
raw_data_root = raw_data_dir
else:
raw_data_root = os.path.join(self.curr_dir, 'runs_raw_results')
if not os.path.exists(raw_data_root):
os.mkdir(raw_data_root)
# Can rely on raw_data_root being defined and existing:
if raw_data_dir is None:
return None
# Create both a raw dir sub-directory and a .csv file
# for this run:
csv_subdir_name = FileUtils.construct_filename(self.config.Training,
prefix='Run',
incl_date=True)
os.makedirs(csv_subdir_name)
# Create a csv file name:
csv_file_nm = FileUtils.construct_filename(self.config.Training,
prefix='run',
suffix='.csv',
incl_date=True)
csv_path = os.path.join(raw_data_root, csv_file_nm)
# Get csv_raw_fd appropriately:
if os.path.exists(csv_path):
do_overwrite = FileUtils.user_confirm(
f"File {csv_path} exists; overwrite?", default='N')
if not do_overwrite:
do_append = FileUtils.user_confirm(f"Append instead?",
default='N')
if not do_append:
return None
else:
mode = 'a'
else:
mode = 'w'
csv_writer = CSVWriterCloseable(csv_path, mode=mode, delimiter=',')
header = [
'epoch', 'train_preds', 'train_labels', 'val_preds', 'val_labels'
]
csv_writer.writerow(header)
return csv_writer