def __init__(self,
model_paths,
samples_path,
batch_size=1,
labels_path=None,
gpu_ids=0):
'''
Given the path to a trained model,
and the path to the root of a set
of data, compute predictions.
If labels_path is None, the subdirectory
names between the samples_path root,
and the samples themselves are used as
the ground truth labels.
By default: run batches of size 1,
because we always have drop_last set
to True. For small test sets leaving
out any data at all isn't good. Caller
can still set batch_size higher to gain
speed if the testset is very large, so that
not inferencing on up to batch_size - 1
samples is OK
:param model_paths:
:type model_paths:
:param samples_path:
:type samples_path:
:param batch_size:
:type batch_size:
:param labels_path:
:type labels_path:
:param gpu_ids: Device number of GPU, in case
one is available
:type gpu_ids: {int | [int]}
'''
self.model_paths = model_paths
self.samples_path = samples_path
self.labels_path = labels_path
self.gpu_ids = gpu_ids if type(gpu_ids) == list else [gpu_ids]
if batch_size is not None:
self.batch_size = batch_size
else:
self.batch_size = 1
self.IMG_EXTENSIONS = FileUtils.IMG_EXTENSIONS
self.log = LoggingService()
self.curr_dir = os.path.dirname(__file__)
def get_weights(cls, file_root):
'''
Given to root of a subdirectory,
return a tensor of weights. The order
of the weights corresponds to the
naturally sorted class names.
:param file_root: full path to root
of data file subtree
:type file_root: str
:return weights in naturally sorted class order
:rtype: Tensor
'''
# Full paths of all the non-dot-starting
# dirs under file_root:
# OrderedDict{class_name : [Path(dir1), Path(dir2)]
# The class names are already sorted:
class_name_paths_dir = FileUtils.find_class_paths(file_root)
# Create:
# {'class1' : <num_samples>,
# 'class2' : <num_samples>,
# ...
# }
class_populations = {}
for class_name in class_name_paths_dir.keys():
num_samples = 0
# Each class may have samples in multiple
# directories; add them up:
for class_dir in class_name_paths_dir[class_name]:
num_samples += len([file_name
for file_name
in os.listdir(class_dir)
if Path(file_name).suffix in FileUtils.IMG_EXTENSIONS
])
class_populations[class_name] = num_samples
if len(class_populations) == 0:
LoggingService().err(f"No target classes found under {file_root}")
sys.exit(1)
majority_class_population = max(class_populations.values())
weights = []
for class_name in class_name_paths_dir.keys():
weights.append(class_populations[class_name] / majority_class_population)
return torch.tensor(weights)
def __init__(self, unittesting=False):
self.hostname = socket.getfqdn()
if unittesting:
# Let unittests create an instance
# and call individual methods:
return
# Logging to console during launch:
self.log = LoggingService()
# Convenience: directory of this
# script, and project root directory
curr_dir = Path(__file__).parent
proj_root = curr_dir.joinpath('../..').resolve()
self.curr_dir = str(curr_dir)
self.proj_root = str(proj_root)
args_parser = BirdsTrainingArgumentsParser(
formatter_class=BirdsTrainingArgumentsParser.
BlankLinesHelpFormatter,
description="PyTorch distributed training launch "
"helper to spawn multiple distributed "
"birds_train_parallel.py processes")
all_args = args_parser.parse_args()
# Separate the args for this launch script
# from the args destined for the copies of
# the train script:
self.launch_args = all_args['launch_args']
self.script_args = all_args['script_args']
# Build the gpu_landscape dict:
self.gather_world_layout(self.launch_args)
self.GPUS_USED_THIS_MACHINE = self.gpu_landscape[
self.hostname]['num_gpus']
def __init__(self, gpu_ids, log=None):
'''
****
:param gpu_ids: ids of GPUs on this machine
that may be used
:type gpu_ids: [int]
'''
if GPUManager.__is_initialized:
return
else:
GPUManager.__is_initialized = True
if len(gpu_ids) == 0:
self.cpu_only = True
else:
self.cpu_only = False
#**************
# No SIGSEGV or SIGABRT yet:
self.hardware_error = False
stacktrace_fd = open(
os.path.join(os.path.dirname(__file__), 'seg_abrt.log'), 'w')
faulthandler.enable(stacktrace_fd)
#**************
self.log = LoggingService() if log is None else log
self.gpus_available = len(gpu_ids)
self.gpu_ids = gpu_ids
self.who_is_who = {}
self.lock = Lock()
# Callback for psutil.wait_proc() to
# call when a process finishes. The
# currying is to get 'self' passed to the
# method, along with the finished process:
self.proc_finished_callback = partial(self.proc_termination_callback,
self)
def test_bad_wav_file(self):
with tempfile.TemporaryDirectory(dir='/tmp',
prefix='test_spectro') as in_dir:
with tempfile.TemporaryDirectory(dir='/tmp',
prefix='test_spectro') as out_dir:
log_file = os.path.join(out_dir, 'err_log.txt')
SpectrogramCreator.log = LoggingService()
SpectrogramCreator.log.log_file=log_file
bad_species_path = os.path.join(in_dir, 'BADBIRD')
os.mkdir(bad_species_path)
bad_bird_fname = 'bad_audio.wav'
assignments = ([('BADBIRD', bad_bird_fname)])
bad_bird_path = os.path.join(bad_species_path, bad_bird_fname)
# Create a 0-length file:
Path(bad_bird_path).touch()
ret_value_slot = mp.Value("b", False)
# Ensure that an error is logged, though
# none is raised:
SpectrogramCreator.create_from_file_list(
assignments,
in_dir,
out_dir,
WhenAlreadyDone.OVERWRITE,
ret_value_slot,
env=None)
# Read the log file:
with open(log_file, 'r') as fd:
log_entry = fd.read()
# The log msg should include:
# "ERROR: One file could not be processed ... AudioLoadException('Audio file to load is empty ..."
self.assertTrue(log_entry.find('to load is empty') > -1)
def __init__(self, config_info, debugging=False):
'''
Constructor
'''
self.log = LoggingService()
if debugging:
self.log.logging_level = DEBUG
self.curr_dir = os.path.dirname(os.path.abspath(__file__))
try:
self.config = self.initialize_config_struct(config_info)
except Exception as e:
msg = f"During config init: {repr(e)}"
self.log.err(msg)
raise RuntimeError(msg) from e
try:
self.root_train_test_data = self.config.getpath(
'Paths', 'root_train_test_data', relative_to=self.curr_dir)
except ValueError as e:
raise ValueError(
"Config file must contain an entry 'root_train_test_data' in section 'Paths'"
) from e
self.batch_size = self.config.getint('Training', 'batch_size')
self.kernel_size = self.config.getint('Training', 'kernel_size')
self.min_epochs = self.config.Training.getint('min_epochs')
self.max_epochs = self.config.Training.getint('max_epochs')
self.lr = self.config.Training.getfloat('lr')
self.net_name = self.config.Training.net_name
self.pretrained = self.config.Training.getboolean('pretrained', False)
self.freeze = self.config.Training.getint('freeze', 0)
self.to_grayscale = self.config.Training.getboolean(
'to_grayscale', True)
self.set_seed(42)
self.log.info("Parameter summary:")
self.log.info(f"network {self.net_name}")
self.log.info(f"pretrained {self.pretrained}")
if self.pretrained:
self.log.info(f"freeze {self.freeze}")
self.log.info(f"min epochs {self.min_epochs}")
self.log.info(f"max epochs {self.max_epochs}")
self.log.info(f"batch_size {self.batch_size}")
self.fastest_device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
self.num_classes = self.find_num_classes(self.root_train_test_data)
self.model = NetUtils.get_net(self.net_name,
num_classes=self.num_classes,
pretrained=self.pretrained,
freeze=self.freeze,
to_grayscale=self.to_grayscale)
self.log.debug(
f"Before any gpu push: \n{'none--on CPU' if self.fastest_device.type == 'cpu' else torch.cuda.memory_summary()}"
)
FileUtils.to_device(self.model, 'gpu')
self.log.debug(
f"Before after model push: \n{'none--on CPU' if self.fastest_device.type == 'cpu' else torch.cuda.memory_summary()}"
)
# No cross validation:
self.folds = 0
self.opt_name = self.config.Training.get('optimizer',
'Adam') # Default
self.optimizer = self.get_optimizer(self.opt_name, self.model, self.lr)
self.loss_fn = nn.CrossEntropyLoss()
self.scheduler = optim.lr_scheduler.CosineAnnealingLR(
self.optimizer, self.min_epochs)
sample_width = self.config.getint('Training', 'sample_width', 400)
sample_height = self.config.getint('Training', 'sample_height', 400)
self.train_loader, self.val_loader = self.get_dataloader(
sample_width, sample_height)
self.class_names = self.train_loader.dataset.classes
log_dir = os.path.join(self.curr_dir, 'runs')
raw_data_dir = os.path.join(self.curr_dir, 'runs_raw_results')
self.setup_tensorboard(log_dir, raw_data_dir=raw_data_dir)
# Log a few example spectrograms to tensorboard;
# one per class:
TensorBoardPlotter.write_img_grid(
self.writer,
self.root_train_test_data,
len(self.class_names), # Num of train examples
)
# All ResultTally instances are
# collected here (two per epoch, for
# for all training loop runs, and one
# for all val loop runs:
self.step_results = ResultCollection()
self.log.debug(
f"Just before train: \n{'none--on CPU' if self.fastest_device.type == 'cpu' else torch.cuda.memory_summary()}"
)
try:
final_epoch = self.train()
self.visualize_final_epoch_results(final_epoch)
finally:
self.close_tensorboard()
class Inferencer:
'''
classdocs
'''
#------------------------------------
# Constructor
#-------------------
def __init__(self,
model_paths,
samples_path,
batch_size=1,
labels_path=None,
gpu_ids=0):
'''
Given the path to a trained model,
and the path to the root of a set
of data, compute predictions.
If labels_path is None, the subdirectory
names between the samples_path root,
and the samples themselves are used as
the ground truth labels.
By default: run batches of size 1,
because we always have drop_last set
to True. For small test sets leaving
out any data at all isn't good. Caller
can still set batch_size higher to gain
speed if the testset is very large, so that
not inferencing on up to batch_size - 1
samples is OK
:param model_paths:
:type model_paths:
:param samples_path:
:type samples_path:
:param batch_size:
:type batch_size:
:param labels_path:
:type labels_path:
:param gpu_ids: Device number of GPU, in case
one is available
:type gpu_ids: {int | [int]}
'''
self.model_paths = model_paths
self.samples_path = samples_path
self.labels_path = labels_path
self.gpu_ids = gpu_ids if type(gpu_ids) == list else [gpu_ids]
if batch_size is not None:
self.batch_size = batch_size
else:
self.batch_size = 1
self.IMG_EXTENSIONS = FileUtils.IMG_EXTENSIONS
self.log = LoggingService()
self.curr_dir = os.path.dirname(__file__)
#------------------------------------
# prep_model_inference
#-------------------
def prep_model_inference(self, model_path):
'''
1. Parses model_path into its components, and
creates a dict: self.model_props, which
contains the network type, grayscale or not,
whether pretrained, etc.
2. Creates self.csv_writer to write results measures
into csv files. The destination file is determined
as follows:
<script_dir>/runs_raw_inferences/inf_csv_results_<datetime>/<model-props-derived-fname>.csv
3. Creates self.writer(), a tensorboard writer with destination dir:
<script_dir>/runs_inferences/inf_results_<datetime>
4. Creates an ImageFolder classed dataset to self.samples_path
5. Creates a shuffling DataLoader
6. Initializes self.num_classes and self.class_names
7. Creates self.model from the passed-in model_path name
:param model_path: path to model that will be used for
inference by this instance of Inferencer
:type model_path: str
'''
model_fname = os.path.basename(model_path)
# Extract model properties
# from the model filename:
self.model_props = FileUtils.parse_filename(model_fname)
csv_results_root = os.path.join(self.curr_dir, 'runs_raw_inferences')
#self.csv_dir = os.path.join(csv_results_root, f"inf_csv_results_{uuid.uuid4().hex}")
ts = FileUtils.file_timestamp()
self.csv_dir = os.path.join(csv_results_root, f"inf_csv_results_{ts}")
os.makedirs(self.csv_dir, exist_ok=True)
csv_file_nm = FileUtils.construct_filename(self.model_props,
prefix='inf',
suffix='.csv',
incl_date=True)
csv_path = os.path.join(self.csv_dir, csv_file_nm)
self.csv_writer = CSVWriterCloseable(csv_path)
ts = FileUtils.file_timestamp()
tensorboard_root = os.path.join(self.curr_dir, 'runs_inferences')
tensorboard_dest = os.path.join(tensorboard_root, f"inf_results_{ts}")
#f"inf_results_{ts}{uuid.uuid4().hex}")
os.makedirs(tensorboard_dest, exist_ok=True)
self.writer = SummaryWriterPlus(log_dir=tensorboard_dest)
dataset = SingleRootImageDataset(
self.samples_path, to_grayscale=self.model_props['to_grayscale'])
# Make reproducible:
Utils.set_seed(42)
#********Utils.set_seed(56)
self.loader = DataLoader(dataset,
batch_size=self.batch_size,
shuffle=True,
drop_last=True)
self.class_names = dataset.class_names()
self.num_classes = len(self.class_names)
# Get the right type of model,
# Don't bother getting it pretrained,
# of freezing it, b/c we will overwrite
# the weights:
self.model = NetUtils.get_net(
self.model_props['net_name'],
num_classes=self.num_classes,
pretrained=False,
freeze=0,
to_grayscale=self.model_props['to_grayscale'])
self.log.info(f"Tensorboard info written to {tensorboard_dest}")
self.log.info(f"Result measurement CSV file(s) written to {csv_path}")
#------------------------------------
# __call__
#-------------------
def __call__(self, gpu_id_model_path_pair):
gpu_id, self.model_path = gpu_id_model_path_pair
self.prep_model_inference(self.model_path)
self.log.info(
f"Begining inference with model {FileUtils.ellipsed_file_path(self.model_path)} on gpu_id {gpu_id}"
)
#****************
#return self.run_inference(gpu_to_use=gpu_id)
dicts_from_runs = []
for i in range(3):
self.curr_dict = {}
dicts_from_runs.append(self.curr_dict)
self.run_inference(gpu_to_use=gpu_id)
print(dicts_from_runs)
#****************
#------------------------------------
# go
#-------------------
def go(self):
# Pair models to GPUs; example for
# self.gpu_ids == [0,4], and three models:
# [(gpu0, model0) (gpu4, model1), (gpu0, model3)]
repeats = int(np.ceil(len(self.model_paths) / len(self.gpu_ids)))
gpu_model_pairings = list(zip(self.gpu_ids * repeats,
self.model_paths))
#************* No parallelism for debugging
self(gpu_model_pairings[0])
return
#************* END No parallelism for debugging
with Pool(len(self.gpu_ids)) as inf_pool:
# Run as many inferences in parallel as
# there are models to try. The first arg,
# (self): means to invoke the __call__() method
# on self.
result_it = inf_pool.imap(self,
gpu_model_pairings,
chunksize=len(self.gpu_ids))
results = [res.get() for res in result_it]
print(f"******Results: {results}")
#------------------------------------
# run_inferencer
#-------------------
def run_inference(self, gpu_to_use=0):
'''
Runs model over dataloader. Along
the way: creates ResultTally for each
batch, and maintains dict instance variable
self.raw_results for later conversion of
logits to class IDs under different threshold
assumptions.
self.raw_results:
{'all_outputs' : <arr>,
'all_labels' : <arr>
}
Returns a ResultCollection with the
ResultTally instances of each batch.
:param gpu_to_use: which GPU to deploy to (if it is available)
:type gpu_to_use: int
:return: collection of tallies, one for each batch,
or None if something went wrong.
:rtype: {None | ResultCollection}
'''
# Just in case the loop never runs:
batch_num = -1
overall_start_time = datetime.datetime.now()
try:
try:
if torch.cuda.is_available():
self.model.load_state_dict(torch.load(self.model_path))
FileUtils.to_device(self.model, 'gpu', gpu_to_use)
else:
self.model.load_state_dict(
torch.load(self.model_path,
map_location=torch.device('cpu')))
except RuntimeError as e:
emsg = repr(e)
if emsg.find("size mismatch for conv1") > -1:
emsg += " Maybe model was trained with to_grayscale=False, but local net created for grayscale?"
raise RuntimeError(emsg) from e
loss_fn = nn.CrossEntropyLoss()
result_coll = ResultCollection()
# Save all per-class logits for ability
# later to use different thresholds for
# conversion to class IDs:
all_outputs = []
all_labels = []
self.model.eval()
num_test_samples = len(self.loader.dataset)
self.log.info(
f"Begin inference ({num_test_samples} test samples)...")
samples_processed = 0
loop_start_time = overall_start_time
with torch.no_grad():
for batch_num, (batch, targets) in enumerate(self.loader):
if torch.cuda.is_available():
images = FileUtils.to_device(batch, 'gpu')
labels = FileUtils.to_device(targets, 'gpu')
else:
images = batch
labels = targets
outputs = self.model(images)
loss = loss_fn(outputs, labels)
images = FileUtils.to_device(images, 'cpu')
outputs = FileUtils.to_device(outputs, 'cpu')
labels = FileUtils.to_device(labels, 'cpu')
loss = FileUtils.to_device(loss, 'cpu')
#**********
max_logit = outputs[0].max().item()
max_idx = (outputs.squeeze() == max_logit).nonzero(
as_tuple=False).item()
smpl_id = torch.utils.data.dataloader.sample_id_seq[-1]
lbl = labels[0].item()
pred_cl = max_idx
self.curr_dict[smpl_id] = (smpl_id, lbl, pred_cl)
#**********
# Specify the batch_num in place
# of an epoch, which is not applicatble
# during testing:
tally = ResultTally(batch_num, LearningPhase.TESTING,
outputs, labels, loss,
self.num_classes, self.batch_size)
result_coll.add(tally, step=None)
all_outputs.append(outputs)
all_labels.append(labels)
samples_processed += len(labels)
del images
del outputs
del labels
del loss
torch.cuda.empty_cache()
time_now = datetime.datetime.now()
# Sign of life every 6 seconds:
if (time_now - loop_start_time).seconds >= 5:
self.log.info(
f"GPU{gpu_to_use} processed {samples_processed}/{num_test_samples} samples"
)
loop_start_time = time_now
finally:
#*********
print(f"Sample seq: {torch.utils.data.dataloader.sample_id_seq}")
torch.utils.data.dataloader.sample_id_seq = []
#*********
time_now = datetime.datetime.now()
test_time_duration = time_now - overall_start_time
# A human readable duration st down to minutes:
duration_str = FileUtils.time_delta_str(test_time_duration,
granularity=4)
self.log.info(
f"Done with inference: {samples_processed} test samples; {duration_str}"
)
# Total number of batches we ran:
num_batches = 1 + batch_num # b/c of zero-base
# If loader delivered nothing, the loop
# never ran; warn, and get out:
if num_batches == 0:
self.log.warn(
f"Dataloader delivered no data from {self.samples_path}")
self.close()
return None
# Var all_outputs is now:
# [tensor([pred_cl0, pred_cl1, pred_cl<num_classes - 1>], # For sample0
# tensor([pred_cl0, pred_cl1, pred_cl<num_classes - 1>], # For sample1
# ...
# ]
# Make into one tensor: (num_batches, batch_size, num_classes),
# unless an exception was raised at some point,
# throwing us into this finally clause:
if len(all_outputs) == 0:
self.log.info(
f"No outputs were produced; thus no results to report")
return None
self.all_outputs_tn = torch.stack(all_outputs)
# Be afraid...be very afraid:
assert(self.all_outputs_tn.shape == \
torch.Size([num_batches,
self.batch_size,
self.num_classes])
)
# Var all_labels is now num-batches tensors,
# each containing batch_size labels:
assert (len(all_labels) == num_batches)
# list of single-number tensors. Make
# into one tensor:
self.all_labels_tn = torch.stack(all_labels)
assert(self.all_labels_tn.shape == \
torch.Size([num_batches, self.batch_size])
)
# And equivalently:
assert(self.all_labels_tn.shape == \
(self.all_outputs_tn.shape[0],
self.all_outputs_tn.shape[1]
)
)
self.report_results(result_coll)
self.close()
return result_coll
#------------------------------------
# report_results
#-------------------
def report_results(self, tally_coll):
self._report_textual_results(tally_coll, self.csv_dir)
self._report_conf_matrix(tally_coll, show_in_tensorboard=True)
self._report_charted_results()
#------------------------------------
# _report_conf_matrix
#-------------------
def _report_conf_matrix(self,
tally_coll,
show=True,
show_in_tensorboard=None):
'''
Computes the confusion matrix CM from tally collection.
Creates an image from CM, and displays it via matplotlib,
if show arg is True. If show_in_tensorboard is a Tensorboard
SummaryWriter instance, the figure is posted to tensorboard,
no matter the value of the show arg.
Returns the Figure object.
:param tally_coll: all ResultTally instances to be included
in the confusion matrix
:type tally_coll: result_tallying.ResultCollection
:param show: whether or not to call show() on the
confusion matrix figure, or only return the Figure instance
:type show: bool
:param show_in_tensorboard: whether or not to post the image
to tensorboard
:type show_in_tensorboard: bool
:return: Figure instance containing confusion matrix heatmap
with color legend.
:rtype: matplotlib.pyplot.Figure
'''
all_preds = []
all_labels = []
for tally in tally_coll.tallies(phase=LearningPhase.TESTING):
all_preds.extend(tally.preds)
all_labels.extend(tally.labels)
conf_matrix = Charter.compute_confusion_matrix(all_labels,
all_preds,
self.class_names,
normalize=True)
# Normalization in compute_confusion_matrix() is
# to 0-1. Turn those values into percentages:
conf_matrix_perc = (100 * conf_matrix).astype(int)
# Decide whether or not to write
# confusion cell values into the cells.
# The decision depends on how many species
# are represented in the conf matrix; too many,
# and having numbers in all cells is too cluttered:
if len(self.class_names
) > CELL_LABELING.CONF_MATRIX_CELL_LABEL_LIMIT.value:
write_in_fields = CELL_LABELING.DIAGONAL
else:
write_in_fields = CELL_LABELING.ALWAYS
fig = Charter.fig_from_conf_matrix(
conf_matrix_perc,
supertitle='Confusion Matrix\n',
subtitle='Normalized to percentages',
write_in_fields=write_in_fields)
if show_in_tensorboard:
self.writer.add_figure('Inference Confusion Matrix',
fig,
global_step=0)
if show:
# Something above makes fig lose its
# canvas manager. Add that back in:
Utils.add_pyplot_manager_to_fig(fig)
fig.show()
return fig
#------------------------------------
# _report_charted_results
#-------------------
def _report_charted_results(self, thresholds=None):
'''
Computes and (pyplot-)shows a set of precision-recall
curves in one plot. If precision and/or recall are
undefined (b/c of division by zero) for all curves, then
returns False, else True. If no curves are defined,
logs a warning.
:param thresholds: list of cutoff thresholds
for turning logits into class ID predictions.
If None, the default at Charters.compute_multiclass_pr_curves()
is used.
:type thresholds: [float]
:return: True if curves were computed and show. Else False
:rtype: bool
'''
# Obtain a dict of CurveSpecification instances,
# one for each class, plus the mean Average Precision
# across all curves. The dict will be keyed
# by class ID:
(all_curves_info, mAP) = \
Charter.compute_multiclass_pr_curves(
self.all_labels_tn,
self.all_outputs_tn,
thresholds
)
# Separate out the curves without
# ill defined prec, rec, or f1:
well_defined_curves = list(filter(
lambda crv_obj: not(crv_obj['undef_prec'] or\
crv_obj['undef_rec'] or\
crv_obj['undef_f1']
),
all_curves_info.values()
)
)
if len(well_defined_curves) == 0:
self.log.warn(
f"For all thresholds, one or more of precision, recall or f1 are undefined. No p/r curves to show"
)
return False
# Too many curves are clutter. Only
# show the best and worst by optimal f1:
f1_sorted = sorted(well_defined_curves,
key=lambda obj: obj['best_op_pt']['f1'])
curves_to_show = {
crv_obj['class_id']: crv_obj
for crv_obj in (f1_sorted[0], f1_sorted[-1])
}
#********** Mixup with objs blurring together
(_num_classes, fig) = \
ClassificationPlotter.chart_pr_curves(curves_to_show)
fig.show()
return True
#------------------------------------
# _report_textual_results
#-------------------
def _report_textual_results(self, tally_coll, res_dir):
'''
Give a sequence of tallies with results
from a series of batches, create long
outputs, and inputs lists from all tallies
Computes information retrieval type values:
precision (macro/micro/weighted/by-class)
recall (macro/micro/weighted/by-class)
f1 (macro/micro/weighted/by-class)
acuracy
balanced_accuracy
Combines these results into a Pandas series,
and writes them to a csv file. That file is constructed
from the passed-in res_dir, appended with 'ir_results.csv'.
Finally, constructs Github flavored tables from the
above results, and posts them to the 'text' tab of
tensorboard.
Returns the results measures Series
:param tally_coll: collect of tallies from batches
:type tally_coll: ResultCollection
:param res_dir: directory where all .csv and other
result files are to be written
:type res_dir: str
:return results of information retrieval-like measures
:rtype: pandas.Series
'''
all_preds = []
all_labels = []
for tally in tally_coll.tallies(phase=LearningPhase.TESTING):
all_preds.extend(tally.preds)
all_labels.extend(tally.labels)
res = OrderedDict({})
res['prec_macro'] = precision_score(all_labels,
all_preds,
average='macro',
zero_division=0)
res['prec_micro'] = precision_score(all_labels,
all_preds,
average='micro',
zero_division=0)
res['prec_weighted'] = precision_score(all_labels,
all_preds,
average='weighted',
zero_division=0)
res['prec_by_class'] = precision_score(all_labels,
all_preds,
average=None,
zero_division=0)
res['recall_macro'] = recall_score(all_labels,
all_preds,
average='macro',
zero_division=0)
res['recall_micro'] = recall_score(all_labels,
all_preds,
average='micro',
zero_division=0)
res['recall_weighted'] = recall_score(all_labels,
all_preds,
average='weighted',
zero_division=0)
res['recall_by_class'] = recall_score(all_labels,
all_preds,
average=None,
zero_division=0)
res['f1_macro'] = f1_score(all_labels,
all_preds,
average='macro',
zero_division=0)
res['f1_micro'] = f1_score(all_labels,
all_preds,
average='micro',
zero_division=0)
res['f1_weighted'] = f1_score(all_labels,
all_preds,
average='weighted',
zero_division=0)
res['f1_by_class'] = f1_score(all_labels,
all_preds,
average=None,
zero_division=0)
res['accuracy'] = accuracy_score(all_labels, all_preds)
res['balanced_accuracy'] = balanced_accuracy_score(
all_labels, all_preds)
res_series = pd.Series(list(res.values()), index=list(res.keys()))
# Write information retrieval type results
# to a one-line .csv file, using pandas Series
# as convenient intermediary:
res_csv_path = os.path.join(res_dir, 'ir_results.csv')
res_series.to_csv(res_csv_path)
res_rnd = {}
for meas_nm, meas_val in res.items():
# Measure results are either floats (precision, recall, etc.),
# or np arrays (e.g. precision-per-class). For both
# cases, round each measure to one digit:
res_rnd[meas_nm] = round(meas_val,1) if type(meas_val) == float \
else meas_val.round(1)
ir_measures_skel = {
'col_header': ['precision', 'recall', 'f1'],
'row_labels': ['macro', 'micro', 'weighted'],
'rows': [[
res_rnd['prec_macro'], res_rnd['recall_macro'],
res_rnd['f1_macro']
],
[
res_rnd['prec_micro'], res_rnd['recall_micro'],
res_rnd['f1_micro']
],
[
res_rnd['prec_weighted'], res_rnd['recall_weighted'],
res_rnd['f1_weighted']
]]
}
ir_per_class_rows = [[
prec_class, recall_class, f1_class
] for prec_class, recall_class, f1_class in zip(
res_rnd['prec_by_class'], res_rnd['recall_by_class'],
res_rnd['f1_by_class'])]
ir_by_class_skel = {
'col_header': ['precision', 'recall', 'f1'],
'row_labels': self.class_names,
'rows': ir_per_class_rows
}
accuracy_skel = {
'col_header': ['accuracy', 'balanced_accuracy'],
'row_labels': ['Overall'],
'rows': [[res_rnd['accuracy'], res_rnd['balanced_accuracy']]]
}
ir_measures_tbl = GithubTableMaker.make_table(ir_measures_skel,
sep_lines=False)
ir_by_class_tbl = GithubTableMaker.make_table(ir_by_class_skel,
sep_lines=False)
accuracy_tbl = GithubTableMaker.make_table(accuracy_skel,
sep_lines=False)
# Write the markup tables to Tensorboard:
self.writer.add_text('Information retrieval measures',
ir_measures_tbl,
global_step=0)
self.writer.add_text('Per class measures',
ir_by_class_tbl,
global_step=0)
self.writer.add_text('Accuracy', accuracy_tbl, global_step=0)
return res_series
#------------------------------------
# close
#-------------------
def close(self):
try:
self.writer.close()
except Exception as e:
self.log.err(f"Could not close tensorboard writer: {repr(e)}")
def __init__(self,
starting_config_src,
hparms_spec,
training_script=None,
logfile=None,
quiet=False,
dryrun=False,
unittesting=False):
'''
Specifications expected like this
*Ordered* dict (i.e. sequence of
keys and values always the same for
keys()/values()/items() methods:
{<hparm1> : [val1_1, val1_2, ...],
<hparm2> : [val2_1, val2_2, ...]
}
:param starting_config_src: a configuration
whose neural net related parameters will
be modified below for each run.
:type starting_config_src: {str | NeuralNetConfig}
:param hparms_spec:
:type hparms_spec:
:param training_script: path to the training script
of which to run multiple copies. If None, will
look in config for Path:train_script.
:type training_script: {None | str}
:param logfile: where to log runtime information. If None,
log to console
:type logfile: {None | str}
:param quiet: whether or not to report progress
:type quiet: bool
:param unittesting: set to True if unittesting so that
__init__() will only do a minimum, and allows unittests
to call other methods individually
:type bool
'''
if logfile is not None:
self.log = LoggingService(logfile=logfile)
else:
self.log = LoggingService()
self.quiet = quiet
self.curr_dir = os.path.dirname(__file__)
self.hostname = socket.getfqdn()
# No GPUs identified so far:
self.WORLD_SIZE = 0
starting_config = NeuralNetConfig(starting_config_src)
if unittesting:
# Leave calling of the methods below
# to the unittests
return
self.training_script = training_script
if training_script is None:
# Try to find it in config:
try:
self.training_script = starting_config.getpath(
'Paths', 'train_script', relative_to=self.curr_dir)
except KeyError:
raise ValueError(
"Did not provide training script path on cmd line or in config"
)
self.gpu_landscape = self.obtain_world_map(starting_config)
# Get list of dicts of hparm-name/hparm_value pairs;
# one for each of the runs
the_run_dicts = self.get_runs_hparm_specs(hparms_spec)
# Turn the run dicts into configurations
# that that modify the starting config:
the_run_configs = self.gen_configurations(starting_config,
the_run_dicts)
if dryrun:
print("Dryrun:")
print(
f"Would run {len(the_run_dicts)} processes with these configs:"
)
for configs in the_run_dicts:
print(configs)
return
# Provide support for cnt-c terminating the training
# script processes nicely:
self.cnt_c_received = False
signal.signal(signal.SIGTERM, self.handle_cnt_c)
# Start one training script for each configuration:
self.run_configurations(the_run_configs)
def __init__(self,
config_info,
device=0,
percentage=None,
debugging=False):
'''
:param config_info: all path and training parameters
:type config_info: NeuralNetConfig
:param debugging: output lots of debug info
:type debugging: bool
:param device: number of GPU to use; default is dev 0
if any GPU is available
:type device: {None | int}
:param percentage: percentage of training data to
use
:type percentage: {int | float}
'''
self.log = LoggingService()
if debugging:
self.log.logging_level = DEBUG
if percentage is not None:
# Integrity check:
if type(percentage) not in [int, float]:
raise TypeError(
f"Percentage must be int or float, not {type(percentage)}")
if percentage < 1 or percentage > 100:
raise ValueError(
f"Percentage must be between 1 and 100, not {percentage}")
if device is None:
device = 0
torch.cuda.set_device(device)
else:
available_gpus = torch.cuda.device_count()
if available_gpus == 0:
self.log.info("No GPU available; running on CPU")
else:
if device > available_gpus - 1:
raise ValueError(
f"Asked to operate on device {device}, but only {available_gpus} are available"
)
torch.cuda.set_device(device)
self.curr_dir = os.path.dirname(os.path.abspath(__file__))
try:
self.config = self.initialize_config_struct(config_info)
except Exception as e:
msg = f"During config init: {repr(e)}"
self.log.err(msg)
raise RuntimeError(msg) from e
try:
self.root_train_test_data = self.config.getpath(
'Paths', 'root_train_test_data', relative_to=self.curr_dir)
except ValueError as e:
raise ValueError(
"Config file must contain an entry 'root_train_test_data' in section 'Paths'"
) from e
self.batch_size = self.config.getint('Training', 'batch_size')
self.kernel_size = self.config.getint('Training', 'kernel_size')
self.min_epochs = self.config.Training.getint('min_epochs')
self.max_epochs = self.config.Training.getint('max_epochs')
self.lr = self.config.Training.getfloat('lr')
self.net_name = self.config.Training.net_name
self.pretrained = self.config.Training.getboolean('pretrained', False)
self.num_folds = self.config.Training.getint('num_folds')
self.freeze = self.config.Training.getint('freeze', 0)
self.to_grayscale = self.config.Training.getboolean(
'to_grayscale', True)
self.set_seed(42)
self.log.info("Parameter summary:")
self.log.info(f"network {self.net_name}")
self.log.info(f"pretrained {self.pretrained}")
if self.pretrained:
self.log.info(f"freeze {self.freeze}")
self.log.info(f"min epochs {self.min_epochs}")
self.log.info(f"max epochs {self.max_epochs}")
self.log.info(f"batch_size {self.batch_size}")
self.fastest_device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
self.device = self.fastest_device
self.num_classes = self.find_num_classes(self.root_train_test_data)
self.initialize_model()
sample_width = self.config.getint('Training', 'sample_width', 400)
sample_height = self.config.getint('Training', 'sample_height', 400)
self.train_loader = self.get_dataloader(sample_width,
sample_height,
perc_data_to_use=percentage)
self.log.info(f"Expecting {len(self.train_loader)} batches per epoch")
num_train_samples = len(self.train_loader.dataset)
num_classes = len(self.train_loader.dataset.class_names())
self.log.info(
f"Training set contains {num_train_samples} samples across {num_classes} classes"
)
self.class_names = self.train_loader.dataset.class_names()
log_dir = os.path.join(self.curr_dir, 'runs')
raw_data_dir = os.path.join(self.curr_dir, 'runs_raw_results')
self.setup_tensorboard(log_dir, raw_data_dir=raw_data_dir)
# Log a few example spectrograms to tensorboard;
# one per class:
TensorBoardPlotter.write_img_grid(
self.writer,
self.root_train_test_data,
len(self.class_names), # Num of train examples
)
# All ResultTally instances are
# collected here: (num_folds * num-epochs)
# each for training and validation steps.
self.step_results = ResultCollection()
self.log.debug(
f"Just before train: \n{'none--on CPU' if self.fastest_device.type == 'cpu' else torch.cuda.memory_summary()}"
)
try:
final_step = self.train()
self.visualize_final_epoch_results(final_step)
finally:
self.close_tensorboard()
def __init__(self,
config,
num_classes,
history_len=8,
model_root=None,
log=None):
'''
Constructor:
:param config: configuration structure
:type config: NeuralNetConfig
:param num_classes: number of target classes
:type num_classes: int
:param history_len: number of model snapshots to
maintain
:type history_len: int
:param model_root: path to where models
will be deposited
:type model_root: str
:param log: logging service to use. If
None, create new one for display output
:type log: LoggingService
'''
self.curr_dir = os.path.dirname(os.path.abspath(__file__))
# Model root directory:
if model_root is None:
self.model_root = os.path.abspath(
os.path.join(self.curr_dir,
'../runs_models')
)
else:
self.model_root = model_root
if os.path.exists(self.model_root) and \
not os.path.isdir(self.model_root):
raise FileExistsError(f"{self.model_root} exists but is not a directory")
# Ensure that intermediate dirs exist:
try:
os.makedirs(self.model_root)
except FileExistsError:
pass
if log is None:
self.log = LoggingService()
else:
self.log = log
self.history_len = history_len
# Create a subdirectory of model_root
# where this archive keeps its models.
# The subdir is guaranteed to be unique
# among model_root's siblings, and it will
# be created:
self.run_subdir = self._construct_run_subdir(config,
num_classes,
self.model_root)
# Queue to track models, keeping the
# number of saved models to history_len:
self.model_fnames = deque(maxlen=self.history_len)
class TrainScriptLauncher:
#------------------------------------
# Constructor
#-------------------
# Use distributed torch default port:
COMM_PORT = '5678'
def __init__(self, unittesting=False):
self.hostname = socket.getfqdn()
if unittesting:
# Let unittests create an instance
# and call individual methods:
return
# Logging to console during launch:
self.log = LoggingService()
# Convenience: directory of this
# script, and project root directory
curr_dir = Path(__file__).parent
proj_root = curr_dir.joinpath('../..').resolve()
self.curr_dir = str(curr_dir)
self.proj_root = str(proj_root)
args_parser = BirdsTrainingArgumentsParser(
formatter_class=BirdsTrainingArgumentsParser.
BlankLinesHelpFormatter,
description="PyTorch distributed training launch "
"helper to spawn multiple distributed "
"birds_train_parallel.py processes")
all_args = args_parser.parse_args()
# Separate the args for this launch script
# from the args destined for the copies of
# the train script:
self.launch_args = all_args['launch_args']
self.script_args = all_args['script_args']
# Build the gpu_landscape dict:
self.gather_world_layout(self.launch_args)
self.GPUS_USED_THIS_MACHINE = self.gpu_landscape[
self.hostname]['num_gpus']
#------------------------------------
# gather_world_layout
#-------------------
def gather_world_layout(self, launch_args):
'''
# Compute a unique number for each GPU within
# the group of nodes (machines). Starting with
# the master node's first GPU as 0 (if master node
# has a GPU.
# The resulting GPU layout is assigned to
# variable gpu_landscape:
:param launch_args:
:type launch_args:
'''
try:
config_file = launch_args['config']
if not os.path.exists(config_file):
raise ConfigError(
f"Configuration file {config_file} that was provided as command line arg does not exist."
)
except KeyError:
raise RuntimeError(
"Error: launch args must include a config file. See config.cfg.Example in project root"
)
self.config = DottableConfigParser(config_file)
# Ensure that the launch_args contains
# the path to the training script. It
# will be there if provided on the cmd line.
# But it may instead be under Path:train_script
# in the configuration:
try:
self.launch_args['training_script']
except KeyError:
# The training script was not specified
# on the command line. Is it in the config
# file:
try:
self.launch_args['training_script'] = self.config.getpath(
'Paths', 'train_script', relative_to=self.curr_dict)
except KeyError:
raise ValueError(
"No training script specified on command line or in config file"
)
try:
self.world_map_path = self.config.getpath(
'Paths', 'world_map', relative_to=self.curr_dir)
except KeyError:
raise RuntimeError(
f"Could not find entry for 'world_map' in config file {config_file}"
)
self.world_map = self.read_world_map(self.world_map_path)
# Ensure that this machine has an
# entry in the world_map:
try:
# Get this machine's info (sub)dict:
_my_world_info = self.world_map[self.hostname]
except KeyError:
raise ConfigError(
f"World map file does not contain entry for this machine ({self.hostname})"
)
self.compute_landscape = {}
# Whether or not machine running this
# code is the master node:
self.am_master_node = False
# Build gpu_landscape, which maps
# machine names to the rank range
# that they occupy via the number of
# their GPUs
#
# {machine_name1 : [1],
# machine_name2 : [0],
# machine_name3 : [1,2,3],
self.gpu_landscape = self.build_compute_landscape(self.world_map)
if self.master_hostname is None:
raise ConfigError(
f'No master machine in {self.world_map_path}; one entry needs to be "master" : 1'
)
# Common pytorch port is either in the config file,
# or we use the pytorch default
self.MASTER_PORT = self.config.getint('Parallelism', 'master_port',
self.COMM_PORT)
# Handle special case: no GPUs anywere, and
# we are on node 0: in that case start a single
# copy of the training script. If it is written
# properly, it will detect the absence of a GPU,
# and use the CPU. This happens during debugging
# on a laptop:
if self.WORLD_SIZE == 0 and self.am_master_node:
self.WORLD_SIZE += 1
# If trying to launch on a node without GPUs,
# when GPUs are available elsewhere, refuse to
# start the script (is this needed?):
if not TESTING:
if self.my_gpus == 0 and self.WORLD_SIZE > 0:
raise RuntimeError(
"This machine does not have any GPU, but others do; training script not started."
)
#------------------------------------
# launch_scripts
#-------------------
def launch_scripts(self):
'''
Launch (possibly) multiple copies of
the training script. Use world_map.json
to know how many, and which GPUs this
machine is to use.
Each copy is told:
o MASTER_ADDR # Where to reach the coordinating process
o MASTER_PORT # Corresponding port
o RANK # The copy's sequence number, which is
# Unique across all participating machines
o LOCAL_RANK # Which of this machine's GPU to use (0-origin)
o WORLD_SIZE # How many GPUs are used on all machines together
o GPUS_USED_THIS_MACHINE # Number of GPUs *used* on this
# machine, according to the world_map.
'''
# Compute a unique number for each GPU within
# the group of nodes (machines). Starting with
# the master node's first GPU as 0 (if master node
# has a GPU.
# The resulting GPU layout is assigned to
# variable gpu_landscape:
#
# {<machine_name> :
# This machine's range of ranks:
rank_range = self.gpu_landscape[self.hostname]['rank_range']
this_machine_gpu_ids = self.gpu_landscape[
self.hostname]['gpu_device_ids']
min_rank_this_machine = self.gpu_landscape[self.hostname]['start_rank']
local_rank = 0
# Map from process object to rank (for debug msgs):
self.who_is_who = OrderedDict()
for rank in rank_range:
cmd = self.training_script_start_cmd(
rank, len(this_machine_gpu_ids), local_rank,
min_rank_this_machine, self.launch_args, self.script_args)
# Copy stdin, and give the copy to the subprocess.
# This enables the subprocess to ask user whether
# to save training state in case of a cnt-C:
newstdin = os.fdopen(os.dup(sys.stdin.fileno()))
# Spawn one training script.
process = subprocess.Popen(
cmd,
stdin=newstdin,
stdout=None, # Script inherits this launch
stderr=None # ... script's stdout/stderr
)
self.who_is_who[process] = rank
local_rank += 1
if not self.launch_args['quiet']:
print(
f"Node {self.hostname} {os.path.basename(sys.argv[0])}: Num processes launched: {len(self.who_is_who)}"
)
if self.am_master_node:
print(f"Awaiting {self.WORLD_SIZE} process(es) to finish...")
else:
print(f"Awaiting {self.my_gpus} process(es) to finish...")
failed_processes = []
try:
for process in self.who_is_who.keys():
process.wait()
if process.returncode != 0:
failed_processes.append(process)
continue
except KeyboardInterrupt:
# Gently kill the training scripts:
self.handle_cnt_c()
pass # See which processes get the interrupt
num_failed = len(failed_processes)
if num_failed > 0:
print(f"Number of failed training scripts: {num_failed}")
for failed_process in failed_processes:
train_script = self.launch_args['training_script']
script_rank = self.who_is_who[failed_process]
msg = (
f"Training script {train_script} (rank {script_rank}) encountered error(s); see logfile"
)
print(msg)
#------------------------------------
# training_script_start_cmd
#-------------------
def training_script_start_cmd(self, rank, gpus_used_this_machine,
local_rank, min_rank_this_machine,
launch_args, script_args):
'''
From provided information, creates a legal
command string for starting the training script.
:param rank: rank of the script; i.e. it's process' place
in the sequence of all train script processes
across all machines
:type rank: int
:param gpus_used_this_machine: number of GPU devices to
be used, according to the world_map; may be less than
number of available GPUs
:type gpus_used_this_machine: int
:param local_rank: index into the local sequence of GPUs
for for the GPU that the script is to use
:type local_rank: int
:param min_rank_this_machine: the lowest of the ranks among
the training scripts on this machine
:type min_rank_this_machine: int
:param launch_args: command line arguments intended for the
launch script, as opposed to being destined for the
train script
:type launch_args: {str : Any}
:param script_args: additional args for the train script
:type script_args: {str : Any}
'''
# Build the shell command line,
# starting with 'python -u':
cmd = [sys.executable, "-u"]
cmd.append(launch_args['training_script'])
# Add the args for the script that were
# in the command line:
for arg_name in script_args.keys():
script_arg_val = script_args[arg_name]
if script_arg_val is None or arg_name == 'config':
# Skip over non-specified CLI args:
continue
cmd.append(f"--{arg_name}={script_args[arg_name]}")
# Add the 'secret' args that tell the training
# script all the communication parameters:
cmd.extend([
f"--MASTER_ADDR={self.MASTER_ADDR}",
f"--MASTER_PORT={self.MASTER_PORT}", f"--RANK={rank}",
f"--LOCAL_RANK={local_rank}",
f"--MIN_RANK_THIS_MACHINE={min_rank_this_machine}",
f"--WORLD_SIZE={self.WORLD_SIZE}",
f"--GPUS_USED_THIS_MACHINE={gpus_used_this_machine}"
])
# Finally, the obligatory non-option arg
# to the training script: the configuration
# file:
config_file_name = script_args['config']
cmd.append(config_file_name)
self.log.debug(f"****** Launch: the cmd is {cmd}")
return cmd
#------------------------------------
# read_world_map
#-------------------
def read_world_map(self, path):
'''
Read the JSON5 world map file, and
return a corresponding dict. JSON5
allows something like:
/*
This is a block comment.
Notice the lacking quote
chars around the keys below.
The are optional in JSON5
*/
{quintus.stanford.edu : {
"master" : Yes
"gpus" : 2
},
quatro.stanford.edu : {
"gpus" : 2,
"devices" : [1,2]
}
}
BUT: JSON5 gets angry at dots in the
keys.
So we first read the file, and try to find
the machine names. We temporarily replace
them with an acceptable marker, and then
convert back.
:param path: path to world map file
:type path: string
'''
dot_substitute = '___'
try:
# Read all the world map file lines:
with open(path, 'r') as world_map_fd:
tmp_world_map = world_map_fd.readlines()
except IOError as e:
raise IOError(f"World map file at {path} not found") from e
# Replace occurrences of '.' with dot_substitute:
new_text = []
for line in tmp_world_map:
new_text.append(line.replace('.', dot_substitute))
# ... and make one string from all the lines:
json_str = '\n'.join(new_text)
try:
# Hopefully, JSON5 will eat it now:
world_map_almost = json5.loads(json_str)
except JSONError as e:
raise JSONError(
f"World map file at {path} contains bad JSON") from e
# Need to fix all the dot substitutions.
# At this point the data structure is
# { <machine_name> : {spec_attr1 : val1,
# spec_attr2 : val2,
# }
# }
# Fix the machine names first:
mach_names_fixed = [
machine_name.replace(dot_substitute, '.')
for machine_name in world_map_almost.keys()
]
machine_specs_fixed = []
# Now dig into each of the nested machine spec
# dicts, and fix attrs and values there:
for spec in world_map_almost.values():
# Spec is a dict nested inside the outer one:
spec_fixed = {
key.replace(dot_substitute, '.'): val.replace(
dot_substitute, '.') if isinstance(val, str) else val
for key, val in spec.items()
}
machine_specs_fixed.append(spec_fixed)
# Put it all together:
world_map = {
machine_name: spec_dict
for machine_name, spec_dict in zip(mach_names_fixed,
machine_specs_fixed)
}
return world_map
#------------------------------------
# build_compute_landscape
#-------------------
def build_compute_landscape(self, world_map):
'''
# Using the world_map.json config file, build
# a dict self.gpu_landscape like this:
#
# {'machine_name1' : {'start_rank' : <int>,
# 'num_gpus' : <int>,
# 'gpu_device_ids': [<int>,<int>,...]
# {'machine_name2' : {'start_rank' : <int>,
# 'num_gpus' : <int>,
# 'gpu_device_ids': [<int>,<int>,...]
# }
#
# Also sets
# o self.master_hostname, the hostname
# running the one process that coordinates all others.
# o self.WORLD_SIZE, number of GPUs used across all machines
# o self.my_gpus, the number of GPUs on this machine
:param world_map:
:type world_map:
:return: information about how many GPUs are
on each node
:rtype: OrderedDict
'''
if not self.hostname in world_map.keys():
raise ConfigError(
f"World map does not contain an entry for this machine {self.hostname}"
)
# World size is the number of training script processes,
# which is equal to number of GPUs used on all participating
# machines combined:
# Number of GPUs across all machines:
self.WORLD_SIZE = 0
self.master_hostname = None
# Go through the world map, machine (a.k.a. node)
# one at a time, in alpha order of the machine
# names to ensure all copies of this script
# come to the same conclusions about ranks
# Build gpu_landscape:
#
# {'machine_name1' : {'start_rank' : <int>,
# 'num_gpus' : <int>,
# 'gpu_device_ids': [<int>,<int>,...]
# {'machine_name2' : {'start_rank' : <int>,
# 'num_gpus' : <int>,
# 'gpu_device_ids': [<int>,<int>,...]
# }
#
# The structure is an OrderedDict(), containing
# machines alphabetically by name. This discipline
# is required so that all copies of this launch script
# (one copy per machine) arrive at the same ordering of
# GPUs:
gpu_landscape = OrderedDict({})
for machine_name in sorted(world_map.keys()):
# Get dict of info about the machine:
machine_info = world_map[machine_name]
try:
machine_gpus = machine_info['gpus']
except KeyError:
print(
"World map must include a 'gpus' entry; the value may be 0"
)
gpu_landscape[machine_name] = {}
gpu_landscape[machine_name]['num_gpus'] = machine_gpus
# List of GPU numbers to use is optional
# in world_maps:
machine_gpus_to_use = machine_info.get('devices', None)
if machine_gpus_to_use is None:
# Use all GPUs on that machine:
machine_gpus_to_use = list(range(machine_gpus))
gpu_landscape[machine_name]['gpu_device_ids'] = machine_gpus_to_use
# Accept all kinds of affirmatives as values:
# for identification of the master node entry:
is_master_node = machine_info.get('master', False) \
in [1, 'True', 'true', 'Yes', 'yes']
if is_master_node:
self.master_hostname = machine_name
if machine_name == self.hostname:
self.am_master_node = True
try:
self.MASTER_ADDR = socket.gethostbyname(machine_name)
except socket.gaierror:
# For machines that have no
# findable IP address:
self.MASTER_ADDR = '127.0.0.1'
self.WORLD_SIZE += machine_gpus
# Go through the machine enries in gpu_landscape, and
# assign rank ranges to each. Must start with
# the master node, b/c it must start with rank 0.
# For the master node, it is possible that it has
# no GPUs
master_info = gpu_landscape[self.master_hostname]
master_info['rank_range'] = list(range(master_info['num_gpus']))
master_info['start_rank'] = 0
if len(master_info['rank_range']) == 0:
# Master only has a GPU:
master_info['rank_range'] = [0]
# Start assigning more ranks after
# the GPUs of the master:
running_rank = master_info['rank_range'][-1] + 1
for machine_name in gpu_landscape.keys():
if machine_name == self.master_hostname:
# We already did the master node
continue
mach_info = gpu_landscape[machine_name]
mach_info['start_rank'] = running_rank
num_gpus = mach_info['num_gpus']
range_bound = running_rank + (num_gpus if num_gpus > 0 else 1)
mach_info['rank_range'] = list(range(running_rank, range_bound))
running_rank += (num_gpus if num_gpus > 0 else 1)
self.my_gpus = gpu_landscape[self.hostname]['num_gpus']
self.gpu_landscape = gpu_landscape
return gpu_landscape
#------------------------------------
# handle_cnt_c
#-------------------
def handle_cnt_c(self):
'''
Given a list of process instances,
Send SIGINT (cnt-C) to them:
:param procs:
:type procs:
'''
# Line processes up, highest rank first,
# master process last:
procs_terminate = sorted([proc for proc in self.who_is_who.keys()],
key=lambda obj: self.who_is_who[obj],
reverse=True)
for process in procs_terminate:
# If process is no longer running,
# forget about it:
if process.poll is not None:
# Process dead:
continue
process.send_signal(signal.SIGTERM)
process.wait()
class SpectrogramChopper:
'''
Processes directories of .png files,
chopping them into window_len seconds snippets.
Assumes:
self.input
Species1 Species2 ... Speciesn
spectro1_1.png spectro2_1.png spectro_n_1.png
spectro1_2.png spectro2_2.png spectro_n_2.png
...
Saves the snippets in a new directory.
Resulting directories under self.out_dir will be:
self.out_dir
Species1 Species2 ... Speciesn
snip_1_1_1 snip_2_1_1 snip_n_1_1
snip_1_1_2 snip_2_1_2 snip_n_1_2
snip_1_1_3 snip_2_1_3 snip_n_1_3
snip_2_1_4
...
snip_1_2_1 snip_2_2_1 snip_n_2_1
snip_1_2_2 snip_2_2_2 snip_n_2_2
snip_1_2_3 snip_n_2_3
snip_1_2_4
With snip_b_f_s:
o b is the bird species (manifesting in the file system
as one subdirectory under self.out_dir)
o f is one spectrogram of a full audio recording
o s is a snippet number.
Note the possibility of different numbers of snippets
in each species, and for each original audio recording
(which may be of unequal lengths).
Because many spectrograms are created, speed requirements
call for the use of parallelism. Since each audio file's processing
is independent from the others, the multiprocessing library
is used as follows.
- If command line arg --workers is set to 1, no parallelism
is used.
- If multiple cores are available, some percentage of
of them will be deployed to chopping. Each core runs
a separate copy of this file. The percentage is controlled
by MAX_PERC_OF_CORES_TO_USE.
Method chop_all() is used in the single core scenario.
Method chop_from_file_list() is used when multiprocessing. This
method is the 'target' in the multiprocessing library's sense.
'''
# Common logger for all workers:
log = LoggingService()
MIN_SNIPPET_WIDTH = 256
'''Minimum width of spectrogram snippets to satisfy the
torchvision pretrained model minimum value of
224 for both width and height'''
#------------------------------------
# Constructor
#-------------------
def __init__(self,
in_dir_or_spectro_file,
outdir,
specific_species=None,
overwrite_policy=WhenAlreadyDone.ASK):
'''
The overwrite_policy is one of the WhenAlreadyDone
enum members: ASK, OVERWRITE, SKIP. If ASK,
request user's permission for each encountered
destination file. SKIP should be used when resuming
an interrupted chopping session. Any sound file
whose destination spectrogram exists is not processed
again.
@param in_dir_or_spectro_file: location of spectrogram root
@type in_dir_or_spectro_file: str
@param outdir: root of spectrograms to create
@type outdir: src
@param specific_species: process only a spectific list of species
@type specific_species: {None | [str]}
@param overwrite_policy: what to do when an output file already exists
@type overwrite_policy: WhenAlreadyDone
'''
# Ensure the outdir and all its intermediate dirs exist:
os.makedirs(outdir, exist_ok=True)
self.in_dir = in_dir_or_spectro_file if os.path.isdir(
in_dir_or_spectro_file) else None
self.out_dir = outdir
self.specific_species = specific_species
self.overwrite_policy = overwrite_policy
self.num_chopped = 0
# The following used to be less convoluted until
# the option to chop just a single spectro, rather
# than a list of subdirectories with spectros
# in them. Sorry! :-(
if self.in_dir is not None:
# We are to process entire directory tree,
# not just a single file:
if self.specific_species is None:
# Process all (species) subdirectories:
self.species_list = os.listdir(self.in_dir)
else:
# Only process certain species:
self.species_list = self.specific_species
# Create destination directories for new spectrogram
# snippets, so that the dest tree will mirror the in tree:
self.spectrogram_dir_path = self.create_dest_dirs(
self.species_list)
else:
# Just do a single spectro, no need for destination
# subdirs:
self.spectrogram_dir_path = outdir
# Allow others outside the instance to find the spectros:
SpectrogramChopper.spectrogram_dir_path = self.spectrogram_dir_path
#------------------------------------
# chop_from_file_list
#-------------------
@classmethod
def chop_from_file_list(cls,
assignments,
in_dir,
out_dir,
global_info,
overwrite_policy,
return_bool,
env=None):
'''
Takes a list like:
[(s1,f1),(s1,f2),(s4,f3)]
where s_n is a species name, and f_m
is the basename of a spectrogram file to chop.
Example: foobar.png
Returns True if all went well, else
raises exception.
Wrinkle: this method is called under two
very different scenarios (S1/S2). S1 is
when the process started by the user calls
this method. That happens when the command
line arg --workers is set to 1, or on a machine
where few enough cores are available that only
one is used. In that case, env is left at None,
and all is as normal.
S2 occurs when the initial process (the one started
from the command line) starts a new Process. That
process normally contains a new environment, i.e.
some default value for all the environment variables.
************ NEEDED WITHOUT LIBROSA USAGE?
In particular, DISPLAY and PYTHONPATH will not be
what is needed. The result is that all spectrogram
creating methods fail, because they cannot find a
graphics backend.
In that case kwarg env is set to the environment of the
initiating process. At the start of this method this
process' default environ is then set to match that
of the initiating process.
:param assignments: list of species/filename pairs
:type assignments: [(str,str)]
:param env: if provided, the environment of the
parent process. If None, the current env
is retained
:type env: {str : Any}
:param return_bool:
:type return_bool:
'''
# During multiprocessing this method is
# the 'target', i.e. the entry point for
# each child. In that case env will be
# the environment of the initiating process.
# We adopt that environment for this new,
# forked process as well:
if env is not None:
os.environ = env
# Optimism!
return_bool.value = True
for species_name, fname in assignments:
# Ex. species_name: AMADEC
# Ex. fname : dysmen_my_bird.png
full_spectro_path = os.path.join(in_dir, species_name, fname)
try:
cls.chop_one_spectro_file(full_spectro_path,
os.path.join(out_dir, species_name),
species_name,
overwrite_policy=overwrite_policy)
except Exception as e:
return_bool.value = False
cls.log.err((f"One file could not be processed \n"
f" ({full_spectro_path}):\n"
f" {repr(e)}"))
continue
#------------------------------------
# chop_one_spectro_file
#-------------------
@classmethod
def chop_one_spectro_file(
cls,
spectro_fname,
out_dir,
species_name,
window_len=5,
skip_size=2,
original_duration=None,
overwrite_policy=WhenAlreadyDone.ASK,
):
"""
Generates window_len second spectrogram snippets
from spectrograms files of arbitrary length.
To compute the number of time slices to extract
for each snippet, the time_slices of the spectrogram time
slices in fractional seconds must be known. The time_slices
can be approximated if the play length of the underlying
audio is known (even if the precise fft settings are unavailable).
If the given .png file contains metadata with a 'duration'
key, then the corresponding value is used as the duration of
the original audio file in fractional seconds. This metadata
will be present if the .png file was created with the
SoundProcessor.create_spectrogram().
To enable use of spectrogram images created elsewhere, callers
can instead supply original_duration in fractional seconds.
For now, if neither the embedded metadata, nor the original_duration
is supplied, a ValueError is raised.
:param spectro_fname: full path to spectrogram file to chop
:type spectro_fname: str
:param out_dir: root directory under which spectrogram
snippets will be saved (in different subdirs)
:type out_dir: str
:param species_name: name of species to embed in the
metadata of this snippet, and use for determining
subdirectory where to place the snippet
:type species_name: str
:param window_len: number of seconds to be covered by each snippet
:type window_len: int
:param skip_size: number of seconds to shift right in
time for the start of each chop
:type skip_size: int
:param original_duration:
:raise ValueError: if neither embedded duration metadata is found
in the given file, nor original_duration is provided
"""
# Read the spectrogram, getting an np array:
spectro_arr, metadata = SoundProcessor.load_spectrogram(spectro_fname)
duration = metadata.get('duration', None)
if duration is None:
if original_duration is None:
raise ValueError(
f"Time duration of original recording cannot be determined for {spectro_fname}"
)
else:
duration = float(original_duration)
else:
duration = float(duration)
# If original file is already at or below
# the single window length, it's a snippet
# in itself. Copy it to the output with an
# appropriate snippet name to match the other
# snippets: wall start time is zero:
if duration < window_len:
# No partial snippets
return
# Note: Also have sample rate ('sr') and species ('label')
# in the metadata, but don't need those here.
_freq_bands, time_slices = spectro_arr.shape
# Time in fractions of second
# per spectrogram column:
twidth = duration / time_slices
# Integer of duration (which is in seconds):
time_dur_int = int(np.ceil(duration))
time_upper_bound = 1 + time_dur_int - skip_size
# Caller specifies skip_size and window
# length in *seconds*. Convert to spectrogram
# time slices (with rounding error):
samples_win_len = int(window_len // twidth)
# Does samples_win_len satisfy the
# minimum spectrogram snippet width for
# pretrained models?
samples_win_len = max(cls.MIN_SNIPPET_WIDTH, samples_win_len)
time_true_each_snippet = samples_win_len * twidth
samples_skip_size = int(skip_size // twidth)
samples_upper_bound = int(time_upper_bound // twidth)
assert (samples_upper_bound <= time_slices)
for _snip_num, samples_start_idx in enumerate(
range(0, samples_upper_bound, samples_skip_size)):
# Absolute start time of this snippet
# within the entire spectrogram:
wall_start_time = samples_start_idx * twidth
# Create a name for the snippet file:
snippet_path = cls.create_snippet_fpath(spectro_fname,
round(wall_start_time),
out_dir)
spectro_done = os.path.exists(snippet_path)
if spectro_done:
if overwrite_policy == WhenAlreadyDone.SKIP:
# Next snippet:
continue
elif overwrite_policy == WhenAlreadyDone.ASK:
if not Utils.user_confirm(
f"Snippet {Path(snippet_path).stem} exists, overwrite?",
default='N'):
continue
# Chop: All rows, columns from current
# window start for window lenth samples:
snippet_data = spectro_arr[:, samples_start_idx:samples_start_idx +
samples_win_len]
_num_rows, num_cols = snippet_data.shape
if num_cols < samples_win_len:
# Leave that little spectrogram
# snippet leftover for Elijah:
break
snippet_info = metadata.copy()
# Add the
snippet_info['duration(secs)'] = samples_win_len * twidth
snippet_info['start_time(secs)'] = wall_start_time
snippet_info['end_time(secs)'] = wall_start_time + (
samples_win_len * twidth)
snippet_info['species'] = species_name
SoundProcessor.save_image(snippet_data, snippet_path, snippet_info)
return time_true_each_snippet
#------------------------------------
# create_dest_dirs
#-------------------
def create_dest_dirs(self, species_list):
'''
Creates all directories that will hold new
spectrogram snippets for each species.
For each directory: if dir exists:
o if overwrite_policy is True, wipe the dir
o if overwrite_policy is SKIP, leave the
directory in place, contents intact
o else ask user.
If response is Yes, wipe the dir
else raise FileExistsError
:param species_list: names of species to process
:type species_list: [str]
:return: top level dir for spectrograms (same as self.out_dir)
:rtype: (str)
:raise FileExistsError: if a dest dir exists and not allowed
to wipe it.
'''
# Root dir of each species' spectro snippets:
Utils.create_folder(self.out_dir,
overwrite_policy=self.overwrite_policy)
# One dir each for the spectrogram snippets of one species:
for species in species_list:
species_spectros_dir = os.path.join(self.out_dir, species)
if not Utils.create_folder(species_spectros_dir,
overwrite_policy=self.overwrite_policy):
raise FileExistsError(
f"Target dir {species_spectros_dir} exists; aborting")
return self.out_dir
#------------------------------------
# create_snippet_fpath
#-------------------
@classmethod
def create_snippet_fpath(cls, origin_nm, wall_start_time, out_dir):
'''
Given constituent elements, construct the
full output path of a new spectrogram snippet.
Name format if full-length spectrogram file were
named my_file.png:
my_file_sw-start123.png
where 123 is the snippet's start time in seconds
from the beginning of the full length file
:param origin_nm: name of full length file; either full path or
just the file name are fine
:type origin_nm: str
:param wall_start_time: snippet start time from beginning
of full length spectrogram
:type wall_start_time: int
:param out_dir: destination directory
:type out_dir: str
:return: full path to the future snippet's destination
:rtype: str
'''
# Prepare snippet file name creation:
# From '/foo/bar/infile.png'
# make 'infile'
snippet_name_stem = Path(origin_nm).stem
snippet_name = f"{snippet_name_stem}_sw-start{str(wall_start_time)}.png"
snippet_path = os.path.join(out_dir, snippet_name)
return snippet_path
# -------------------- Class Methods ------------
#------------------------------------
# compute_worker_assignments
#-------------------
@classmethod
def compute_worker_assignments(cls,
in_dir,
dst_dir,
overwrite_policy=WhenAlreadyDone.ASK,
num_workers=None):
'''
Given the root directory of a set of
directories whose names are species,
and which contain spectrograms by species,
return a multi processing worker assignment.
Expected:
in_dir
Species1 Species2 ... Speciesn
smpl1_1.png smpl2_1.png smpln_1.png
smpl1_2.png smpl2_2.png smpln_2.png
...
Collects number of spectrograms available for
each species. Creates a list of species name
buckets such that all workers asked to process
one of the buckets, will have roughly equal
amounts of work.
Example return:
[['Species1', 'Species2], ['Species3', 'Species4', 'Species5']]
The caller can then assign the first list to
one worker, and the second list to another worker.
The number of buckets, and therefore the number
of eventual workers may be passed in. If None,
80% of the cores available on the current machine
will be assumed. If num_workers is provided and
the number is larger than the number of available
cores, the number is reduced to the number of cores.
Also returned is the number of workers on which the
computation is based. This number is always the same
as the number of species name lists in the return.
But for clarity, the number is returned explicitly.
:param in_dir: root of species recordings
:type in_dir: str
:param num_workers: number of buckets into which to partition
:type num_workers: {int | None}
:return: list of species name lists, and number of workers.
:rtype: ([[int]], int)
'''
# Create:
# {species : num-recordings}
# {species : recordings_dir}
# [(species1, fpath1), (species1, fpath2), (species2, fpath3)...]
sample_size_distrib = OrderedDict({})
sample_dir_dict = {}
species_file_tuples = []
for _dir_name, subdir_list, _file_list in os.walk(in_dir):
for species_name in subdir_list:
species_spectros_dir = os.path.join(in_dir, species_name)
spectro_paths = os.listdir(species_spectros_dir)
# Create new spectro_paths with only spectro files that
# need chopping:
new_rec_paths = cls.cull_spectro_paths(species_name, dst_dir,
spectro_paths,
overwrite_policy)
sample_size_distrib[species_name] = len(spectro_paths)
sample_dir_dict[species_name] = species_spectros_dir
species_file_pairs = list(
zip([species_name] * len(new_rec_paths), new_rec_paths))
species_file_tuples.extend(species_file_pairs)
break
if len(species_file_tuples) == 0:
# If no subdirectories with spectrograms were
# found, warn:
cls.log.warn(
(f"\n"
f" All spectrograms in {in_dir} already chopped.\n"
f" Or did you mean to create an individual file\n"
f" rather than a set of species subdirs?"))
num_cores = mp.cpu_count()
# Use 80% of the cores:
if num_workers is None:
num_workers = round(num_cores * Utils.MAX_PERC_OF_CORES_TO_USE /
100)
elif num_workers > num_cores:
# Limit pool size to number of cores:
num_workers = num_cores
# Create a partitioning into equal sized files,
# regardless of species association.
assignments = cls.partition_by_recordings(species_file_tuples,
num_workers)
num_workers_used = len(assignments)
return assignments, num_workers_used
#------------------------------------
# partition_by_recordings
#-------------------
@classmethod
def partition_by_recordings(cls, species_file_pairs, num_workers):
'''
Given a list of species-name/file-path tuples,
partition that list into num_workers sublists,
such that each list contains roughly the same
number of tuples. If the number of species_file_pairs
tuples is not divisible by num_workers, the left-over
tuples are distributed over the first sublists.
:param species_file_pairs:
:type species_file_pairs:
:param num_workers:
:type num_workers:
:return partitioning of the species_file_pairs tuples
:rtype: [[(str, str)]]
'''
# Compute near-equal number of files per worker:
num_spectros = len(species_file_pairs)
spectros_per_worker = int(np.ceil(num_spectros / num_workers))
# Create list of species-file pair lists:
# [[(s1,f1), (s1,f2)], [s1,f3,s2:f4], ...]
# Each inner list will be handled by one worker:
assignments = []
assign_idx = 0
for _worker_idx in range(num_workers):
assign_sublist = species_file_pairs[assign_idx:assign_idx +
spectros_per_worker]
assignments.append(assign_sublist)
assign_idx += spectros_per_worker
num_tasks = sum([len(ass) for ass in assignments])
# The following seems never to happen, but
# too tired to figure out why:
left_overs = num_spectros - num_tasks
if left_overs > 0:
# Can't have more than num_workers left overs,
# meaning can't have more leftovers than
# sublists. Distribute the leftovers:=
for idx, left_over in enumerate(species_file_pairs[-left_overs:]):
assignments[idx].append(left_over)
# Remove empty assignments:
assignments = [ass for ass in assignments if len(ass) > 0]
return assignments
#------------------------------------
# run_workers
#-------------------
@classmethod
def run_workers(cls,
args,
global_info,
overwrite_policy=WhenAlreadyDone.ASK):
'''
Called by main to run the SpectrogramChopper in
multiple processes at once. Partitions the
audio files to be processed; runs the chopping
while giving visual progress on terminal.
Prints success/failure of each worker. Then
returns. In order to avoid processes repeatedly
reporting the same, or only locally kept info,
the globally visible dict `global_info` is passed in.
This method will add these key/val pairs:
1 The total number of spectros to chop (key 'num_tasks')
2 The number of already created snippets (key 'num_snips')
3 A list with values False for each job, indicating
that the corresponding job is not yet done (key 'jobs_status')
Processes will update 2 and 3 as they report progress:
:param args: all arguments provided to argparse
:type args: {str : Any}
:param global_info: interprocess communication
dict for reporting progress
:type global_info: multiprocessing.manager.dict
'''
# Get a list of lists of species names
# to process. The list is computed such
# that each worker has roughly the same
# number of recordings to chop. We let
# the method determine the number of workers
# by using 80% of the available cores.
(worker_assignments,
num_workers) = SpectrogramChopper.compute_worker_assignments(
args.input, args.outdir, num_workers=args.workers)
print(f"Distributing workload across {num_workers} workers.")
# Initialize the dict with shared information:
# Fill the inter-process list with False.
# Will be used to logging jobs finishing
# many times to the console (i.e. not used
# for functions other than reporting progress:
for _i in range(num_workers):
# NOTE: reportedly one cannot just set the passed-in
# list to [False]*num_workers, b/c
# a regular python list won't be
# seen by other processes, even if
# embedded in a multiprocessing.manager.list
# instance:
global_info['jobs_status'].append(False)
# Number of full spectrograms to chop:
global_info['snips_to_do'] = len(
Utils.find_in_dir_tree(args.input, pattern="*.png"))
# For progress reports, get number of already
# existing .png files in out directory:
global_info['snips_done'] = len(
Utils.find_in_dir_tree(args.outdir, pattern="*.png"))
# Assign each list of species to one worker:
chopping_jobs = []
for ass_num, assignment in enumerate(worker_assignments):
chopper = SpectrogramChopper(args.input,
args.outdir,
overwrite_policy=overwrite_policy)
ret_value_slot = mp.Value("b", False)
job = ProcessWithoutWarnings(
target=chopper.chop_from_file_list,
args=(
assignment,
args.input,
args.outdir,
global_info, # ***NEW
overwrite_policy,
ret_value_slot),
name=f"ass# {ass_num}")
job.ret_val = ret_value_slot
chopping_jobs.append(job)
print(f"Starting chops for {job.name}")
job.start()
start_time = datetime.datetime.now()
# Keep checking on each job, until
# all are done as indicated by all jobs_done
# values being True, a.k.a valued 1:
while sum(global_info['jobs_status']) < num_workers:
for job_idx, job in enumerate(chopping_jobs):
# Timeout 1 sec
job.join(1)
if job.exitcode is not None:
if global_info['jobs_status'][job_idx]:
# One of the processes has already
# reported this job as done. Don't
# report it again:
continue
# Let other processes know that this job
# is done, and they don't need to report
# that fact: we'll do it here below:
global_info['jobs_status'][job_idx] = True
# This job finished, and that fact has not
# been logged yet to the console:
res = "OK" if job.ret_val else "Error"
# New line after the single-line progress msgs:
print("")
print(
f"Worker {job.name}/{num_workers} finished with: {res}"
)
global_info['snips_done'] = cls.sign_of_life(
job,
global_info['snips_done'],
args.outdir,
start_time,
force_rewrite=True)
# Check on next job:
continue
# This job not finished yet.
# Time for sign of life?
global_info['snips_done'] = cls.sign_of_life(
job,
global_info['snips_done'],
args.outdir,
start_time,
force_rewrite=True)
#------------------------------------
# cull_spectro_paths
#-------------------
@classmethod
def cull_spectro_paths(cls,
species_or_recorder_name,
dst_dir,
rec_paths,
overwrite_policy=WhenAlreadyDone.ASK):
#******* DISABLED ************
# method analogous to cull_rec_paths() in create_spectrograms()
# Currently below is just a copy from create_spectrograms().
# If we end up needing culling, update this body
return rec_paths
#******* DISABLED ************
# NEVER REACHED
new_rec_paths = []
for aud_fname in rec_paths:
fname_stem = Path(aud_fname).stem
dst_path = os.path.join(dst_dir, species_or_recorder_name,
f"{fname_stem}.png")
if not os.path.exists(dst_path):
# Destination spectrogram does not exist;
# keep this audio file in the to-do list:
new_rec_paths.append(aud_fname)
continue
if overwrite_policy == WhenAlreadyDone.OVERWRITE:
os.remove(dst_path)
new_rec_paths.append(aud_fname)
continue
if overwrite_policy == WhenAlreadyDone.SKIP:
# Don't even assign audio file to a worker,
# since its spectro already exists:
continue
if overwrite_policy == WhenAlreadyDone.ASK:
if Utils.user_confirm(
f"Spectrogram for {dst_path} exists; overwrite?"):
os.remove(dst_path)
new_rec_paths.append(aud_fname)
continue
return new_rec_paths
#------------------------------------
# sign_of_life
#-------------------
@classmethod
def sign_of_life(cls,
job,
num_already_present_imgs,
outdir,
start_time,
force_rewrite=False):
# Time for sign of life?
now_time = datetime.datetime.now()
time_duration = now_time - start_time
# Every 3 seconds, but at least 3:
if force_rewrite \
or (time_duration.seconds > 0 and time_duration.seconds % 3 == 0):
# A human readable duration st down to minutes:
duration_str = FileUtils.time_delta_str(time_duration,
granularity=4)
# Get current and new spectro imgs in outdir:
num_now_present_imgs = len(
Utils.find_in_dir_tree(outdir, pattern="*.png"))
num_newly_present_imgs = num_now_present_imgs - num_already_present_imgs
# Keep printing number of done snippets in the same
# terminal line:
print((f"{job.name}---Number of spectros: {num_now_present_imgs} "
f"({num_newly_present_imgs} new) after {duration_str}"),
end='\r')
return num_newly_present_imgs
else:
return num_already_present_imgs
def __init__(self,
in_dir_or_spectro_file,
outdir,
specific_species=None,
overwrite_policy=WhenAlreadyDone.ASK,
generate_wav_files=False
):
'''
The overwrite_policy is one of the WhenAlreadyDone
enum members: ASK, OVERWRITE, SKIP. If ASK,
request user's permission for each encountered
destination file. SKIP should be used when resuming
an interrupted chopping session. Any sound file
whose destination spectrogram exists is not processed
again.
If generate_wav_files is True, a .wav file is created
for every window of the source soundfile. Usually
not necessary.
The window_size is the number of seconds by which a
sliding window is moved across the source soundfile
before a spectrogram is created.
@param in_dir_or_spectro_file: location of soundfile root
@type in_dir_or_spectro_file: str
@param outdir: root of spectrograms/wav_files to create
@type outdir: src
@param specific_species: process only a spectific list of species
@type specific_species: {None | [str]}
@param overwrite_policy: what to do when an output file already exists
@type overwrite_policy: WhenAlreadyDone
'''
self.in_dir = in_dir_or_spectro_file
self.out_dir = outdir
self.specific_species = specific_species
self.overwrite_policy = overwrite_policy
self.generate_wav_files = generate_wav_files
self.log = LoggingService()
self.num_chopped = 0
# Don't show the annoying deprecation
# librosa.display() warnings about renaming
# 'basey' to 'base' to match matplotlib:
warnings.simplefilter("ignore", category=MatplotlibDeprecationWarning)
# Hide the UserWarning: PySoundFile failed. Trying audioread instead.
warnings.filterwarnings(action="ignore",
message="PySoundFile failed. Trying audioread instead.",
category=UserWarning,
module='',
lineno=0)
if self.specific_species is None:
self.species_list = os.listdir(self.in_dir)
else:
self.species_list = self.specific_species
# Create directories for new audio snippets
# and spectrograms:
self.wav_dir_path, self.spectrogram_dir_path = self.create_dest_dirs(self.species_list)
# Allow others outside the instance
# find the audio snippet destination
SpectrogramChopper.wav_dir_path = self.wav_dir_path
SpectrogramChopper.spectrogram_dir_path = self.spectrogram_dir_path
class SpectrogramChopper:
'''
Processes directories of .wav or .mp3 files,
chopping them into window_len seconds snippets.
Each audio snippet is saved, and spectrograms
are created for each.
Assumes:
self.in_dir
Species1 Species2 ... Speciesn
smpl1_1.mp3 smpl2_1.mp3 smpln_1.mp3
smpl1_2.mp3 smpl2_2.mp3 smpln_2mp3
...
Saves the snippets in a new directory. Creates a spectrogram
for each snippet, and saves those in a different, new directory.
Resulting directories under self.out_dir will be:
self.out_dir
spectrograms wav-files
Because many spectrograms are created, speed requirements
call for the use of parallelism. Since each audio file's processing
is independent from the others, the multiprocessing library
is used as follows.
- If command line arg --workers is set to 1, no parallelism
is used.
- If multiple cores are available, some percentage of
of them will be deployed to chopping. Each core runs
a separate copy of this file. The percentage is controlled
by MAX_PERC_OF_CORES_TO_USE.
Method chop_all() is used in the single core scenario.
Method chop_from_file_list() is used when multiprocessing. This
method is the 'target' in the multiprocessing library's sense.
'''
# If multiple cores are available,
# only use some percentage of them to
# be nice:
MAX_PERC_OF_CORES_TO_USE = 50
#------------------------------------
# Constructor
#-------------------
def __init__(self,
in_dir_or_spectro_file,
outdir,
specific_species=None,
overwrite_policy=WhenAlreadyDone.ASK,
generate_wav_files=False
):
'''
The overwrite_policy is one of the WhenAlreadyDone
enum members: ASK, OVERWRITE, SKIP. If ASK,
request user's permission for each encountered
destination file. SKIP should be used when resuming
an interrupted chopping session. Any sound file
whose destination spectrogram exists is not processed
again.
If generate_wav_files is True, a .wav file is created
for every window of the source soundfile. Usually
not necessary.
The window_size is the number of seconds by which a
sliding window is moved across the source soundfile
before a spectrogram is created.
@param in_dir_or_spectro_file: location of soundfile root
@type in_dir_or_spectro_file: str
@param outdir: root of spectrograms/wav_files to create
@type outdir: src
@param specific_species: process only a spectific list of species
@type specific_species: {None | [str]}
@param overwrite_policy: what to do when an output file already exists
@type overwrite_policy: WhenAlreadyDone
'''
self.in_dir = in_dir_or_spectro_file
self.out_dir = outdir
self.specific_species = specific_species
self.overwrite_policy = overwrite_policy
self.generate_wav_files = generate_wav_files
self.log = LoggingService()
self.num_chopped = 0
# Don't show the annoying deprecation
# librosa.display() warnings about renaming
# 'basey' to 'base' to match matplotlib:
warnings.simplefilter("ignore", category=MatplotlibDeprecationWarning)
# Hide the UserWarning: PySoundFile failed. Trying audioread instead.
warnings.filterwarnings(action="ignore",
message="PySoundFile failed. Trying audioread instead.",
category=UserWarning,
module='',
lineno=0)
if self.specific_species is None:
self.species_list = os.listdir(self.in_dir)
else:
self.species_list = self.specific_species
# Create directories for new audio snippets
# and spectrograms:
self.wav_dir_path, self.spectrogram_dir_path = self.create_dest_dirs(self.species_list)
# Allow others outside the instance
# find the audio snippet destination
SpectrogramChopper.wav_dir_path = self.wav_dir_path
SpectrogramChopper.spectrogram_dir_path = self.spectrogram_dir_path
#------------------------------------
# chop_all
#-------------------
def chop_all(self):
'''
Workhorse: Assuming self.in_dir is root of all
species audio samples:
self.in_dir
Species1 Species2 ... Speciesn
smpl1_1.mp3 smpl2_1.mp3 smpln_1.mp3
smpl1_2.mp3 smpl2_2.mp3 smpln_2mp3
...
Chops each .mp3 (or .wav) file into window_len snippets.
Saves those snippets in a new directory. Creates a spectrogram
for each snippet, and saves those in a different, new directory.
Resulting directories under self.out_dir will be:
self.out_dir
spectrograms wav-files
If self.specific_species is None, audio files under all
species are chopped. Else, self.specific_species is
expected to be a list of species names that correspond
to the names of species directories above: Species1, Species2, etc.
Returns a 2-tuple: (number of created .wav audio snippet files,
number of created .png spectrogram snippet files,
'''
for species in self.species_list:
audio_files = os.listdir(os.path.join(self.in_dir, species))
num_files = len(audio_files)
for i, sample_name in enumerate(audio_files):
# Chop one audio file:
self.log.info(f"Chopping {species} audio {i}/{num_files}")
self.chop_one_audio_file(self.in_dir, species, sample_name, self.out_dir)
self.num_chopped += num_files
num_spectros = utils.find_in_dir_tree(self.spectrogram_dir_path, pattern='*.png')
num_audios = utils.find_in_dir_tree(self.wav_dir_path, pattern='*.wav')
return (num_audios, num_spectros)
#------------------------------------
# chop_from_file_list
#-------------------
def chop_from_file_list(self, assignments, return_bool, env=None):
'''
Takes a list like:
[(s1,f1),(s1,f2),(s4,f3)]
where s_n is a species name, and f_m
is the basename of an audio file to chop.
Example: foobar.mp3
Returns True if all went well, else
raises exception.
Wrinkle: this method is called under two
very different scenarios (S1/S2). S1 is
when the process started by the user calls
this method. That happens when the command
line arg --workers is set to 1, or on a machine
where few enough cores are available that only
one is used. In that case, env is left at None,
and all is as normal.
S2 occurs when the initial process (the one started
from the command line) starts a new Process. That
process normally contains a new environment, i.e.
some default value for all the environment variables.
In particular, DISPLAY and PYTHONPATH will not be
what is needed. The result is that all spectrogram
creating methods fail, because they cannot find a
graphics backend.
In that case kwarg env is set to the environment of the
initiating process. At the start of this method this
process' default environ is then set to match that
of the initiating process.
:param assignments: list of species/filename pairs
:type assignments: [(str,str)]
:param env: if provided, the environment of the
parent process. If None, the current env
is retained
:type env: {str : Any}
:param return_bool:
:type return_bool:
'''
# During multiprocessing this method is
# the target, i.e. the entry point for
# each child. In that case env will be
# the environment of the initiating process.
# We adopt that environment for this new,
# forked process as well:
if env is not None:
os.environ = env
for species_name, fname in assignments:
try:
self.chop_one_audio_file(self.in_dir,
species_name,
fname,
self.out_dir
)
except Exception as e:
return_bool.value = False
raise e
return_bool.value = True
#------------------------------------
# chop_one_audio_file
#-------------------
def chop_one_audio_file(self, in_dir, species, spectro_fname, out_dir, window_len = 5):
"""
Generates window_len second sound file snippets
and associated spectrograms from sound files of
arbitrary length.
Performs a time shift on all the wav files in the
species directories. The shift is 'rolling' such that
no information is lost.
:param in_dir: directory of the audio file to chop
:type file_name: str
:param species: the directory names of the species to
modify the wav files of. If species=None, all
subdirectories will be processed.
:type species: {None | [str]}
:param spectro_fname: basefile name of audio file to chop
:type spectro_fname: str
:param out_dir: root directory under which spectrogram
and audio snippets will be saved (in different subdirs)
:type out_dir: str
"""
orig, sample_rate = librosa.load(os.path.join(in_dir, species, spectro_fname))
length = int(librosa.get_duration(orig, sample_rate))
for start_time in range(length - window_len):
fpath = Path(spectro_fname)
window_name = f"{fpath.stem}_sw-start{str(start_time)}"
window_file_name = str(Path.joinpath(fpath.parent, window_name))
outfile_spectro = os.path.join(out_dir,
'spectrograms/',
species,
f"{window_file_name}.png")
outfile_audio = os.path.join(out_dir,
'wav-files',
species,
f"{window_file_name}.{'wav'}")
spectro_done = os.path.exists(outfile_spectro)
audio_done = os.path.exists(outfile_audio)
if spectro_done and audio_done and WhenAlreadyDone.SKIP:
# No brainer no need to even read the audio excerpt:
continue
if spectro_done and not audio_done and not self.generate_wav_files:
continue
# Need an audio snippet either for
# a spectrogram or wav file:
window_audio, sr = librosa.load(os.path.join(in_dir, species, spectro_fname),
offset=start_time, duration=window_len)
if not spectro_done or (spectro_done and self.overwrite_policy != WhenAlreadyDone.SKIP):
SoundProcessor.create_spectrogram(window_audio,sr,outfile_spectro)
if self.generate_wav_files:
if audio_done and self.overwrite_policy == WhenAlreadyDone.SKIP:
continue
else:
sf.write(outfile_audio, window_audio, sr)
#------------------------------------
# create_dest_dirs
#-------------------
def create_dest_dirs(self, species_list):
'''
Creates all directories that will hold new
audio snippets and spectrograms for each species.
For each directory: if dir exists:
o if overwrite_policy is True, wipe the dir
o else ask user.
If response is Yes, wipe the dir
else raise FileExistsError
:param species_list: names of species to process
:type species_list: [str]
:return: top level dirs for audio snippets and spectrograms
:rtype: (str)
:raise FileExistsError: if a dest dir exists and not allowed
to wipe it.
'''
# Root dir of the two dirs that will hold new
# audio snippet and spectrogram files
utils.create_folder(self.out_dir, overwrite_policy=self.overwrite_policy)
# Below the rootP
spectrogram_dir_path = os.path.join(self.out_dir,'spectrograms/')
wav_dir_path = os.path.join(self.out_dir,'wav-files/')
if not utils.create_folder(spectrogram_dir_path, overwrite_policy=self.overwrite_policy):
raise FileExistsError(f"Target dir {spectrogram_dir_path} exists; aborting")
if not utils.create_folder(wav_dir_path, overwrite_policy=self.overwrite_policy):
raise FileExistsError(f"Target dir {spectrogram_dir_path} exists; aborting")
# One dir each for the audio and spectrogram
# snippets of one species:
for species in species_list:
species_spectros_dir = os.path.join(spectrogram_dir_path, species)
if not utils.create_folder(species_spectros_dir,
overwrite_policy=self.overwrite_policy):
raise FileExistsError(f"Target dir {species_spectros_dir} exists; aborting")
species_audio_dir = os.path.join(wav_dir_path, species)
if not utils.create_folder(species_audio_dir,
overwrite_policy=self.overwrite_policy):
raise FileExistsError(f"Target dir {species_audio_dir} exists; aborting")
return(wav_dir_path, spectrogram_dir_path)
# -------------------- Class Methods ------------
#------------------------------------
# compute_worker_assignments
#-------------------
@classmethod
def compute_worker_assignments(cls, in_dir, num_workers=None):
'''
Given the root directory of a set of
directories whose names are species,
and which contain recordings by species,
return a multi processing worker assignment.
Expected:
in_dir
Species1 Species2 ... Speciesn
smpl1_1.mp3 smpl2_1.mp3 smpln_1.mp3
smpl1_2.mp3 smpl2_2.mp3 smpln_2mp3
...
Collects number of recordings available for
each species. Creates a list of species name
buckets such that all workers asked to process
one of the buckets, will have roughly equal
amounts of work.
Example return:
[['Species1', 'Species2], ['Species3', 'Species4', 'Species5']]
The caller can then assign the first list to
one worker, and the second list to another worker.
The number of buckets, and therefore the number
of eventual workers may be passed in. If None,
80% of the cores available on the current machine
will be assumed. If num_workers is provided and
the number is larger than the number of available
cores, the number is reduced to the number of cores.
Also returned is the number of workers on which the
computation is based. This number is always the same
as the number of species name lists in the return.
But for clarity, the number is returned explicitly.
:param in_dir: root of species recordings
:type in_dir: str
:param num_workers: number of buckets into which to partition
:type num_workers: {int | None}
:return: list of species name lists, and number of workers.
:rtype: ([[int]], int)
'''
# Create:
# {species : num-recordings}
# {species : recordings_dir}
# [(species1, fpath1), (species1, fpath2), (species2, fpath3)...]
sample_size_distrib = OrderedDict({})
sample_dir_dict = {}
species_file_tuples = []
for _dir_name, subdir_list, _file_list in os.walk(in_dir):
for species_name in subdir_list:
species_recordings_dir = os.path.join(in_dir, species_name)
rec_paths = os.listdir(species_recordings_dir)
sample_size_distrib[species_name] = len(rec_paths)
sample_dir_dict[species_name] = species_recordings_dir
species_file_pairs = list(zip([species_name]*len(rec_paths), rec_paths))
species_file_tuples.extend(species_file_pairs)
break
num_cores = mp.cpu_count()
# Use 80% of the cores:
if num_workers is None:
num_workers = round(num_cores * SpectrogramChopper.MAX_PERC_OF_CORES_TO_USE / 100)
elif num_workers > num_cores:
# Limit pool size to number of cores:
num_workers = num_cores
# Create a partitioning into equal sized files,
# regardless of species association.
assignments = cls.partition_by_recordings(species_file_tuples,
num_workers)
num_workers_used = len(assignments)
return assignments, num_workers_used
#------------------------------------
# partition_by_recordings
#-------------------
@classmethod
def partition_by_recordings(cls, species_file_pairs, num_workers):
'''
Given a list of species-name/file-path tuples,
partition that list into num_workers sublists,
such that each list contains roughly the same
number of tuples. If the number of species_file_pairs
tuples is not divisible by num_workers, the left-over
tuples are distributed over the first sublists.
:param species_file_pairs:
:type species_file_pairs:
:param num_workers:
:type num_workers:
:return partitioning of the species_file_pairs tuples
:rtype: [[(str, str)]]
'''
# Compute near-equal number of files per worker:
num_recordings = len(species_file_pairs)
recs_per_worker = int(np.ceil(num_recordings / num_workers))
# Create list of species-file pair lists:
# [[(s1,f1), (s1,f2)], [s1,f3,s2:f4], ...]
# Each inner list will be handled by one worker:
assignments = []
assign_idx = 0
for _worker_idx in range(num_workers):
assign_sublist = species_file_pairs[assign_idx:assign_idx+recs_per_worker]
assignments.append(assign_sublist)
assign_idx += recs_per_worker
left_overs = num_recordings % num_workers
if left_overs > 0:
# Can't have more than num_workers left overs,
# meaning can't have more leftovers than
# sublists. Distribute the leftovers:=
for idx, left_over in enumerate(species_file_pairs[-left_overs:]):
assignments[idx].append(left_over)
# Remove empty assignments:
assignments = [ass for ass in assignments if len(ass) > 0]
return assignments
#------------------------------------
# run_workers
#-------------------
@classmethod
def run_workers(cls, args, overwrite_policy=WhenAlreadyDone.ASK):
'''
Called by main to run the SpectrogramChopper in
multiple processes at once. Pajcrtitions the
audio files to be processed; runs the chopping
while giving visual progress on terminal.
Prints success/failure of each worker. Then
returns
:param args: all arguments provided to argparse
:type args: {str : Any}
'''
in_dir = args.input
# Get a list of lists of species names
# to process. The list is computed such
# that each worker has roughly the same
# number of recordings to chop. We let
# the method determine the number of workers
# by using 80% of the available cores.
(worker_assignments, num_workers) = SpectrogramChopper.compute_worker_assignments(
in_dir,
num_workers=args.workers)
print(f"Distributing workload across {num_workers} workers.")
# Assign each list of species to one worker:
chopping_jobs = []
for ass_num, assignment in enumerate(worker_assignments):
chopper = SpectrogramChopper(in_dir,
args.output_dir,
overwrite_policy=overwrite_policy
)
ret_value_slot = mp.Value("b", False)
job = ProcessWithoutWarnings(target=chopper.chop_from_file_list,
args=([assignment, ret_value_slot]),
name=f"ass# {ass_num}"
)
job.ret_val = ret_value_slot
chopping_jobs.append(job)
print(f"Starting chops for {job.name}")
job.start()
for job in chopping_jobs:
job_done = False
while not job_done:
# Check for job done with one sec timeout:
job.join(1)
# Get number of generated snippets:
num_chopped_snippets = \
len(utils.find_in_dir_tree(SpectrogramChopper.spectrogram_dir_path))
# Keep printing number of done snippets in the same
# terminal line:
print(f"Number of audio snippets: {num_chopped_snippets}", end='\r')
# If the call to join() timed out
if job.exitcode is None:
# Job not done:
continue
res = "OK" if job.ret_val else "Error"
# New line after the progress msgs:
print("")
print(f"Chops of {job.name}/{num_workers}: {res}")
job_done = True
class Charter:
# Value to assign to precision,
# recall, or f1 when divide by 0
DIV_BY_ZERO = 0
log = LoggingService()
#------------------------------------
# Constructor
#-------------------
def __init__(self, actions=None):
if actions is None:
return
for action in actions:
try:
if type(action) == VizConfMatrixReq:
cm = Charter.read_conf_matrix_from_file(action.path)
fig = self.fig_from_conf_matrix(
cm,
supertitle=action.supertitle,
title=action.title,
write_in_fields=action.write_in_fields)
fig.show()
elif action == 'pr_curves':
pass #data = 0 #********************
except Exception as _e:
pass
#------------------------------------
# visualize_testing_result
#-------------------
@classmethod
def visualize_testing_result(cls, truth_labels, pred_class_ids):
'''
Use to visualize results from using a
saved model on a set of test-set samples.
Draws a PR curve, and adds a table with
the average precison (AP) of each class.
'''
# Find number of classes involved:
all_class_ids = set(truth_labels)
num_classes = len(all_class_ids)
# Will alternately treat each class
# prediction as a one-vs-all binary
# classification. For each class ID (cid<n>),
# get 0/1 guess separately for each sample:
#
# cid0 cid1
# pred_sample0 1 0
# pred_sample1 0 0
# pred_sample2 0 1
# ...
# Same with labels:
# cid0 cid1
# labl_sample0 1 0
# labl_sample1 0 0
# labl_sample2 0 1
# ...
bin_labels = label_binarize(truth_labels,
classes=list(range(num_classes)))
# Make tensors just for manipulation
# convenience:
bin_labels_tn = torch.tensor(bin_labels)
preds_tn = torch.tensor(pred_class_ids)
precisions = dict()
recalls = dict()
average_precisions = dict()
# Go through each column, i.e. the
# 1/0 labels/preds for one class at
# a time, and get the prec/rec numbers.
# The [1] in prec & rec is b/c precision_recall_curve
# returns a triplet for binary classification:
# prec/rec at thresholds 0, 1, putting 1 as the
# last element. The prec/rec we want is the
# where 1 is the thresholds:
for i in range(num_classes):
bin_labels_arr = bin_labels_tn[:, i].tolist()
preds_arr = preds_tn.tolist()
# Get precision and recall at each
# of the default thresholds:
precs, recs = \
cls.compute_binary_pr_curve(bin_labels_arr,
preds_arr
)
precisions[i] = precs
recalls[i] = recs
# Avg prec is:
#
# AP = SUM_ovr_n((R_n - R_n-1)*P_n
#
# I.e. the increase in recalls times current
# precisions as each pred/sample pair is
# processed:
average_precisions[i] = \
average_precision_score(bin_labels_arr,
preds_arr,
average='macro',
)
mAP = np.mean(list(average_precisions.values()))
return (mAP, precisions, recalls, average_precisions)
# ----------------- Computations ---------------
#------------------------------------
# compute_binary_pr_curve
#-------------------
@classmethod
def compute_binary_pr_curve(
cls,
labels,
preds,
class_id,
thresholds=None,
):
'''
Return the recall (x-axis) and precision (y-axis)
values of a PR curve, its average precision (AP),
and optimal threshold with corresponding f1, precision,
and recall values
The optimal threshold's prec and rec yield the
maximum f1 score. Information provided in the
BestOperatingPoint instance that is part of this
method's return:
threshold
f1
prec
rec
The result is packaged as a CurveSpecification
that contains:
best_op_pt
precisions
recalls
thresholds
avg_prec'
Procedure:
A prec/rec point is computed for each
threshold point.
Works for binary classification.
But can use sklearn's label_binaries to compute
separate curves for each class
(see compute_multiclass_pr_curves())
Differs from sklearn.precision_recall_curve() in
that the sklearn method does not take a list
of thresholds.
Example:
(preds are probabilities, but they
are from one class, different samples.
So dont' add to 1):
labels = [1,1,0,1]
preds = [0.2, 0.4, 0.1, 0.2]
thresholds = [0.3, 0.7]
The predictions are turned into decisions like this:
preds_decided_0.3 = [0, 1, 0, 0]
preds_decided_0.5 = [0, 0, 0, 0]
Two prec and rec computations are executed:
pr0: prec and rec from [1, 1, 0, 1]
[0, 1, 0, 0]
pr1: prec and rec from [1, 1, 0, 1]
[0, 0, 0, 0]
resulting in:
precs = [p0, p1]
recs = [r0, r1]
F1 values fs = [f0, f1] are computed for p0/r0,
and p1/r1. The position idx (argmax) of
the highest f1 is determined.
best_op_pt = {
'threshold' : thresholds[idx],
'f1' : fs[idx],
'prec' : precs[idx]
'rec' : recs[idx]
}
Finally the average precision (AP) is
computed. It derives from precs and recs:
for k=0 to k=n-1
AP = sum_ovr_k((recs_k - recs_[k-1]) * preds_k)
where n is number of thresholds,
recs_k and precs_k are precision and
recall at the kth threshold. By definition,
preds_n = 1, recs_n = 0.
Returned: a CurveSpecification instance
containing:
best_op_pt
precisions
recalls
avg_prec
:param labels: integer binary class labels.
Exs.: [1,1,0,0], ['yes', 'yes', 'no', 'yes']
:type labels: [int | str]
:param preds: predictions output from a classifier.
May be floats or integers
:type preds: [float | int]
:param class_id: ID of target class for which this
curve is being constructed
:type class_id: {int | str}
:param thresholds: list of decision thresholds to
decide whether preds are one class or the other.
If None, uses [0.2, 0.4, 0.6, 0.8, 1]
:type thresholds: [float | int]
:return: CurveSpecification instances with optimal
operating point, and lists with prec and recall
ready for drawing a PR curve
:rtype: CurveSpecification
:raises ValueError if labels hold more than
two distinct values
'''
if type(labels) != list:
labels = labels.tolist()
uniq_classes = set(labels)
if len(uniq_classes) > 2:
raise ValueError(
f"Labels limited to up to two distinct values; got {uniq_classes}"
)
if thresholds is None:
thresholds = [0.2, 0.4, 0.6, 0.8]
precisions = []
recalls = []
class_list = list(uniq_classes)
# Degenerate case: Only a single
# class ever occurs in the labels.
# To make the code below work, we
# add a copy of that only class to
# the class list of known classes,
# and log a warning:
if len(class_list) == 1:
cls.log.warn(
f"Only label {class_list[0]} occurs; always guessing that value."
)
class_list.append(class_list[0])
# So far, no undefined recall or precision
# i.e. no 0-denominator found:
undef_prec = False
undef_rec = False
undef_f1 = False
for threshold in thresholds:
y_pred = []
for pred in preds:
# Instead of just class_list[1],
# must guard against only one
# class (ID=0) in the labels.
# In that special case, we always
# predict 0:
if pred >= threshold:
y_pred.append(class_list[1])
else:
y_pred.append(class_list[0])
y_pred_tn = torch.tensor(y_pred)
# For 'No positive exist and classifier
# properly doesn't predict a positive,
# use:
# precision=1
# recall =1
# In this case prec and rec are undefined,
# causing division by 0:
try:
with warnings.catch_warnings():
# Action to take: Ignore
warnings.filterwarnings(
"error",
#category=UndefinedMetricWarning,
category=UserWarning,
)
precision = precision_score(y_true=labels,
y_pred=y_pred_tn,
pos_label=class_list[1],
zero_division='warn')
except Exception as _e:
# Was it a div by zero from the prec calc?
undef_prec = True
precision = Charter.DIV_BY_ZERO
try:
with warnings.catch_warnings():
# Action to take: Ignore
warnings.filterwarnings(
"error",
#category=UndefinedMetricWarning
category=UserWarning)
recall = recall_score(y_true=labels,
y_pred=y_pred_tn,
pos_label=class_list[1],
zero_division=Charter.DIV_BY_ZERO)
except Exception as _e:
# Was it a div by zero from the prec calc?
undef_rec = True
recall = Charter.DIV_BY_ZERO
precisions.append(precision)
recalls.append(recall)
precs_np = np.array(precisions)
recs_np = np.array(recalls)
with warnings.catch_warnings():
try:
warnings.filterwarnings(
"error",
#category=UndefinedMetricWarning
category=UserWarning)
warnings.filterwarnings(
"true_divide",
#category=UndefinedMetricWarning
category=RuntimeWarning)
f1_scores = 2 * (precs_np * recs_np) / (precs_np + recs_np)
except Exception as _e:
# Was it a div by zero from the prec calc?
undef_f1 = True
# When both prec and recall are 0,
# set f1 to zero:
f1_scores = torch.tensor([Charter.DIV_BY_ZERO] * len(precs_np))
best_op_idx = np.argmax(f1_scores)
best_operating_pt = BestOperatingPoint(thresholds[best_op_idx],
f1_scores[best_op_idx],
precisions[best_op_idx],
recalls[best_op_idx])
# To make average_precision computation
# work:
recs_np_padded = np.append(recs_np, [0])
precs_np_padded = np.append(precs_np, [1])
avg_precision = \
np.sum((recs_np_padded[:-1] - recs_np_padded[1:]) * precs_np_padded[:-1])
res = CurveSpecification(best_operating_pt,
recs_np_padded,
precs_np_padded,
thresholds,
avg_precision,
class_id,
undef_prec=undef_prec,
undef_rec=undef_rec,
undef_f1=undef_f1)
return res
#------------------------------------
# compute_confusion_matrix
#-------------------
@classmethod
def compute_confusion_matrix(cls,
truth_labels,
predicted_class_ids,
class_names,
normalize=False):
'''
Example Confustion matrix for 16 samples,
in 3 classes:
C_1-pred, C_2-pred, C_3-pred
C_1-true 3 1 0
C_2-true 2 6 1
C_3-true 0 0 3
The number of classes is needed to let
sklearn to know even about classes that were not
encountered.
Assumption: self.class_names contains list
of class names, i.e. not the numeric IDs, but the
ones to use when labeling the matrix.
:param truth_labels: truth labels as list of class ids
:type truth_labels: [int]
:param predicted_class_ids: list of class_ids that were
predicted, in same order as truth_labels
:type predicted_class_ids: [int]
:param class_names: list of class names as known to the
user, i.e. not the numeric class ints. But the names
to use as matrix labels in class id order!
:type class_names: [str]
:param normalize: whether or not to normalize ROWS
to add to 1. I.e. turn cells into percentages
:type normalize: bool
:return: a dataframe of the confusion matrix; columns
and rows (i.e. index) set to class ids
:rtype: pd.DataFrame
'''
conf_matrix = torch.tensor(
confusion_matrix(
truth_labels, # Truth
predicted_class_ids, # Prediction
labels=list(range(len(class_names))) # Numeric class ID labels
))
if normalize:
conf_matrix = cls.calc_conf_matrix_norm(conf_matrix)
# Turn conf matrix from tensors to numpy, and
# from there to a dataframe:
conf_matrix_df = pd.DataFrame(conf_matrix.numpy(),
index=class_names,
columns=class_names)
return conf_matrix_df
#------------------------------------
# calc_conf_matrix_norm
#-------------------
@classmethod
def calc_conf_matrix_norm(cls, conf_matrix):
'''
Calculates a normalized confusion matrix.
Normalizes by the number of samples that each
species contributed to the confusion matrix.
Each cell in the returned matrix will be a
percentage of the number of samples for the
row. If no samples were present for a
particular class, the respective cells will
contain -1.
It is assumed that rows correspond to the classes
truth labels, and cols to the classes of the
predictions.
:param conf_matrix: confusion matrix to normalize
:type conf_matrix: {pd.DataFrame[int] | np.array | torch.Tensor}
:returned a new confusion matrix with cells replaced
by the percentage of time that cell's prediction
was made. Cells of classes without any samples in
the dataset will contain -1
:rtype matches input type
'''
# Get the sum of each row, which is the number
# of samples in that row's class. Then divide
# each element in the row by that num of samples
# to get the percentage of predictions that ended
# up in each cell:
# When a class had no samples at all,
# there will be divide-by-zero occurrences.
# Suppress related warnings. The respective
# cells will contain nan:
with np.errstate(divide='ignore', invalid='ignore'):
if type(conf_matrix) == np.ndarray:
return sklearn.preprocessing.normalize(conf_matrix, norm='l1')
elif type(conf_matrix) == torch.Tensor:
return conf_matrix.true_divide(
torch.sum(conf_matrix, axis=1).unsqueeze(-1))
elif type(conf_matrix) == pd.DataFrame:
return conf_matrix.div(conf_matrix.sum(axis='columns'),
axis='rows')
else:
raise TypeError(
f"Matrix must be a dataframe, numpy array, or tensor, not {type(conf_matrix)}"
)
#------------------------------------
# compute_multiclass_pr_curves
#-------------------
@classmethod
def compute_multiclass_pr_curves(cls,
truth_labels,
raw_preds,
thresholds=[0.2, 0.4, 0.6, 0.8]):
'''
Computes the data needed to draw
a family of PR curves for the results
of multiclass classifier output.
Returns a dict of the constituent
single-class curve specs, and a
mean average precision (mAP) score
for all curves combined.
Each result dict maps a class ID
to all info needed for one of the
curves:
1:
{'best_op_pt' : best_operating_pt,
'precisions' : precisions,
'recalls' : recalls,
'thresholds' : thresholds,
'avg_prec' : avg_precision
}
2:
{'best_op_pt' : best_operating_pt,
'precisions' : precisions,
'recalls' : recalls,
'thresholds' : thresholds,
'avg_prec' : avg_precision
}
where best_op_pt is:
{'threshold' : <optimal decision probability value>
'f1' : <f1 at the optimal threshold>
'prec' : <precision at the optimal threshold>
'thresholds' : thresholds,
'rec' : <recall at the optimal threshold>
}
Each of the avg_prec is the
the average of precisions across the
samples of one class (AP). I.e. there will
be as many elements in average_precisions
as there are classes.
The Mean Average Precision (mAP) is
the mean of the average_precision values.
This measure summarizes the family of PR curves.
It is comparable to AUC ROC.
The precisions and recalls returns are dicts.
The keys are class IDs, and the values are the
precisions for that class. They are the quantities
from which the average_precision values are
computed.
Summary:
o precisions/recalls are the lowest granularity
of information: the per class precs and recs
at different thresholds.
There are as many entries in these dicts as
there are classes. And prec/rec value pair
from the precisions and recalls dict are results
of one threshold.
TODO: o finish this sentence by running and
seeing what's what
o A unit test for this method
o Finally: the actual drawing of the
curves with pyplot
o average_precision aggregates the precisions
of one class across multiple thresholds. There
will be as many entries in this dict as there
are classes.
o mAP aggregates the average_precision values
across all classes. This is one number.
:param truth_labels: all truth labels shaped
torch.Size([num-batches, batch-size])
:type truth_labels: Tensor
:param raw_preds: the logits for each class for
each sample as
torch.Shape([num-batches, batch-size, num-classes])
:type raw_preds: Tensor
:return: (precisions, recalls, average_precisions, mAP)
:rtype: ({int : [floats]}, {int : [floats]}, [floats], float)
'''
(num_batches, batch_size, num_classes) = raw_preds.shape
num_samples = num_batches * batch_size
# Will alternately treat each class
# prediction as a one-vs-all binary
# classification.
#
# Ex. let labels = [1,0,0,1,2]
# and preds = [0.3,0.6,0.1,0.7,0.9]
#
# Convert the labels to a one-hot vector;
# the width of the binarized labels is
# num_classes:
#
# L A B E L S P R E D S
# ------------ ----------
# [1, [[0, 1, 0], [0.3,
# 0, [1, 0, 0], 0.6,
# 0, ==> [1, 0, 0], 0.1,
# 1, [0, 1, 0], 0.7,
# 2] [0, 0, 1]] 0.9]
#
# Then evaluate each label column vector
# separately.
bin_labels = label_binarize(truth_labels.flatten(),
classes=list(range(num_classes)))
assert (bin_labels.shape == torch.Size([num_samples, num_classes]))
assert(raw_preds.shape == \
torch.Size([num_batches, batch_size, num_classes])
)
# Want straight down: logits for each class, for
# each sample ('lst' for 'list'):
raw_preds_lst = raw_preds.reshape([num_samples, num_classes])
assert (raw_preds_lst.shape == bin_labels.shape)
# Turn logits into probs, rowise:
preds = torch.softmax(raw_preds_lst, dim=1)
# Place to hold the result dicts
# from compute_binary_pr_curve()
# for each of the classes. This
# will be class-name : binary-result-dict
all_curves_info = {}
# Go through each column, class_id i.e. the
# 1/0-vector label columns and preds
# columns for one class at
# a time, and get the prec/rec numbers.
for col_idx in range(num_classes):
bin_label_col = torch.tensor(bin_labels[:, col_idx])
preds_col = preds[:, col_idx]
# Get all info for this single, binary
# classification: list of 1/0 labels, and
# list of floats, which are the preds for
# the current class:
#**************
# # Using sklearn's precision_recall_curve,
# # which determines thresholds by its own
# # algorithm:
#
# from sklearn.metrics import precision_recall_curve
# sklearn_precs,\
# sklearn_recs,\
# sklearn_thresholds = \
# precision_recall_curve(bin_label_col, preds_col)
#**************
# Obtain the information needed to
# draw one PR curve: a CurveSpecification
# instance:
one_class_curve = cls.compute_binary_pr_curve(
bin_label_col,
preds_col,
col_idx, # class_id
thresholds)
# Accumulate the curve indices
# in a dict, keyed by class ID:
all_curves_info[col_idx] = one_class_curve
avg_precs = [
binary_curve_info['avg_prec']
for binary_curve_info in all_curves_info.values()
]
mAP = np.mean(np.array(avg_precs)).tolist()
return (all_curves_info, mAP)
# ----------------- Visualizations ---------------
#------------------------------------
# fig_from_conf_matrix
#-------------------
@classmethod
def fig_from_conf_matrix(cls,
conf_matrix,
supertitle='Confusion Matrix\n',
subtitle='',
write_in_fields=CELL_LABELING.DIAGONAL):
'''
Given a confusion matrix, return a
matplotlib.pyplot Figure with a heatmap of the matrix.
The write_in_fields arg controls whether or not
each cell is filled with a label indicating its
value. If:
o CELL_LABELING.ALWAYS : always write the labels
o CELL_LABELING.NEVER : never write the labels
o CELL_LABELING.DIAGONAL : only label the diagonals
o CELL_LABELING.AUTO : only write labels if number of classes
is <= CELL_LABELING.AUTO.value
Result form:
C_1-pred, C_2-pred, C_3-pred
C_1-true 3 1 0
C_2-true 2 6 1
C_3-true 0 0 3
:param conf_matrix: nxn confusion matrix representing
rows:truth, cols:predicted for n classes
:type conf_matrix: pd.DataFrame
:param supertitle: main title at top of figure
:type supertitle: str
:param subtitle: title for the confusion matrix
only. Ex: "data normalized to percentages"
:type subtitle: str
:param write_in_fields: how many cells, if any should
contain labels with the cell values.
:type write_in_fields: CELL_LABELING
:return: matplotlib figure with confusion
matrix heatmap.
:rtype: pyplot.Figure
'''
if type(write_in_fields) != CELL_LABELING:
raise TypeError(
f"Arg write_in_fields must be a CELL_LABELING enum member, not {write_in_fields}"
)
class_names = conf_matrix.columns
# Subplot 111: array of subplots has
# 1 row, 1 col, and the requested axes
# is in position 1 (1-based):
# Need figsize=(10, 5) somewhere
fig, ax = plt.subplots()
# Make a copy of the cmap, so
# we can modify it:
cmap = copy.copy(col_map.Blues)
fig.set_tight_layout(True)
fig.suptitle(supertitle, fontsize='large', fontweight='extra bold')
# Later matplotlib versions want us
# to use the mticker axis tick locator
# machinery:
ax.xaxis.set_major_locator(mticker.MaxNLocator('auto'))
ticks_loc = ax.get_xticks().tolist()
ax.xaxis.set_major_locator(mticker.FixedLocator(ticks_loc))
ax.set_xticklabels([class_name for class_name in ticks_loc],
rotation=45)
ax.yaxis.set_major_locator(mticker.MaxNLocator('auto'))
ticks_loc = ax.get_yticks().tolist()
ax.yaxis.set_major_locator(mticker.FixedLocator(ticks_loc))
ax.set_yticklabels([class_name for class_name in ticks_loc])
# Add cell labels if requested:
if write_in_fields == CELL_LABELING.ALWAYS or \
(CELL_LABELING.AUTO and len(class_names) <= CELL_LABELING.AUTO.value):
annot = conf_matrix.copy()
mask = None
elif write_in_fields == CELL_LABELING.DIAGONAL:
# Fill with a copy of conf matrix with strings,
# but room for up to 3 chars:
#**************
#annot = np.full_like(conf_matrix, '', dtype='U4')
#np.fill_diagonal(annot, np.diag(conf_matrix).astype(str))
#annot = np.empty_like(conf_matrix)
#np.fill_diagonal(annot, np.diag(conf_matrix).astype(str))
# Fill a new df with True, where df is same
# dimensions as another df: annot:
mask = pd.DataFrame(
np.array([True] * annot.size).reshape(annot.shape))
np.fill_diagonal(mask.values, False)
#**************
else:
annot = None
mask = None
cmap.set_bad('gray')
heatmap_ax = sns.heatmap(
conf_matrix,
cmap=cmap,
square=True,
annot=annot, # Cell labels
mask=mask,
fmt='d', # Round to integers
cbar=True, # Do draw color bar legend
ax=ax,
linewidths=1, # Pixel,
linecolor='gray',
robust=True # Compute colors from quantiles instead of
# most extreme values
)
# Add '%' after cell numbers; note that fmt='d%'
# leads to an error; I suspect there is a seaborn
# heatmap fmt value that would add '%', but I don't
# have time for frigging format strings:
for txt in heatmap_ax.texts:
txt.set_text(txt.get_text() + " %")
heatmap_ax.set_xticklabels(heatmap_ax.get_xticklabels(), rotation=45)
heatmap_ax.set_title(
subtitle,
fontdict={
'fontsize': 'medium',
'fontweight': 'bold',
},
pad=12 # Distance above matrix in pt
)
# Label x and y to clarify what's predicted,
# and what's truth; also: have the x-axis label
# at the top:
heatmap_ax.set_xlabel('True Classes', fontweight='bold')
heatmap_ax.xaxis.set_label_position('top')
heatmap_ax.set_ylabel('Predicted Classes', fontweight='bold')
fig = heatmap_ax.get_figure()
fig.suptitle(supertitle, fontsize='large', fontweight='extra bold')
return fig
# -------------------- Utilities for Charter Class --------------
@classmethod
def read_conf_matrix_from_file(cls, cm_path):
'''
Read a previously computed confusion matrix from
file. Return a dataframe containing the cm.
Depending on the original dataframe/tensor,np_array
from which which the .csv was created, the first line
has a leading comma. This results in:
Unnamed: 0 foo bar fum
0 foo 1 2 3
1 bar 4 5 6
2 fum 7 8 9
Rather than the correct:
foo bar fum
foo 1 2 3
bar 4 5 6
fum 7 8 9
Since conf matrices are square, we can check
and correct for that.
NOTE: if arrays of predicted and truth classes are
available, rather than an already computed confusion
matrix saved to file, see compute_confusion_matrix().
:param cm_path: path to confusion matrix in csv format
:type cm_path: str
:return: confusion matrix as dataframe; no processing on numbers
:rtype: pd.DataFrame
'''
df = pd.read_csv(cm_path)
# If comma was missing, we have one fewer
# col names than row names:
if len(df.columns) != len(df.index):
df_good = df.iloc[:, 1:]
df_good.index = df.columns[1:]
else:
df_good = df
return df_good
class ModelArchive:
'''
classdocs
'''
#------------------------------------
# Constructor
#-------------------
def __init__(self,
config,
num_classes,
history_len=8,
model_root=None,
log=None):
'''
Constructor:
:param config: configuration structure
:type config: NeuralNetConfig
:param num_classes: number of target classes
:type num_classes: int
:param history_len: number of model snapshots to
maintain
:type history_len: int
:param model_root: path to where models
will be deposited
:type model_root: str
:param log: logging service to use. If
None, create new one for display output
:type log: LoggingService
'''
self.curr_dir = os.path.dirname(os.path.abspath(__file__))
# Model root directory:
if model_root is None:
self.model_root = os.path.abspath(
os.path.join(self.curr_dir,
'../runs_models')
)
else:
self.model_root = model_root
if os.path.exists(self.model_root) and \
not os.path.isdir(self.model_root):
raise FileExistsError(f"{self.model_root} exists but is not a directory")
# Ensure that intermediate dirs exist:
try:
os.makedirs(self.model_root)
except FileExistsError:
pass
if log is None:
self.log = LoggingService()
else:
self.log = log
self.history_len = history_len
# Create a subdirectory of model_root
# where this archive keeps its models.
# The subdir is guaranteed to be unique
# among model_root's siblings, and it will
# be created:
self.run_subdir = self._construct_run_subdir(config,
num_classes,
self.model_root)
# Queue to track models, keeping the
# number of saved models to history_len:
self.model_fnames = deque(maxlen=self.history_len)
#------------------------------------
# save_model
#-------------------
def save_model(self, model, epoch):
'''
Saves and retains trained models
on disk.
Within a subdir the method maintains a queue
of files of len history_len:
fname_1_ep_0.pth
fname_2_ep_1.pth
...
fname_<history_len>.pth
where ep_<n> is the epoch during training
where the model of that moment is being
saved.
When history_len model files are already present,
removes the oldest.
Assumptions:
o self.fname_els_dict contains prop/value
pairs for use in FileUtils.construct_filename()
{'bs' : 32,
'lr' : 0.001,
...
}
o self model_fnames is a deque the size of
which indicates how many models to save
before discarding the oldest one as new
ones are added
:param model: model to save
:type model: nn.module
:param epoch: the epoch that created the model
:type epoch: int
:param history_len: number of snapshot to retain
:type history_len: int
'''
deque_len = len(self.model_fnames)
if deque_len >= self.history_len:
# Pushing a new model fname to the
# front will pop the oldest from the
# end. That file needs to be deleted:
oldest_model_path = self.model_fnames[-1]
else:
# No file will need to be deleted.
# Still filling our allotment:
oldest_model_path = None
model_fname = FileUtils.construct_filename(self.fname_els_dict,
prefix='mod',
suffix=f"_ep{epoch}.pth",
incl_date=True)
model_path = os.path.join(self.run_subdir, model_fname)
# As recommended by pytorch, save the
# state_dict for portability:
torch.save(model.state_dict(), model_path)
self.model_fnames.appendleft(model_path)
if oldest_model_path is not None:
try:
os.remove(oldest_model_path)
except Exception as e:
self.log.warn(f"Could not remove old model: {repr(e)}")
#------------------------------------
# restore_model
#-------------------
def restore_model(self, model_path, config=None):
'''
Given the path to a saved model,
load and return it. The saved file
is the saved model's state_dict.
So, the method must first create a
model instance of the correct type.
Then the state is loaded into that
instance.
:param model_path:
:type model_path:
:param config: a config structure that will be
use to decide which model class to instantiate.
If None, attempts to reconstruct the
information from the model_path.
:type config: NeuralNetConfig
:return: loaded model
:rtype: torch.nn.module
'''
if config is None:
model = self._instantiate_model(config=config)
else:
model = self._instantiate_model(run_path_str=model_path)
model.load_state_dict(torch.load(model_path))
return model
#------------------------------------
# _instantiate_model
#-------------------
def _instantiate_model(self, run_path_str=None, config=None):
'''
Returns a model based on information in
the config structure, or the info encoded
in the run_path_str file name.
One of run_path_str or config must be non-None.
If both are non-None, uses config.
File paths that encode run parameters look like
this horror:
model_2021-03-11T10_59_02_net_resnet18_pretrain_0_lr_0.01_opt_SGD_bs_64_ks_7_folds_0_gray_True_classes_10.pth
:param run_path_str: a path name associated with
a model.
:type run_path_str:
:param config: run configuration structure
:type config: NeuralNetConfig
:return: a model
:rtype: torch.nn.module
'''
if config is None:
# Get a dict with info
# in a standard (horrible) file name:
fname_props = FileUtils.parse_filename(run_path_str)
else:
fname_props = config.Training
data_root = config.Paths.root_train_test_data
class_names = FileUtils.find_class_names(data_root)
fname_props['classes'] = len(class_names)
fname_props['pretrain'] = config.Training.getint('freeze', 0)
model = NetUtils.get_net(net_name=fname_props['net_name'],
num_classes=fname_props['classes'],
freeze=fname_props['pretrain'],
to_grayscale=fname_props['to_grayscale']
)
return model
# ---------------- Utils -------------
#------------------------------------
# _construct_run_subdir
#-------------------
def _construct_run_subdir(self,
config,
num_classes,
model_root):
'''
Constructs a directory name composed of
elements specified in utility.py's
FileUtils file/config info dicts.
Ensures that <model_root>/subdir_name does
not exist. If it does, keeps adding '_r<n>'
to the end of the dir name.
Final str will look like this:
model_2021-03-23T15_38_39_net_resnet18_pre_True_frz_6_bs_2_folds_5_opt_SGD_ks_7_lr_0.01_gray_False
Details will depend on the passed in
configuration.
Instance var fname_els_dict will contain
all run attr/values needed for calls to
FileUtils.construct_filename()
:param config: run configuration
:type config: NeuralNetConfig
:param num_classes: number of target classes
:type num_classes: int
:param model_root: full path to dir where the
subdir is to be created
:type model_root: str
:return: unique subdir name of self.model_root,
which has been created
:rtype: str
'''
# Using config, gather run-property/value
# pairs to include in the dir name:
fname_els_dict = {}
section_dict = config.Training
for el_name, el_abbr in FileUtils.fname_long_2_short.items():
el_type = FileUtils.fname_el_types[el_abbr]
if el_type == int:
fname_els_dict[el_name] = section_dict.getint(el_name)
elif el_type == str:
fname_els_dict[el_name] = section_dict.get(el_name)
elif el_type == float:
fname_els_dict[el_name] = section_dict.getfloat(el_name)
elif el_type == bool:
fname_els_dict[el_name] = section_dict.getboolean(el_name)
elif callable(el_type):
# A lambda or func. Apply it:
fname_els_dict[el_name] = el_type(section_dict[el_name])
fname_els_dict['num_classes'] = num_classes
# Save this root name:
self.fname_els_dict = fname_els_dict
# Get the subdir name (without leading path):
dir_basename = FileUtils.construct_filename(
fname_els_dict,
prefix='models',
suffix=None,
incl_date=True)
final_dir_path = os.path.join(model_root, dir_basename)
# Disambiguate by appending '_r<n>' as needed:
disambiguation = 1
while os.path.exists(final_dir_path):
new_basename = f"{dir_basename}_r{disambiguation}"
final_dir_path = os.path.join(model_root, new_basename)
disambiguation += 1
os.makedirs(final_dir_path)
return final_dir_path
class CrossValidatingDataLoader(DataLoader):
'''
Subclass of torch.utils.data.DataLoader. Provides
stratified k-fold crossvalidation in single-machine,
(optionally) single-GPU context.
Instantiate this class if running only on a
single machine, optionally using a single GPU. Else,
instantiate the MultiprocessingDataLoader subclass
instead.
An instance of this class wraps any dict-API dataset instance,
which provides tuples , for instance (<img-tensor>, class-label-int)
from the file system when given a sample ID.
This subclass of torch.utils.data.DataLoader specilizes
the default by using a stratified k-fold cross validation
sampler. That underlying sampler manages partitioning of
samples into folds, and successively feeding samples from
the training folds. The sampler also manages the 'switching out'
of folds to take the role of test fold in round robin fashion.
This DataLoader instance also managing combination of
samples into batches.
An instance of this class presents an iterator API, additionally
serving the test samples whenever one set of train folds are
exhausted. Example: assume
o k-fold cross validation k = 5
for split in range(k):
for batch in my_dataloader:
try:
<feed training batch to emerging model>
except EndOfSplit as e:
print(e.message) # Just for debugging
break
# Exhausted all train folds of one split
# Now test current state of the
# model using this split's test samples,
# which are available as an iterator from the
# dataloader:
for (img_tensor, label) in my_ataloader.validation_samples():
<test model on img_tensor>
# next split
The validation_samples() method is a generator that provides the content of
the just exhausted split's validation samples.
NOTE: when re-setting an instance of this class
for a new epoch, client must call set_epoch()
with the new epoch number to ensure proper
shuffling randomness. Such a reset occurs implicitly
with the often used idiom:
for i,res = enumerate(dataloader)
The enumerate() starts the same dataloader instance
from the beginning.
If shuffle is False, set_epoch() needs not be called.
But doing so does no harm.
'''
#------------------------------------
# Constructor
#-------------------
def __init__(self,
dataset,
batch_size=32,
shuffle=True,
seed=42,
num_workers=0,
pin_memory=False,
prefetch_factor=2,
drop_last=True,
num_folds=10,
sampler=None,
logger=None):
'''
This instance will use cross validation
as it serves out samples. The client determines
the number of folds to use. Example for
num_folds of 2:
Split1:
TrainFold1 TrainFold2 ValidationFold
sample1 sample2 sample3
sample4 sample5 sample6
Split2:
TrainFold1 TrainFold2 ValidationFold
sample3 sample4 sample2
sample1 sample6 sample5
This dataloader will create two sequences,
like this:
For use with training: [sample1, sample4, sample2, sample5]
For use with validation: [sample4, sample6]
after the training
sequence is used up
Assuming batch_size of two, this dataloader's
client will receive one row from each
call to next():
[[sample1, sample4],
[sample2, sample5],
[None , None]
]
The None tuple indicates that this split has
been exhausted, and it is time to validate.
The client then calls validation_samples() on
this dataloader instance to receive one validation
sample at a time. The client will predict the
(target) class for each of these validation samples,
and tally successes and failures. The client should
then compute the compute validation accuracy from
that series of successes and failures.
Calling next() again will create a new split,
and again feed out the samples in the respective
new folds.
The feed terminates after as many splits as there
are folds. Any following call to next() will raise
a StopIteration exception.
:param dataset: underlying map-store that
supplies(img_torch, label) tuples
:type dataset: BirdDataset
:param batch_size: number of samples to combine into
a batch to feed model during training
:type batch_size: int
:param shuffle: whether or not to shuffle the
dataset once, initially.
:type shuffle: bool
:param seed: random seed to use if shuffle is True
:type shuffle: int
:param num_workers: number of threads used to preload
:type num_workers: int
:param pin_memory: set to True if using a GPU. Speeds
transfer of tensors from CPU to GPU
:type pin_memory: bool
:param prefetch_factor: how many samples to prefetch from
underlying database to speed access to file system
:type prefetch_factor: int
:param drop_last: whether or not to serve only partially
filled batches. Those occur when samples cannot be
evenly packed into batches.
:type drop_last: bool
:param num_folds: the 'k' in k-fold cross validation
:type num_folds: int
:param sampler: Only used when MultiprocessingDataLoader
is being instantiated, and that class's __init__()
calls super(). Leave out for singleprocess/single-GPU
use
:type sampler: {None | DistributedSKFSampler}
:param logger: the LoggingService instance to use
for logging info/warnings/errors. If None, fetches
the LoggingService singleton.
:type logger: LoggingService
'''
if len(dataset) == 0:
raise ValueError("Dataset is empty, nothing to load")
self.drop_last = drop_last
if logger is None:
self.log = LoggingService()
else:
self.log = logger
# Sampler will only be set if a subclass instance
# of MultiprocessingDataLoader is being initialized.
# Else, running single process:
if sampler is None:
self.sampler = SKFSampler(dataset,
num_folds=num_folds,
shuffle=shuffle,
drop_last=drop_last,
seed=seed)
else:
self.sampler = sampler
if not isinstance(batch_size, int) or batch_size <= 0:
msg = f"Batch size must be a positive int, not "
# Complete the error msg according which of
# the two failure conditions occurred:
msg += type(batch_size).__name__ if not isinstance(batch_size, int)\
else f"{batch_size}"
raise ValueError(msg)
self.batch_size = batch_size
self.num_folds = num_folds
# Total num of batches served when
# rotating through all folds is computed
# the first time __len__() is called:
self.num_batches = None
self.curr_split_idx = -1
super().__init__(dataset,
batch_size=batch_size,
sampler=self.sampler,
num_workers=num_workers,
pin_memory=pin_memory,
prefetch_factor=prefetch_factor,
drop_last=drop_last)
#------------------------------------
# __len__
#-------------------
def __len__(self):
'''
Number of batches this loader will
feed out. Example:
o 12 samples total
o 3 folds
o 2 batch size
o 4 samples in each split (12/3)
o 2 batches per split (samples-each-split / batch-size)
o 3 number of trips through folds
o 2 number of folds in each of the 3
trips (num-folds - hold-out-fold)
o 12 batches total: batches-per-fold * folds-per-trip * num-folds
2*2*3 = 12
'''
# Compute number of batches only once:
if self.num_batches is None:
# This computation can surely be more
# concise and direct. But it happens only
# once, and this step by step is easier
# on the eyes than one minimal expression:
num_samples = len(self.sampler)
if num_samples == 0:
raise ValueError("No samples to serve.")
# Rounded-down number of samples that fit into each fold.
# Having 34 samples with 3 folds, that is 34/3 == ~11
samples_per_fold = num_samples // self.num_folds
# For training we get 2 folds worth of samples,
# with one fold held out: 11*2 = 22
samples_per_split = samples_per_fold * (self.num_folds - 1)
# As many permutations as there are folds: 3 * 22: 66
total_train_samples = self.num_folds * samples_per_split
# Convert to batches. Assume batch_size of 2:
# 66 // 2 = 33
self.total_num_batches = total_train_samples // self.batch_size
if self.total_num_batches == 0:
self.log.warn(
f"Not enough data ({total_train_samples}) for even one batch (of size {self.batch_size})"
)
remainder_samples = total_train_samples % self.batch_size
if not self.drop_last and remainder_samples > 0:
# Add the final partially filled batch,
# if num_samples not a multiple of batches += 1
self.total_num_batches += 1
return self.total_num_batches
# #------------------------------------
# # __iter__
# #-------------------
#
def __iter__(self):
# Call to __next__() returns
# a generator, which does the
# right thing with next(), list(),
# and for loops. Return that iterator:
return (self.__next__())
#------------------------------------
# __next__
#-------------------
def __next__(self):
# Loop over all splits (i.e. over all
# configurations of which fold is for
# validation.
# Get one list of sample IDs that
# covers all train samples in one split.
# And one list of sample IDs that
# are to be used for validation in this
# split.
# Raise EndOfSplit exception at the end of
# each split, i.e. when client is to validate.
# When all splits are exhausted, raise StopIteration.
for split_train_idxs, split_test_idxs in self.sampler:
# Keep track of which split we are working
# on. Needed only as info for client; not
# used for logic in this method:
self.curr_split_idx += 1
# split_train_idxs has all sample IDs
# to use for training in this split.
# The split_test_idxs holds the left-out
# sample IDs to use for testing once
# the split_train_idxs have been served out
# one batch at a time.
# Set this split's test sample ids aside for client
# to retrieve later via: get_split_test_sample_ids()
# once they pulled all the batches of this
# split:
self.curr_test_sample_ids = []
for sample_idx in split_test_idxs:
self.curr_test_sample_ids.append(
self.dataset.sample_id_by_sample_idx(sample_idx))
# Create one batch:
num_train_sample_ids = len(split_train_idxs)
num_batches = num_train_sample_ids // self.batch_size
num_remainder_samples = num_train_sample_ids % self.batch_size
batch_start_idx = 0
# Create num_batches batches from the
# training data of this split:
for _batch_count in range(num_batches):
batch = None
# Truth labels for each sample in
# the current batch:
y = []
batch_end_idx = batch_start_idx + self.batch_size
curr_batch_range = range(batch_start_idx, batch_end_idx)
for train_sample_idx in curr_batch_range:
# Index into the current split's list
# of training sample ids:
sample_idx = split_train_idxs[train_sample_idx]
# Get one pair: <img-tensor>, class_id_int:
(img_tensor,
label) = self.dataset.sample_by_idx(sample_idx)
expanded_img_tensor = unsqueeze(img_tensor, dim=0)
batch = (cat((batch, expanded_img_tensor), dim=0)
if batch is not None else expanded_img_tensor)
y.append(label)
# Got one batch ready:
yield (batch, torch.tensor(y))
# Client consumed one batch in current split.
# Next batch: Starts another batch size
# samples onwards in the train split:
batch_start_idx += self.batch_size
# Put together next batch:
continue
# Done all full batches. Any partial batch
# left over that we should include?
if num_remainder_samples > 0 and not self.drop_last:
batch = None
y = []
for sample_id in range(batch_start_idx, batch_start_idx +
num_remainder_samples):
(img_tensor, label) = self.dataset[sample_id]
expanded_img_tensor = unsqueeze(img_tensor, dim=0)
batch = (cat((batch, expanded_img_tensor))
if batch is not None else expanded_img_tensor)
y.append(label)
yield (batch, torch.tensor(y))
# Let client know that all batches for one split
# have been delivered by a None/None pair:
raise EndOfSplit()
# Next split:
continue
#------------------------------------
# get_curr_fold_idx
#-------------------
def get_curr_fold_idx(self):
return self.curr_split_idx
#------------------------------------
# get_split_test_sample_ids
#-------------------
def get_split_test_sample_ids(self):
try:
return self.curr_test_sample_ids
except:
return None
#------------------------------------
# validation_samples
#-------------------
def validation_samples(self):
'''
Generator that runs through every
test sample_id of the current fold,
and feeds (<img_tensor, label) pairs.
for (img_tensor, label) in my_bird_dataloader.validation_samples():
<test model>
'''
for sample_id in self.get_split_test_sample_ids():
yield self.dataset[sample_id]
#------------------------------------
# file_from_sample_id
#-------------------
def file_from_sample_id(self, sample_id):
'''
Given a sample_id, return the absolute
file path of the corresponding sample
in the file system.
We use the public dataset method.
:param sample_id: sample ID to look up
:type sample_id: int
'''
return self.dataset.file_from_sample_id(sample_id)
#------------------------------------
# class_from_sample_id
#-------------------
def class_from_sample_id(self, sample_id):
'''
Given a sample ID, return its class index.
:param sample_id: ID to look up
:type sample_id: int
:return: given sample's class ID
:rtype: int
'''
return self.dataset.sample_id_to_class[sample_id]
#------------------------------------
# set_epoch
#-------------------
def set_epoch(self, new_epoch):
'''
Must be called by client every time
a new epoch starts. The epoch number
is used by the sampler to shuffle
the dataset before beginning to draw
samples.
:param new_epoch: the epoch under which the dataloader
is (re)started
:type new_epoch: int
'''
self.sampler.set_epoch(new_epoch)
class BirdsBasicTrainerCV:
'''
classdocs
'''
# Number of intermediate models to save
# during training:
MODEL_ARCHIVE_SIZE = 20
# For some tensorboard displays:
# for how many epochs in the past
# to display data:
DISPLAY_HISTORY_LEN = 10
#------------------------------------
# Constructor
#-------------------
def __init__(self,
config_info,
device=0,
percentage=None,
debugging=False):
'''
:param config_info: all path and training parameters
:type config_info: NeuralNetConfig
:param debugging: output lots of debug info
:type debugging: bool
:param device: number of GPU to use; default is dev 0
if any GPU is available
:type device: {None | int}
:param percentage: percentage of training data to
use
:type percentage: {int | float}
'''
self.log = LoggingService()
if debugging:
self.log.logging_level = DEBUG
if percentage is not None:
# Integrity check:
if type(percentage) not in [int, float]:
raise TypeError(
f"Percentage must be int or float, not {type(percentage)}")
if percentage < 1 or percentage > 100:
raise ValueError(
f"Percentage must be between 1 and 100, not {percentage}")
if device is None:
device = 0
torch.cuda.set_device(device)
else:
available_gpus = torch.cuda.device_count()
if available_gpus == 0:
self.log.info("No GPU available; running on CPU")
else:
if device > available_gpus - 1:
raise ValueError(
f"Asked to operate on device {device}, but only {available_gpus} are available"
)
torch.cuda.set_device(device)
self.curr_dir = os.path.dirname(os.path.abspath(__file__))
try:
self.config = self.initialize_config_struct(config_info)
except Exception as e:
msg = f"During config init: {repr(e)}"
self.log.err(msg)
raise RuntimeError(msg) from e
try:
self.root_train_test_data = self.config.getpath(
'Paths', 'root_train_test_data', relative_to=self.curr_dir)
except ValueError as e:
raise ValueError(
"Config file must contain an entry 'root_train_test_data' in section 'Paths'"
) from e
self.batch_size = self.config.getint('Training', 'batch_size')
self.kernel_size = self.config.getint('Training', 'kernel_size')
self.min_epochs = self.config.Training.getint('min_epochs')
self.max_epochs = self.config.Training.getint('max_epochs')
self.lr = self.config.Training.getfloat('lr')
self.net_name = self.config.Training.net_name
self.pretrained = self.config.Training.getboolean('pretrained', False)
self.num_folds = self.config.Training.getint('num_folds')
self.freeze = self.config.Training.getint('freeze', 0)
self.to_grayscale = self.config.Training.getboolean(
'to_grayscale', True)
self.set_seed(42)
self.log.info("Parameter summary:")
self.log.info(f"network {self.net_name}")
self.log.info(f"pretrained {self.pretrained}")
if self.pretrained:
self.log.info(f"freeze {self.freeze}")
self.log.info(f"min epochs {self.min_epochs}")
self.log.info(f"max epochs {self.max_epochs}")
self.log.info(f"batch_size {self.batch_size}")
self.fastest_device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
self.device = self.fastest_device
self.num_classes = self.find_num_classes(self.root_train_test_data)
self.initialize_model()
sample_width = self.config.getint('Training', 'sample_width', 400)
sample_height = self.config.getint('Training', 'sample_height', 400)
self.train_loader = self.get_dataloader(sample_width,
sample_height,
perc_data_to_use=percentage)
self.log.info(f"Expecting {len(self.train_loader)} batches per epoch")
num_train_samples = len(self.train_loader.dataset)
num_classes = len(self.train_loader.dataset.class_names())
self.log.info(
f"Training set contains {num_train_samples} samples across {num_classes} classes"
)
self.class_names = self.train_loader.dataset.class_names()
log_dir = os.path.join(self.curr_dir, 'runs')
raw_data_dir = os.path.join(self.curr_dir, 'runs_raw_results')
self.setup_tensorboard(log_dir, raw_data_dir=raw_data_dir)
# Log a few example spectrograms to tensorboard;
# one per class:
TensorBoardPlotter.write_img_grid(
self.writer,
self.root_train_test_data,
len(self.class_names), # Num of train examples
)
# All ResultTally instances are
# collected here: (num_folds * num-epochs)
# each for training and validation steps.
self.step_results = ResultCollection()
self.log.debug(
f"Just before train: \n{'none--on CPU' if self.fastest_device.type == 'cpu' else torch.cuda.memory_summary()}"
)
try:
final_step = self.train()
self.visualize_final_epoch_results(final_step)
finally:
self.close_tensorboard()
#------------------------------------
# train
#-------------------
def train(self):
overall_start_time = datetime.datetime.now()
# Just for sanity: keep track
# of number of batches...
total_batch_num = 0
# Note: since we are cross validating, the
# data loader's set_epoch() method is only
# called once (automatically) during instantiation
# of the associated sampler. Moving from split
# to split includes shuffling if the caller
# specified that.
# Training
for split_num in range(self.train_loader.num_folds):
split_start_time = datetime.datetime.now()
self.initialize_model()
for epoch in range(self.max_epochs):
# Set model to train mode:
self.model.train()
epoch_start_time = datetime.datetime.now()
self.log.info(f"Starting epoch {epoch} training")
# Sanity check record: will record
# how many samples from each class were
# used:
self.class_coverage = {}
# Sanity records: will record number
# of samples of each class that are used
# during training and validation:
label_distrib = {}
batch_num = 0
self.log.info(
f"Train epoch {epoch}/{self.max_epochs} split {split_num}/{self.train_loader.num_folds}"
)
try:
for batch, targets in self.train_loader:
# Update the sanity check
# num of batches seen, and distribution
# of samples across classes:
batch_num += 1
total_batch_num += 1
# Update sanity check records:
for lbl in targets:
lbl = int(lbl)
try:
label_distrib[lbl] += 1
except KeyError:
label_distrib[lbl] = 1
try:
self.class_coverage[lbl]['train'] += 1
except KeyError:
self.class_coverage[lbl] = {
'train': 1,
'val': 0
}
self.log.debug(
f"Top of training loop: \n{'none--on CPU' if self.fastest_device.type == 'cpu' else torch.cuda.memory_summary()}"
)
images = FileUtils.to_device(batch, 'gpu')
labels = FileUtils.to_device(targets, 'gpu')
outputs = self.model(images)
loss = self.loss_fn(outputs, labels)
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
# Remember the last batch's train result of this
# split (results for earlier batches of
# the same split will be overwritten). This statement
# must sit before deleting output and labels:
step_num = self.step_number(epoch, split_num,
self.num_folds)
self.remember_results(LearningPhase.TRAINING, step_num,
outputs, labels, loss)
self.log.debug(
f"Just before clearing gpu: \n{'none--on CPU' if self.fastest_device.type == 'cpu' else torch.cuda.memory_summary()}"
)
images = FileUtils.to_device(images, 'cpu')
outputs = FileUtils.to_device(outputs, 'cpu')
labels = FileUtils.to_device(labels, 'cpu')
loss = FileUtils.to_device(loss, 'cpu')
del images
del outputs
del labels
del loss
torch.cuda.empty_cache()
self.log.debug(
f"Just after clearing gpu: \n{'none--on CPU' if self.fastest_device.type == 'cpu' else torch.cuda.memory_summary()}"
)
except EndOfSplit:
end_time = datetime.datetime.now()
train_time_duration = end_time - epoch_start_time
# A human readable duration st down to minutes:
duration_str = FileUtils.time_delta_str(
train_time_duration, granularity=4)
self.log.info(
f"Done training epoch {epoch} of split {split_num} (duration: {duration_str})"
)
#***********
#print(f"****** num_batches in split: {batch_num}" )
#print(f"****** LblDist: {label_distrib}")
#***********
self.validate_split(step_num)
self.visualize_step(step_num)
# Save model, keeping self.model_archive_size models:
self.model_archive.save_model(self.model, epoch)
self.log.debug(
f"After eval: \n{'none--on CPU' if self.fastest_device.type == 'cpu' else torch.cuda.memory_summary()}"
)
# Next Epoch
continue
end_time = datetime.datetime.now()
train_time_duration = end_time - split_start_time
# A human readable duration st down to minutes:
duration_str = FileUtils.time_delta_str(train_time_duration,
granularity=4)
self.log.info(
f"Done training split {split_num} (duration: {duration_str})")
# Next split
continue
end_time = datetime.datetime.now()
epoch_duration = end_time - epoch_start_time
epoch_dur_str = FileUtils.time_delta_str(epoch_duration, granularity=4)
cumulative_dur = end_time - overall_start_time
cum_dur_str = FileUtils.time_delta_str(cumulative_dur, granularity=4)
msg = f"Done epoch {epoch} (epoch duration: {epoch_dur_str}; cumulative: {cum_dur_str})"
self.log.info(msg)
#******self.scheduler.step()
# Fresh results tallying
#self.results.clear()
self.log.info(
f"Training complete after {self.train_loader.num_folds} splits")
# Report the sanity checks:
self.log.info(f"Total batches processed: {total_batch_num}")
for cid in self.class_coverage.keys():
train_use, val_use = self.class_coverage[cid].items()
self.log.info(
f"{self.class_names[cid]} Training: {train_use}, Validation: {val_use}"
)
# All seems to have gone well. Report the
# overall result of the final epoch for the
# hparms config used in this process:
self.report_hparams_summary(self.latest_result)
# The final epoch number:
return epoch
#------------------------------------
# validate_split
#-------------------
def validate_split(self, step):
'''
Validate one split, using that split's
validation fold. Return time taken. Record
results for tensorboard and other record keeping.
:param step: current combination of epoch and
split
:type step: int
:return: number of epoch seconds needed for the validation
:rtype: int
'''
# Validation
self.log.debug(
f"Start of validation: \n{'none--on CPU' if self.fastest_device.type == 'cpu' else torch.cuda.memory_summary()}"
)
start_time = datetime.datetime.now()
self.log.info(f"Starting validation for step {step}")
self.model.eval()
with torch.no_grad():
for img_tensor, target in self.train_loader.validation_samples():
expanded_img_tensor = unsqueeze(img_tensor, dim=0)
expanded_target = unsqueeze(target, dim=0)
# Update sanity record:
self.class_coverage[int(target)]['val'] += 1
images = FileUtils.to_device(expanded_img_tensor, 'gpu')
label = FileUtils.to_device(expanded_target, 'gpu')
outputs = self.model(images)
loss = self.loss_fn(outputs, label)
images = FileUtils.to_device(images, 'cpu')
outputs = FileUtils.to_device(outputs, 'cpu')
label = FileUtils.to_device(label, 'cpu')
loss = FileUtils.to_device(loss, 'cpu')
self.remember_results(LearningPhase.VALIDATING, step, outputs,
label, loss)
del images
del outputs
del label
del loss
torch.cuda.empty_cache()
end_time = datetime.datetime.now()
val_time_duration = end_time - start_time
# A human readable duration st down to minues:
duration_str = FileUtils.time_delta_str(val_time_duration,
granularity=4)
self.log.info(f"Done validation (duration: {duration_str})")
return val_time_duration
# ------------- Utils -----------
#------------------------------------
# report_acc_loss
#-------------------
def report_acc_loss(self, phase, epoch, accumulated_loss):
self.writer.add_scalar(f"loss/{phase}", accumulated_loss, epoch)
#------------------------------------
# remember_results
#-------------------
def remember_results(
self,
phase,
step,
outputs,
labels,
loss,
):
# Add the results
tally = ResultTally(step, phase, outputs, labels, loss,
self.num_classes, self.batch_size)
# Add result to intermediate results collection of
# tallies:
self.results[step] = tally
# Same with the session-wide
# collection:
self.step_results.add(tally)
#------------------------------------
# visualize_step
#-------------------
def visualize_step(self, step):
'''
Take the ResultTally instances
in the train and val ResultCollections
in self.results, and report appropriate
aggregates to tensorboard. Computes
f1 scores, accuracies, etc. for given
step.
Separately for train and validation
results: build one long array
of predictions, and a corresponding
array of labels. Also, average the
loss across all instances.
The preds and labels as rows to csv
files.
'''
val_tally = self.results[(step, str(LearningPhase.VALIDATING))]
train_tally = self.results[(step, str(LearningPhase.TRAINING))]
result_coll = ResultCollection()
result_coll.add(val_tally, step)
result_coll.add(train_tally, step)
self.latest_result = {'train': train_tally, 'val': val_tally}
# If we are to write preds and labels to
# .csv for later additional processing:
if self.csv_writer is not None:
self.csv_writer.writerow([
step, train_tally.preds, train_tally.labels, val_tally.preds,
val_tally.labels
])
TensorBoardPlotter.visualize_step(
result_coll, self.writer,
[LearningPhase.TRAINING, LearningPhase.VALIDATING], step,
self.class_names)
# History of learning rate adjustments:
lr_this_step = self.optimizer.param_groups[0]['lr']
self.writer.add_scalar('learning_rate', lr_this_step, global_step=step)
#------------------------------------
# visualize_final_epoch_results
#-------------------
def visualize_final_epoch_results(self, epoch):
'''
Reports to tensorboard just for the
final epoch.
Expect self.latest_result to be the latest
ResultTally.
'''
# DISPLAY_HISTORY_LEN holds the number
# of historic epochs we will show. Two
# results per epochs --> need
# 2*DISPLAY_HISTORY_LEN results. But check
# that there are that many, and show fewer
# if needed:
num_res_to_show = min(len(self.step_results),
2 * self.DISPLAY_HISTORY_LEN)
f1_hist = self.step_results[-num_res_to_show:]
# First: the table of train and val f1-macro
# scores for the past few epochs:
#
# |phase|ep0 |ep1 |ep2 |
# |-----|-----|----|----|
# |train| f1_0|f1_1|f1_2|
# | val| f1_0|f1_1|f1_2|
f1_macro_tbl = TensorBoardPlotter.make_f1_train_val_table(f1_hist)
self.writer.add_text('f1/history', f1_macro_tbl)
# Now, in the same tensorboard row: the
# per_class train/val f1 scores for each
# class separately:
#
# |class|weighted mean f1 train|weighted mean f1 val|
# |-----|----------------------|--------------------|
# | c1 |0.1 |0.6 |
# | c2 |0.1 |0.6 |
# | c3 |0.1 |0.6 |
# ------|----------------------|--------------------|
f1_all_classes = TensorBoardPlotter.make_all_classes_f1_table(
self.latest_result, self.class_names)
self.writer.add_text('f1/per-class', f1_all_classes)
#------------------------------------
# report_hparams_summary
#-------------------
def report_hparams_summary(self, latest_result):
'''
Called at the end of training. Constructs
a summary to report for the hyperparameters
used in this process. Reports to the tensorboard.
Hyperparameters reported:
o lr
o optimizer
o batch_size
o kernel_size
Included in the measures are:
o balanced_accuracy (train and val)
o mean_accuracy_train (train and val)
o epoch_prec_weighted
o epoch_recall_weighted
o epoch_mean_loss (train and val)
:param latest_result: dict with keys 'train' and
'val', holding the respective most recent
(i.e. last-epoch) ResultTally
:type latest_result: {'train' : ResultTally,
'val' : ResultTally
}
'''
# Get the latest validation tally:
train_tally = latest_result['train']
val_tally = latest_result['val']
hparms_vals = OrderedDict({
'net':
self.net_name,
'pretrained':
f"{self.pretrained}",
'lr_initial':
self.config.Training.lr,
'optimizer':
self.config.Training.opt_name,
'batch_size':
self.config.getint('Training', 'batch_size'),
'kernel_size':
self.config.getint('Training', 'kernel_size'),
'to_grayscale':
self.to_grayscale
})
metric_results = {
'zz_balanced_adj_acc_train': train_tally.balanced_acc,
'zz_balanced_adj_acc_val': val_tally.balanced_acc,
'zz_acc_train': train_tally.accuracy,
'zz_acc_val': val_tally.accuracy,
'zz_epoch_weighted_prec': val_tally.prec_weighted,
'zz_epoch_weighted_recall': val_tally.recall_weighted,
'zz_epoch_mean_loss_train': train_tally.mean_loss,
'zz_epoch_mean_loss_val': val_tally.mean_loss
}
self.writer.add_hparams(hparms_vals, metric_results)
#------------------------------------
# get_dataloader
#-------------------
def get_dataloader(self,
sample_width,
sample_height,
perc_data_to_use=None):
'''
Returns a cross validating dataloader.
If perc_data_to_use is None, all samples
under self.root_train_test_data will be
used for training. Else percentage indicates
the percentage of those samples to use. The
selection is random.
:param sample_width: pixel width of returned images
:type sample_width: int
:param sample_height: pixel height of returned images
:type sample_height: int
:param perc_data_to_use: amount of available training
data to use.
:type perc_data_to_use: {None | int | float}
:return: a data loader that serves batches of
images and their assiated labels
:rtype: CrossValidatingDataLoader
'''
data_root = self.root_train_test_data
train_dataset = SingleRootImageDataset(data_root,
sample_width=sample_width,
sample_height=sample_height,
percentage=perc_data_to_use,
to_grayscale=True)
sampler = SKFSampler(train_dataset,
num_folds=self.num_folds,
seed=42,
shuffle=True,
drop_last=True)
train_loader = CrossValidatingDataLoader(train_dataset,
batch_size=self.batch_size,
shuffle=True,
drop_last=True,
sampler=sampler,
num_folds=self.num_folds)
return train_loader
#------------------------------------
# initialize_model
#-------------------
def initialize_model(self):
self.model = NetUtils.get_net(self.net_name,
num_classes=self.num_classes,
pretrained=self.pretrained,
freeze=self.freeze,
to_grayscale=self.to_grayscale)
self.log.debug(
f"Before any gpu push: \n{'none--on CPU' if self.fastest_device.type == 'cpu' else torch.cuda.memory_summary()}"
)
FileUtils.to_device(self.model, 'gpu')
self.log.debug(
f"Before after model push: \n{'none--on CPU' if self.fastest_device.type == 'cpu' else torch.cuda.memory_summary()}"
)
self.opt_name = self.config.Training.get('optimizer',
'Adam') # Default
self.optimizer = self.get_optimizer(self.opt_name, self.model, self.lr)
self.loss_fn = nn.CrossEntropyLoss()
self.scheduler = optim.lr_scheduler.CosineAnnealingLR(
self.optimizer, self.min_epochs)
#------------------------------------
# find_num_classes
#-------------------
def find_num_classes(self, data_root):
'''
Expect two subdirectories under data_root:
train and validation. Underneath each are
further subdirectories whose names are the
classes:
train validation
class1 class2 class3 class1 class2 class3
imgs imgs imgs imgs imgs imgs
No error checking to confirm this structure
:param data_root: path to parent of train/validation
:type data_root: str
:return: number of unique classes as obtained
from the directory names
:rtype: int
'''
self.classes = FileUtils.find_class_names(data_root)
return len(self.classes)
#------------------------------------
# setup_tensorboard
#-------------------
def setup_tensorboard(self, logdir, raw_data_dir=True):
'''
Initialize tensorboard. To easily compare experiments,
use runs/exp1, runs/exp2, etc.
Method creates the dir if needed.
Additionally, sets self.csv_pred_writer and self.csv_label_writer
to None, or open CSV writers, depending on the value of raw_data_dir,
see create_csv_writer()
:param logdir: root for tensorboard events
:type logdir: str
'''
if not os.path.isdir(logdir):
os.makedirs(logdir)
# For storing train/val preds/labels
# for every epoch. Used to create charts
# after run is finished:
self.csv_writer = self.create_csv_writer(raw_data_dir)
# Place to store intermediate models:
self.model_archive = \
self.create_model_archive(self.config,
self.num_classes
)
# Use SummaryWriterPlus to avoid confusing
# directory creations when calling add_hparams()
# on the writer:
self.writer = SummaryWriterPlus(log_dir=logdir)
# Intermediate storage for train and val results:
self.results = ResultCollection()
self.log.info(
f"To view tensorboard charts: in shell: tensorboard --logdir {logdir}; then browser: localhost:6006"
)
#------------------------------------
# create_csv_writer
#-------------------
def create_csv_writer(self, raw_data_dir):
'''
Create a csv_writer that will fill a csv
file during training/validation as follows:
epoch train_preds train_labels val_preds val_labels
Cols after the integer 'epoch' col will each be
an array of ints:
train_preds train_lbls val_preds val_lbls
2,"[2,5,1,2,3]","[2,6,1,2,1]","[1,2]", "[1,3]"
If raw_data_dir is provided as a str, it is
taken as the directory where csv file with predictions
and labels are to be written. The dir is created if necessary.
If the arg is instead set to True, a dir 'runs_raw_results' is
created under this script's directory if it does not
exist. Then a subdirectory is created for this run,
using the hparam settings to build a file name. The dir
is created if needed. Result ex.:
<script_dir>
runs_raw_results
Run_lr_0.001_br_32
run_2021_05_ ... _lr_0.001_br_32.csv
Then file name is created, again from the run
hparam settings. If this file exists, user is asked whether
to remove or append. The inst var self.csv_writer is
initialized to:
o None if csv file exists, but is not to
be overwritten nor appended-to
o A filed descriptor for a file open for either
'write' or 'append.
:param raw_data_dir: If simply True, create dir and file names
from hparams, and create as needed. If a string, it is
assumed to be the directory where a .csv file is to be
created. If None, self.csv_writer is set to None.
:type raw_data_dir: {None | True | str|
:return: CSV writer ready for action. Set either to
write a fresh file, or append to an existing file.
Unless file exists, and user decided not to overwrite
:rtype: {None | csv.writer}
'''
# Ensure the csv file root dir exists if
# we'll do a csv dir and run-file below it:
if type(raw_data_dir) == str:
raw_data_root = raw_data_dir
else:
raw_data_root = os.path.join(self.curr_dir, 'runs_raw_results')
if not os.path.exists(raw_data_root):
os.mkdir(raw_data_root)
# Can rely on raw_data_root being defined and existing:
if raw_data_dir is None:
return None
# Create both a raw dir sub-directory and a .csv file
# for this run:
csv_subdir_name = FileUtils.construct_filename(self.config.Training,
prefix='Run',
incl_date=True)
os.makedirs(csv_subdir_name)
# Create a csv file name:
csv_file_nm = FileUtils.construct_filename(self.config.Training,
prefix='run',
suffix='.csv',
incl_date=True)
csv_path = os.path.join(raw_data_root, csv_file_nm)
# Get csv_raw_fd appropriately:
if os.path.exists(csv_path):
do_overwrite = FileUtils.user_confirm(
f"File {csv_path} exists; overwrite?", default='N')
if not do_overwrite:
do_append = FileUtils.user_confirm(f"Append instead?",
default='N')
if not do_append:
return None
else:
mode = 'a'
else:
mode = 'w'
csv_writer = CSVWriterCloseable(csv_path, mode=mode, delimiter=',')
header = [
'epoch', 'train_preds', 'train_labels', 'val_preds', 'val_labels'
]
csv_writer.writerow(header)
return csv_writer
#------------------------------------
# create_model_archive
#-------------------
def create_model_archive(self, config, num_classes):
'''
Creates facility for saving partially trained
models along the way.
:param config:
:type config:
:param num_classes:
:type num_classes:
:return: ModelArchive instance ready
for calls to save_model()
:rtype: ModelArchive
'''
model_archive = ModelArchive(config,
num_classes,
history_len=self.MODEL_ARCHIVE_SIZE,
log=self.log)
return model_archive
#------------------------------------
# close_tensorboard
#-------------------
def close_tensorboard(self):
if self.csv_writer is not None:
try:
self.csv_writer.close()
except Exception as e:
self.log.warn(f"Could not close csv file: {repr(e)}")
try:
self.writer.close()
except AttributeError:
self.log.warn(
"Method close_tensorboard() called before setup_tensorboard()?"
)
except Exception as e:
raise RuntimeError(
f"Problem closing tensorboard: {repr(e)}") from e
#------------------------------------
# get_optimizer
#-------------------
def get_optimizer(self, optimizer_name, model, lr):
optimizer_name = optimizer_name.lower()
if optimizer_name == 'adam':
optimizer = optim.Adam(model.parameters(),
lr=lr,
eps=1e-3,
amsgrad=True)
return optimizer
if optimizer_name == 'sgd':
optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.9)
return optimizer
if optimizer_name == 'rmsprop':
optimizer = optim.RMSprop(model.parameters(), lr=lr, momentum=0.9)
return optimizer
raise ValueError(f"Optimizer {optimizer_name} not supported")
#------------------------------------
# initialize_config_struct
#-------------------
def initialize_config_struct(self, config_info):
'''
Initialize a config dict of dict with
the application's configurations. Sections
will be:
config['Paths'] -> dict[attr : val]
config['Training'] -> dict[attr : val]
config['Parallelism'] -> dict[attr : val]
The config read method will handle config_info
being None.
If config_info is a string, it is assumed either
to be a file containing the configuration, or
a JSON string that defines the config.
Else config_info is assumed to be a NeuralNetConfig.
The latter is relevant only if using this file
as a library, rather than a command line tool.
If given a NeuralNetConfig instance, it is returned
unchanged.
:param config_info: the information needed to construct
the structure
:type config_info: {NeuralNetConfig | str}
:return a NeuralNetConfig instance with all parms
initialized
:rtype NeuralNetConfig
'''
if isinstance(config_info, str):
# Is it a JSON str? Should have a better test!
if config_info.startswith('{'):
# JSON String:
config = NeuralNetConfig.from_json(config_info)
else:
config = self.read_configuration(config_info)
elif isinstance(config_info, NeuralNetConfig):
config = config_info
else:
msg = f"Error: must have a config file, not {config_info}. See config.cfg.Example in project root"
# Since logdir may be in config, need to use print here:
print(msg)
raise ConfigError(msg)
return config
#------------------------------------
# read_configuration
#-------------------
def read_configuration(self, conf_file):
'''
Parses config file that describes training parameters,
various file paths, and how many GPUs different machines have.
Syntax follows Python's configfile package, which includes
sections, and attr/val pairs in each section.
Expected sections:
o Paths: various file paths for the application
o Training: holds batch sizes, number of epochs, etc.
o Parallelism: holds number of GPUs on different machines
For Parallelism, expect entries like:
foo.bar.com = 4
127.0.0.1 = 5
localhost = 3
172.12.145.1 = 6
Method identifies which of the entries is
'localhost' by comparing against local hostname.
Though 'localhost' or '127.0.0.1' may be provided.
Returns a dict of dicts:
config[section-names][attr-names-within-section]
Types of standard entries, such as epochs, batch_size,
etc. are coerced, so that, e.g. config['Training']['epochs']
will be an int. Clients may add non-standard entries.
For those the client must convert values from string
(the type in which values are stored by default) to the
required type. This can be done the usual way: int(...),
or using one of the configparser's retrieval methods
getboolean(), getint(), and getfloat():
config['Training'].getfloat('learning_rate')
:param other_gpu_config_file: path to configuration file
:type other_gpu_config_file: str
:return: a dict of dicts mirroring the config file sections/entries
:rtype: dict[dict]
:raises ValueErr
:raises TypeError
'''
if conf_file is None:
return self.init_defaults()
config = DottableConfigParser(conf_file)
if len(config.sections()) == 0:
# Config file exists, but empty:
return (self.init_defaults(config))
# Do type conversion also in other entries that
# are standard:
types = {
'epochs': int,
'batch_size': int,
'kernel_size': int,
'sample_width': int,
'sample_height': int,
'seed': int,
'pytorch_comm_port': int,
'num_pretrained_layers': int,
'root_train_test_data': str,
'net_name': str,
}
for section in config.sections():
for attr_name in config[section].keys():
try:
str_val = config[section][attr_name]
required_type = types[attr_name]
config[section][attr_name] = required_type(str_val)
except KeyError:
# Current attribute is not standard;
# users of the corresponding value need
# to do their own type conversion when
# accessing this configuration entry:
continue
except TypeError:
raise ValueError(
f"Config file error: {section}.{attr_name} should be convertible to {required_type}"
)
return config
#------------------------------------
# set_seed
#-------------------
def set_seed(self, seed):
'''
Set the seed across all different necessary platforms
to allow for comparison of different models and runs
:param seed: random seed to set for all random num generators
:type seed: int
'''
torch.manual_seed(seed)
cuda.manual_seed_all(seed)
# Not totally sure what these two do!
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
np.random.seed(seed)
os.environ['PYTHONHASHSEED'] = str(seed)
random.seed(seed)
#------------------------------------
# time_delta_str
#-------------------
def time_delta_str(self, epoch_delta, granularity=2):
'''
Takes the difference between two datetime times:
start_time = datetime.datetime.now()
<some time elapses>
end_time = datetime.datetime.now()
delta = end_time - start_time
time_delta_str(delta
Depending on granularity, returns a string like:
Granularity:
1 '160.0 weeks'
2 '160.0 weeks, 4.0 days'
3 '160.0 weeks, 4.0 days, 6.0 hours'
4 '160.0 weeks, 4.0 days, 6.0 hours, 42.0 minutes'
5 '160.0 weeks, 4.0 days, 6.0 hours, 42.0 minutes, 13.0 seconds'
For smaller time deltas, such as 10 seconds,
does not include leading zero times. For
any granularity:
'10.0 seconds'
If duration is less than second, returns '< 1sec>'
:param epoch_delta:
:type epoch_delta:
:param granularity:
:type granularity:
'''
intervals = (
('weeks', 604800), # 60 * 60 * 24 * 7
('days', 86400), # 60 * 60 * 24
('hours', 3600), # 60 * 60
('minutes', 60),
('seconds', 1),
)
secs = epoch_delta.total_seconds()
result = []
for name, count in intervals:
value = secs // count
if value:
secs -= value * count
if value == 1:
name = name.rstrip('s')
result.append("{} {}".format(value, name))
dur_str = ', '.join(result[:granularity])
if len(dur_str) == 0:
dur_str = '< 1sec>'
return dur_str
#------------------------------------
# step_number
#-------------------
def step_number(self, epoch, split_num, num_folds):
'''
Combines an epoch with a split number into
a single integer series as epochs increase,
and split_num cycles from 0 to num_folds.
:param epoch: epoch to encode
:type epoch: int
:param split_num: split number to encode
:type split_num: int
:param num_folds: number of folds for CV splitting
must be contant!
:type num_folds: int
:return: an integer the combines epoch and split-num
:rtype: int
'''
step_num = epoch * num_folds + split_num
return step_num
#------------------------------------
# cleanup
#-------------------
def cleanup(self):
'''
Recover resources taken by collaborating
processes. OK to call multiple times.
'''
# self.clear_gpu()
try:
self.writer.close()
except Exception as e:
self.log.err(f"Could not close tensorboard writer: {repr(e)}")
def __init__(self,
dataset,
batch_size=32,
shuffle=True,
seed=42,
num_workers=0,
pin_memory=False,
prefetch_factor=2,
drop_last=True,
num_folds=10,
sampler=None,
logger=None):
'''
This instance will use cross validation
as it serves out samples. The client determines
the number of folds to use. Example for
num_folds of 2:
Split1:
TrainFold1 TrainFold2 ValidationFold
sample1 sample2 sample3
sample4 sample5 sample6
Split2:
TrainFold1 TrainFold2 ValidationFold
sample3 sample4 sample2
sample1 sample6 sample5
This dataloader will create two sequences,
like this:
For use with training: [sample1, sample4, sample2, sample5]
For use with validation: [sample4, sample6]
after the training
sequence is used up
Assuming batch_size of two, this dataloader's
client will receive one row from each
call to next():
[[sample1, sample4],
[sample2, sample5],
[None , None]
]
The None tuple indicates that this split has
been exhausted, and it is time to validate.
The client then calls validation_samples() on
this dataloader instance to receive one validation
sample at a time. The client will predict the
(target) class for each of these validation samples,
and tally successes and failures. The client should
then compute the compute validation accuracy from
that series of successes and failures.
Calling next() again will create a new split,
and again feed out the samples in the respective
new folds.
The feed terminates after as many splits as there
are folds. Any following call to next() will raise
a StopIteration exception.
:param dataset: underlying map-store that
supplies(img_torch, label) tuples
:type dataset: BirdDataset
:param batch_size: number of samples to combine into
a batch to feed model during training
:type batch_size: int
:param shuffle: whether or not to shuffle the
dataset once, initially.
:type shuffle: bool
:param seed: random seed to use if shuffle is True
:type shuffle: int
:param num_workers: number of threads used to preload
:type num_workers: int
:param pin_memory: set to True if using a GPU. Speeds
transfer of tensors from CPU to GPU
:type pin_memory: bool
:param prefetch_factor: how many samples to prefetch from
underlying database to speed access to file system
:type prefetch_factor: int
:param drop_last: whether or not to serve only partially
filled batches. Those occur when samples cannot be
evenly packed into batches.
:type drop_last: bool
:param num_folds: the 'k' in k-fold cross validation
:type num_folds: int
:param sampler: Only used when MultiprocessingDataLoader
is being instantiated, and that class's __init__()
calls super(). Leave out for singleprocess/single-GPU
use
:type sampler: {None | DistributedSKFSampler}
:param logger: the LoggingService instance to use
for logging info/warnings/errors. If None, fetches
the LoggingService singleton.
:type logger: LoggingService
'''
if len(dataset) == 0:
raise ValueError("Dataset is empty, nothing to load")
self.drop_last = drop_last
if logger is None:
self.log = LoggingService()
else:
self.log = logger
# Sampler will only be set if a subclass instance
# of MultiprocessingDataLoader is being initialized.
# Else, running single process:
if sampler is None:
self.sampler = SKFSampler(dataset,
num_folds=num_folds,
shuffle=shuffle,
drop_last=drop_last,
seed=seed)
else:
self.sampler = sampler
if not isinstance(batch_size, int) or batch_size <= 0:
msg = f"Batch size must be a positive int, not "
# Complete the error msg according which of
# the two failure conditions occurred:
msg += type(batch_size).__name__ if not isinstance(batch_size, int)\
else f"{batch_size}"
raise ValueError(msg)
self.batch_size = batch_size
self.num_folds = num_folds
# Total num of batches served when
# rotating through all folds is computed
# the first time __len__() is called:
self.num_batches = None
self.curr_split_idx = -1
super().__init__(dataset,
batch_size=batch_size,
sampler=self.sampler,
num_workers=num_workers,
pin_memory=pin_memory,
prefetch_factor=prefetch_factor,
drop_last=drop_last)
class TensorBoardPlotter:
'''
Support functionality for creating custom
graphs and images for submission to Tensorboard.
Services include:
o Create confusion matrix images
o Bar charts for number of samples in each class
o Placing a grid of images on Tensorboard
o Writing (i.e. overlaying) text onto images
No SummaryWriter is created. A writer is always
passed in
'''
DISPLAY_HISTORY_LEN = 8
log = LoggingService()
#------------------------------------
# collection_to_tensorboard
#-------------------
@classmethod
def collection_to_tensorboard(cls, tally_coll, writer, phases, step):
'''
Reports standard results from all tallies
in the given collection to a tensorboard.
Included are:
o Various charts
o Result text tables
o hparams
:param tally_coll:
:type tally_coll:
'''
cls.visualize_step(tally_coll, writer, phases, step)
#------------------------------------
# visualize_step
#-------------------
@classmethod
def visualize_step(cls, tally_coll, writer, phases, step, class_names):
'''
Take the ResultTally instances from
the same step from the tally_coll,
and report appropriate aggregates to
tensorboard.
Computes f1 scores, accuracies, etc. for
given step.
Separately for train and validation
results: build one long array
of predictions, and a corresponding
array of labels. Also, average the
loss across all instances.
The the preds and labels as rows to csv
files.
:return: a ResultTally instance with all
metrics computed for display
:rtype: ResultTally
'''
try:
tallies = {
str(phase): tally_coll[(step, phase)]
for phase in phases
}
except KeyError as e:
cls.log.err(f"Step: {step}, phases: {phases}: {repr(e)}")
return
for phase in phases:
# Need learning phase in string forms
# below:
phase_str = str(phase)
tally = tallies[phase_str]
writer.add_scalar(f"loss/{phase_str}",
tally.mean_loss,
global_step=step)
writer.add_scalar(f"balanced_accuracy_score/{phase_str}",
tally.balanced_acc,
global_step=step)
writer.add_scalar(f"accuracy_score/{phase_str}",
tally.accuracy,
global_step=step)
# The following are only sent to the
# tensorboard for validation and test
# phases.
if phase in (LearningPhase.VALIDATING, LearningPhase.TESTING):
# Submit the confusion matrix image
# to the tensorboard. In the following:
# do not provide a separate title, such as
# title=f"Confusion Matrix (Validation): Step{step}"
# That would put each matrix into its own slot
# on tensorboard, rather than having a time slider
TensorBoardPlotter.conf_matrix_to_tensorboard(
writer,
tally.conf_matrix,
class_names,
step=step,
title=f"Confusion Matrix Series")
# Versions of the f1 score:
writer.add_scalar(f"{phase_str}_f1/macro",
tally.f1_macro,
global_step=step)
writer.add_scalar(f"{phase_str}_f1/micro",
tally.f1_micro,
global_step=step)
writer.add_scalar(f"{phase_str}_f1/weighted",
tally.f1_weighted,
global_step=step)
# Versions of precision/recall:
writer.add_scalar(f"{phase_str}_prec/macro",
tally.prec_macro,
global_step=step)
writer.add_scalar(f"{phase_str}_prec/micro",
tally.prec_micro,
global_step=step)
writer.add_scalar(f"{phase_str}_prec/weighted",
tally.prec_weighted,
global_step=step)
writer.add_scalar(f"{phase_str}_recall/macro",
tally.recall_macro,
global_step=step)
writer.add_scalar(f"{phase_str}_recall/micro",
tally.recall_micro,
global_step=step)
writer.add_scalar(f"{phase_str}_recall/weighted",
tally.recall_weighted,
global_step=step)
return tally
#------------------------------------
# conf_matrix_to_tensorboard
#-------------------
@classmethod
def conf_matrix_to_tensorboard(cls,
writer,
conf_matrix,
step=0,
title='Confusion Matrix'):
'''
Add confusion matrix to tensorboard as an
image. Multiple conf matrices (from multiple steps)
maybe be overlaid. Tensorboard will add a slider to
run through them.
:param writer: tensorboard writer
:type writer: tensorboard.SummaryWriter
:param conf_matrix: confusion matrix to draw as heatmap
:type conf_matrix: pd.DataFrame
:param step: the step number that generated the matrix
:type step: int
:param title: title to add at the image
:type title: str
'''
conf_matrix_fig = Charter.fig_from_conf_matrix(conf_matrix,
supertitle=title)
writer.add_figure(title, conf_matrix_fig, global_step=step)
#------------------------------------
# class_support_to_tensorboard
#-------------------
@classmethod
def class_support_to_tensorboard(cls,
data_src,
writer,
step=0,
title='Class Support'):
'''
Create a barchart showing number of training samples
in each class. The chart is converted to
a tensor, and submitted to tensorboard.
The data_src may be:
o a dataset in the pytorch sense, or
o a full path the root of a training data directory, or
If custom_data is None, a barchart with number of samples
in each class is created. Else custom_data is expected
to be a dict mapping class-id => num-samples in that class.
If provided, this data is bar-charted instead of the
entire dataset's distribution
:param data_src: either a path to samples,
or a dataset
:type data_src: {str | {int : int} | torch.utils.data.Dataset}
:param writer: a tensorboard summary writer
:type writer: tensorboard.SummaryWriter
:param step: step for which support is shown
:type step: int
:param custom_data: an optional dict {class-id : sample-count} whose
per-class count is to be bar-charted instead of the entire
dataset
:type custom_data: {int : int}
:param title: optional title above the figure
:type title: str
:return: dict {<class_name> : <num_samples_for_class_name>}
i.e. number of samples in each class.
:rtype: {str : int}
'''
if type(data_src) == str:
# Data source is file path to
# root of training data. Create
# a dataset from that tree:
dataset = SingleRootImageDataset(data_src)
elif type(data_src) != SingleRootImageDataset:
raise ValueError(
f"Data source must be path to data root, or a dataset, not {data_src}"
)
else:
dataset = data_src
# Get dict: {<class_id> : <class_name>}
class_id_to_name = {
class_id: class_name
for class_name, class_id in dataset.class_to_id.items()
}
# Goal is corresponding np arrays:
# class-name, num-samples-in-class.
# First, get correponding tuples of
# class *ids* and sample counts. The
# 'zip(*<list-of-tuples>) notation is
# the inverse of a zip():
# take [(c1,n1), (c2,n2),...] that is returned
# from sample_distribution(), and create two
# arrays: [c1,c2,...], and [n1,n2,...]
[class_id_tuple,
sample_count_tuple] = zip(*dataset.sample_distribution())
# Create np array of class *names* from the class ID tuple:
class_names = np.array(
[class_id_to_name[class_id] for class_id in class_id_tuple])
sample_counts = np.array(sample_count_tuple)
# Make a horizontal chart, so class names are
# Y-axis labels:
y_pos = np.arange(len(class_names))
fig, ax = plt.subplots()
fig.suptitle('Number of Samples in Each Class')
_bar_container = ax.barh(
y_pos,
sample_counts, # Bar length (i.e. width)
tick_label=class_names,
align='center')
ax.set_xlabel('Number of Samples')
# Convert matplotlib figure into
# an image tensor for tensorboard:
writer.add_figure(title, fig, step)
support_dict = {
class_name: num_samples
for class_name, num_samples in zip(class_names, sample_counts)
}
return support_dict
#------------------------------------
# add_image
#-------------------
@classmethod
def add_image(
cls,
writer,
tag,
img_path,
step=0,
to_grayscale=True,
img_height=200, # px
img_width=400 # px
):
'''
Writes a single image to tensorboard.
Can resize image or turn to grayscale
if requested. If img_width or img_height
is None, no scaling is done.
:param writer: the SummaryWriter to use
:type writer: SummaryWriter
:param tag: the name of the image in
tensorboard display
:type tag: str
:param img_path: full path to image
:type img_path: str
:param step: step
:type step: int
:param to_grayscale: whether or not to conver
to grayscale
:type to_grayscale: bool
:param img_height: desired image height
:type img_height: int
:param img_width: desired image width
:type img_width: int
'''
the_transforms = []
if img_height is not None and img_width is not None:
the_transforms.append(transforms.Resize((img_height, img_width)))
if to_grayscale:
the_transforms.append(transforms.Grayscale())
the_transforms.append(transforms.ToTensor())
transform_img = transforms.Compose(the_transforms)
img = Image.open(img_path)
img = transform_img(img).float()
# A 10px frame around each img:
#grid = make_grid(img, padding=10)
#writer.add_image(tag, grid, step)
writer.add_image(tag, img, step)
#------------------------------------
# write_img_grid
#-------------------
@classmethod
def write_img_grid(
cls,
writer,
img_root_dir,
num_imgs=4,
class_sample_file_pairs=None,
img_height=200, # px
img_width=400, # px
to_grayscale=True,
unittesting=False):
'''
Create and log a Tensorboard 'grid' of
example train images. The img_root_dir must
be the 'data root': the dir holding one subdir
per class.
:param writer: a Tensorboard Pytorch SummaryWriter
:type writer: SummaryWriter
:param img_root_dir: directory
that contains sub-directories with samples. The
sub-directory names are taken to be class names.
:type img_root_dir: str
:param num_imgs: total number of images to
include in the grid. If None: all images
:type num_imgs: {None | int}
:param class_sample_file_pairs: <class>/<img_file_name> for
individual images if random choice is not wanted.
:type class_sample_file_pairs: {None | str | [str]}
:param img_height: height of all images
:type img_height: int (pixels)
:param img_width: width of all images
:type img_width: int (pixels)
:param to_grayscale: whether or not to convert
images to grayscale upon import
:type to_grayscale: bool
:param unittesting: controls whether grid is
actually created, or the img tensor that
would be contained in the grid is returned
for testing dimensions.
:type unittesting: bool
'''
if img_root_dir is None:
raise ValueError("Must provide path to image root dir")
# Prepare to resize all images to a given dimension,
# convert to grayscale if requested, and turn into
# a tensor:
the_transforms = [transforms.Resize((img_height, img_width))]
if to_grayscale:
the_transforms.append(transforms.Grayscale())
the_transforms.append(transforms.ToTensor())
transform_img = transforms.Compose(the_transforms)
# Get an ImageFolder instance, from which
# we will easily find classes and samples
img_folder = ImageFolder(img_root_dir,
transform=transform_img,
loader=default_loader)
# Get list of full paths to samples:
sample_idxs = cls._get_sample_indices(
img_folder,
num_imgs=num_imgs,
class_sample_file_pairs=class_sample_file_pairs)
# Get list of img tensor/class_idx pairs:
img_tns_list = [img_folder[idx] for idx in sample_idxs]
# Print <class>/file_name onto
# each spectrogram:
marked_img_tns_list = []
for i, (img_tns, class_idx) in enumerate(img_tns_list):
class_name = img_folder.classes[class_idx]
# img_folder.samples is [ (full_path, class_idx), (..., ...) ]:
img_file_basename = os.path.basename(img_folder.samples[i][0])
marked_img_tns_list.append(
cls.print_onto_image(img_tns,
f"{class_name}/{img_file_basename}"))
# Turn list of img tensors into
# a single tensor with first dim
# being len of list:
marked_img_tns = torch.cat(marked_img_tns_list)
# A 10px frame around each img:
grid = make_grid(marked_img_tns, padding=10)
if unittesting:
return grid
writer.add_image('Train Input Examples', grid)
return grid
#------------------------------------
# make_f1_train_val_table
#-------------------
@classmethod
def make_f1_train_val_table(cls, res_list):
'''
Return a github flavored table:
|phase|ep0 |ep1 |ep2 |
|-----|-----|----|----|
|train| f1_0|f1_1|f1_2|
| val| f1_0|f1_1|f1_2|
for half as many steps back as there are
tallies available in the list of ResultTally
instances in step_results.
Assumption: exactly two ResultTallies are provided
in res_list. One each for train and validation
results.
:param res_list: list of ResultTally
instances in oldest-step-first order
:type res_list: [ResultTally]
:return: a table
:rtype: str
'''
res_len = len(res_list)
# Could catch the following error.
# But it's just a special case of
# num train tallies unequal to num
# of val tallies. Wait till we catch
# that root problem later:
# Should be an even number of result
# objs:
#if res_len % 2 != 0:
# raise ValueError("Must provide two ResultTally instances per step")
num_steps = res_len // 2
# First the header:
header = []
for i in range(num_steps):
header.append(f"f1-macro ep{i}")
# The f1 value results for both
# train and val:
train_f1s = filter(
lambda res_tally: res_tally.phase == LearningPhase.TRAINING,
res_list)
val_f1s = filter(
lambda res_tally: res_tally.phase == LearningPhase.VALIDATING,
res_list)
train_row = []
for res in train_f1s:
train_row.append(str(round(res.f1_macro, 1)))
val_row = []
# Second row: f1's for validation results:
for res in val_f1s:
val_row.append(str(round(res.f1_macro, 1)))
if len(val_row) != len(train_row):
raise ValueError(
f"Must have equal num of train/val tallies; have {len(val_row)} vals and {len(train_row)} trains"
)
tbl_content = {
'col_header': header,
'row_labels': ['training', 'validation'],
'rows': [train_row, val_row]
}
tbl = GithubTableMaker.make_table(tbl_content)
return tbl
#------------------------------------
# make_all_classes_f1_table
#-------------------
@classmethod
def make_all_classes_f1_table(cls, latest_result, class_names):
'''
Return a github flavored table with
with train and val f1 values for every
class:
|class|weighted mean f1 train|weighted mean f1 val|
|-----|----------------------|--------------------|
| c1 | 0.1 | 0.6 |
| c2 | 0.1 | 0.6 |
| c3 | 0.1 | 0.6 |
---------------------------------------------------
'''
# Get the 'all-classes' version of f1 from
# the last ResultTally for both train and val:
t_f1s = latest_result['train'].f1_all_classes
v_f1s = latest_result['val'].f1_all_classes
if t_f1s is None or \
v_f1s is None or \
len(t_f1s) == 0 or\
len(t_f1s) == 0:
raise ValueError(
"Both, train and val values of f1_all_classes must be non-empty lists"
)
# Get [[c1_train, c1_val],
# [c2_train, c2_val],
# ...
# ]
res = torch.tensor([t_f1s, v_f1s]).T
header = ['weighted mean f1 train', 'weighted mean f1 val']
# And the f1 train/val numbers, one
# class in each row:
row_labels = []
rows = []
for class_name, (f1_train, f1_val) in zip(class_names, res):
f1_train = round(float(f1_train), 1)
f1_val = round(float(f1_val), 1)
row_labels.append(class_name)
rows.append([f1_train, f1_val])
tbl_content = {
'col_header': header,
'row_labels': row_labels,
'rows': rows
}
tbl = GithubTableMaker.make_table(tbl_content)
return tbl
#------------------------------------
# print_onto_image
#-------------------
@classmethod
def print_onto_image(cls, img_src, txt, point=(10, 10)):
'''
Given an image, writes given text onto the image.
Returns a tensor of the new image. Acceptable as image
sources are:
o File path to jpg, png, etc.
o A tensor
o A PIL image
:param img_src: image, or a way to get the image
:type img_src: {str | Tensor | PIL}
:param txt: text to be printed onto the image
:type txt: str
:param point: where to place the text. In pixels,
origin upper left
:type point: [int,int]
:return: new image with text 'burned' onto it
:rtype: Tensor
'''
if type(img_src) == str:
# Image is a path:
try:
pil_img = Image.open(img_src)
except Exception as e:
raise ValueError(
f"Could not load img from {img_src}: {repr(e)}")
elif type(img_src) == torch.Tensor:
try:
pil_img = transforms.ToPILImage()(img_src.squeeze_(0))
except Exception as e:
raise ValueError(
f"Could not convert tensor to PIL img ({img_src.size()})")
elif not Image.isImageType(img_src):
raise ValueError(
f"Image src must be path to img, tensor, or PIL image; not {type(img_src)}"
)
else:
pil_img = img_src
# Make a blank image for the text.
# Match the mode (RGB/RGBA/L/...):
txt_img = Image.new(pil_img.mode, pil_img.size, 255)
# get a font
fnt = ImageFont.load_default()
# get a drawing context
drawing = ImageDraw.Draw(txt_img)
# Draw text, half opacity
drawing.text(point, txt, font=fnt) #******, fill=(0,0,0,128))
# Draw text, full opacity
#drawing.text(point, txt, font=fnt, fill=(255,255,255,255))
#*****out_img = Image.alpha_composite(pil_img, txt_img)
out_img = Image.blend(pil_img, txt_img, 0.5)
out_tns = transforms.ToTensor()(out_img).unsqueeze_(0)
#out_img.show()
out_img.close()
return out_tns
#------------------------------------
# _get_sample_indices
#-------------------
@classmethod
def _get_sample_indices(cls,
img_folder,
class_sample_file_pairs,
num_imgs=None):
'''
If class_sample_file_pairs is provided,
then num_imgs is ignored.
:param img_folder: folder instance with training images
:type img_folder: ImageFolder
:param class_sample_file_pairs: optionally, pairs of
class-name and path to training images
:type class_sample_file_pairs: [(<class-name>, <sample-file-name>)]
:param num_imgs: for how many images to create spectrograms
:type num_imgs: int
:return: a list of sample IDs
:rtype: int
'''
# Caller requests particular images?
if class_sample_file_pairs is not None:
# Convert the (<class-name>,<sample-file_name>)
# pairs to (<class_idx>,<sample-file-name>)
requested_class_idx_sample_pairs = [
(img_folder.class_to_idx[class_name], sample_file_nm)
for class_name, sample_file_nm in class_sample_file_pairs
]
# Make a more convenient dict
# {class-idx : [<sample-file-name>]
requests = {}
for class_idx, sample_path in requested_class_idx_sample_pairs:
try:
requests[class_idx].append(sample_path)
except KeyError:
# First sample file for this class:
requests[class_idx] = [sample_path]
found_idxs = []
for i, (sample_path, class_idx) in enumerate(img_folder.samples):
try:
if os.path.basename(sample_path) in requests[class_idx]:
found_idxs.append(i)
except KeyError:
# Not one of the requested samples:
continue
return found_idxs
# We are asked to randomly pick images
# from each class:
num_samples = len(img_folder)
num_classes = len(img_folder.classes)
num_samples_to_get = num_samples \
if num_imgs is None \
else min(num_samples, num_imgs)
# Create a dict {class-idx : <list of indices into img_folder>}
# I.e. for each class, list the int indices i
# such that img_folder[i] is an img in the class.
#
class_dict = {}
for i, (sample_path, class_idx) in enumerate(img_folder.samples):
try:
class_dict[class_idx].append(i)
except KeyError:
# First sample of this class:
class_dict[class_idx] = [i]
# Rough number of images to get per class:
num_imgs_per_class = round(num_samples_to_get / num_classes)
_remaining_imgs = num_samples_to_get % num_classes
to_get_idxs = []
for class_idx, sample_idx_list in class_dict.items():
# Get as many random picks from round's classs
# sample IDs as we want samples per class:
# Do we have fewer samples in this class than
# we want from each class?
if len(sample_idx_list) < num_imgs_per_class:
# Yes: grab them all:
to_get_idxs.extend(sample_idx_list)
else:
sample_idxs = random.sample(sample_idx_list,
num_imgs_per_class)
to_get_idxs.extend(sample_idxs)
return to_get_idxs
class TrainScriptRunner(object):
'''
classdocs
'''
#------------------------------------
# Constructor
#-------------------
def __init__(self,
starting_config_src,
hparms_spec,
training_script=None,
logfile=None,
quiet=False,
dryrun=False,
unittesting=False):
'''
Specifications expected like this
*Ordered* dict (i.e. sequence of
keys and values always the same for
keys()/values()/items() methods:
{<hparm1> : [val1_1, val1_2, ...],
<hparm2> : [val2_1, val2_2, ...]
}
:param starting_config_src: a configuration
whose neural net related parameters will
be modified below for each run.
:type starting_config_src: {str | NeuralNetConfig}
:param hparms_spec:
:type hparms_spec:
:param training_script: path to the training script
of which to run multiple copies. If None, will
look in config for Path:train_script.
:type training_script: {None | str}
:param logfile: where to log runtime information. If None,
log to console
:type logfile: {None | str}
:param quiet: whether or not to report progress
:type quiet: bool
:param unittesting: set to True if unittesting so that
__init__() will only do a minimum, and allows unittests
to call other methods individually
:type bool
'''
if logfile is not None:
self.log = LoggingService(logfile=logfile)
else:
self.log = LoggingService()
self.quiet = quiet
self.curr_dir = os.path.dirname(__file__)
self.hostname = socket.getfqdn()
# No GPUs identified so far:
self.WORLD_SIZE = 0
starting_config = NeuralNetConfig(starting_config_src)
if unittesting:
# Leave calling of the methods below
# to the unittests
return
self.training_script = training_script
if training_script is None:
# Try to find it in config:
try:
self.training_script = starting_config.getpath(
'Paths', 'train_script', relative_to=self.curr_dir)
except KeyError:
raise ValueError(
"Did not provide training script path on cmd line or in config"
)
self.gpu_landscape = self.obtain_world_map(starting_config)
# Get list of dicts of hparm-name/hparm_value pairs;
# one for each of the runs
the_run_dicts = self.get_runs_hparm_specs(hparms_spec)
# Turn the run dicts into configurations
# that that modify the starting config:
the_run_configs = self.gen_configurations(starting_config,
the_run_dicts)
if dryrun:
print("Dryrun:")
print(
f"Would run {len(the_run_dicts)} processes with these configs:"
)
for configs in the_run_dicts:
print(configs)
return
# Provide support for cnt-c terminating the training
# script processes nicely:
self.cnt_c_received = False
signal.signal(signal.SIGTERM, self.handle_cnt_c)
# Start one training script for each configuration:
self.run_configurations(the_run_configs)
#------------------------------------
# get_runs_hparm_specs
#-------------------
def get_runs_hparm_specs(self, hparms_spec):
'''
Create a list of dicts. Each dict
holds the value for each of the hparms
for one run.
:param hparms_spec: client's dict of
{param_name : [val1, val2, ...]}
:type hparms_spec: {str : [Any]}
:return: list of dicts
'''
# Running example:
# {'lr' : [0.001],
# 'optimizer' : ['Adam','RMSprop','SGD'],
# 'batch_size' : [32, 64, 128],
# 'kernel_size': [3, 7]
# })
# Parameters to vary:
parm_names = list(hparms_spec.keys())
# Iterate through list of value combinations:
# (0.001, 'Adam', 32, 3)
# (0.001, 'Adam', 32, 7)
# (0.001, 'Adam', 64, 3)
# ...
# to get a list of dicts, each with a
# unique combination of parameter settings:
#
# [{'lr': 0.001,
# 'optimizer' : 'Adam',
# 'batch_size' : 32,
# 'kernel_size': 3},
# {'lr': 0.001,
# 'optimizer' : 'Adam',
# 'batch_size' : 32,
# 'kernel_size': 7},
# {...}
# ...
# ]
hparms_permutations = []
for _perm_num, ordered_vals_tuple in enumerate(
product(*hparms_spec.values())):
# Have something like:
# (0.001, 'Adam', 32, 3)
# Separate dict for each combo:
conf_dict = dict(zip(parm_names, ordered_vals_tuple))
hparms_permutations.append(conf_dict)
return hparms_permutations
#------------------------------------
# gen_configurations
#-------------------
def gen_configurations(self, config, config_dicts):
'''
Takes a list of dicts, and returns a list
of NeuralNetConfig instances. Each dict
contains one hyperparameter settings combination
that is to be tested. Such as:
[{'lr': 0.001,
'optimizer': 'Adam',
'batch_size': 32,
'kernel_size': 3},
{'lr': 0.001,
'optimizer': 'Adam',
'batch_size': 32,
'kernel_size': 7},
{...}
...
]
Each return configuration is a copy of the
config, modified for the respective
hyperparameter settings. All other parts of
the config are kept.
:param config: a configuration with
all settings; only the hyperparameter
settings will be modified
:type config: NeuralNetConfig
:param config_dicts: one dict of hyperparm-name : value
for each process to run independently
:type config_dicts: [{str : Any}]
:return: list of configurations for the classifier
script to run
:rtype: [NeuralNetConfig]
'''
configs = []
for conf_dict in config_dicts:
conf_copy = config.copy()
for param_name, val in conf_dict.items():
conf_copy.add_neural_net_parm(param_name, val)
configs.append(conf_copy)
return configs
#------------------------------------
# obtain_world_map
#-------------------
def obtain_world_map(self, initial_config):
try:
self.world_map_path = initial_config.getpath(
'Paths', 'world_map', relative_to=self.curr_dir)
except KeyError:
raise RuntimeError(
f"Could not find entry for 'world_map' in initial config")
self.world_map = self.read_world_map(self.world_map_path)
# Ensure that this machine has an
# entry in the world_map:
try:
# Get this machine's info (sub)dict:
_my_world_info = self.world_map[self.hostname]
except KeyError:
raise ConfigError(
f"World map file does not contain entry for this machine ({self.hostname})"
)
self.compute_landscape = {}
gpu_landscape = self.build_compute_landscape(self.world_map)
return gpu_landscape
#------------------------------------
# build_compute_landscape
#-------------------
def build_compute_landscape(self, world_map):
'''
# Using the world_map.json config file, build
# a dict self.gpu_landscape like this:
#
# {'machine_name1' : {'start_rank' : <int>,
# 'num_gpus' : <int>,
# 'gpu_device_ids': [<int>,<int>,...]
# {'machine_name2' : {'start_rank' : <int>,
# 'num_gpus' : <int>,
# 'gpu_device_ids': [<int>,<int>,...]
# }
#
# Also sets
# o self.master_hostname, the hostname
# running the one process that coordinates all others.
# o self.WORLD_SIZE, number of GPUs used across all machines
# o self.my_gpus, the number of GPUs on this machine
:param world_map:
:type world_map:
:return: information about how many GPUs are
on each node
:rtype: OrderedDict
'''
if not self.hostname in world_map.keys():
raise ConfigError(
f"World map does not contain an entry for this machine {self.hostname}"
)
# Go through the world map, machine (a.k.a. node)
# one at a time, in alpha order of the machine
# names to ensure all copies of this script
# come to the same conclusions about ranks
# Build gpu_landscape:
#
# {'machine_name1' : {'start_rank' : <int>,
# 'num_gpus' : <int>,
# 'gpu_device_ids': [<int>,<int>,...]
# {'machine_name2' : {'start_rank' : <int>,
# 'num_gpus' : <int>,
# 'gpu_device_ids': [<int>,<int>,...]
# }
#
# The structure is an OrderedDict(), containing
# machines alphabetically by name. This discipline
# is required so that all copies of this launch script
# (one copy per machine) arrive at the same ordering of
# GPUs:
gpu_landscape = OrderedDict({})
machine_name = self.hostname
machine_info = world_map[self.hostname]
try:
machine_gpus = machine_info['gpus']
except KeyError:
print("World map must include a 'gpus' entry; the value may be 0")
gpu_landscape[machine_name] = {}
gpu_landscape[machine_name]['num_gpus'] = machine_gpus
# List of GPU numbers to use is optional
# in world_maps:
machine_gpus_to_use = machine_info.get('devices', None)
if machine_gpus_to_use is None:
# Use all GPUs on this machine:
machine_gpus_to_use = list(range(machine_gpus))
gpu_landscape[machine_name]['gpu_device_ids'] = machine_gpus_to_use
# Add 1 process for the on this machine,
# which will run on its CPU, b/c no GPUs
# are available:
self.WORLD_SIZE += machine_gpus if machine_gpus > 0 else 1
self.my_gpus = gpu_landscape[self.hostname]['num_gpus']
self.gpu_landscape = gpu_landscape
return gpu_landscape
#------------------------------------
# read_world_map
#-------------------
def read_world_map(self, path):
'''
Read the JSON5 world map file, and
return a corresponding dict. JSON5
allows something like:
/*
This is a block comment.
Notice the lacking quote
chars around the keys below.
The are optional in JSON5
*/
{quintus.stanford.edu : {
"master" : Yes
"gpus" : 2
},
quatro.stanford.edu : {
"gpus" : 2,
"devices" : [1,2]
}
}
BUT: JSON5 gets angry at dots in the
keys.
So we first read the file, and try to find
the machine names. We temporarily replace
them with an acceptable marker, and then
convert back.
:param path: path to world map file
:type path: string
'''
dot_substitute = '___'
try:
# Read all the world map file lines:
with open(path, 'r') as world_map_fd:
tmp_world_map = world_map_fd.readlines()
except IOError as e:
raise IOError(f"World map file at {path} not found") from e
# Replace occurrences of '.' with dot_substitute:
new_text = []
for line in tmp_world_map:
new_text.append(line.replace('.', dot_substitute))
# ... and make one string from all the lines:
json_str = '\n'.join(new_text)
try:
# Hopefully, JSON5 will eat it now:
world_map_almost = json5.loads(json_str)
except JSONError as e:
raise JSONError(
f"World map file at {path} contains bad JSON") from e
# Need to fix all the dot substitutions.
# At this point the data structure is
# { <machine_name> : {spec_attr1 : val1,
# spec_attr2 : val2,
# }
# }
# Fix the machine names first:
mach_names_fixed = [
machine_name.replace(dot_substitute, '.')
for machine_name in world_map_almost.keys()
]
machine_specs_fixed = []
# Now dig into each of the nested machine spec
# dicts, and fix attrs and values there:
for spec in world_map_almost.values():
# Spec is a dict nested inside the outer one:
spec_fixed = {
key.replace(dot_substitute, '.'): val.replace(
dot_substitute, '.') if isinstance(val, str) else val
for key, val in spec.items()
}
machine_specs_fixed.append(spec_fixed)
# Put it all together:
world_map = {
machine_name: spec_dict
for machine_name, spec_dict in zip(mach_names_fixed,
machine_specs_fixed)
}
return world_map
#------------------------------------
# run_configurations
#-------------------
def run_configurations(self, run_configs):
'''
Takes a list of run configuration that
specify the details of a training run
(lr, optimizer to use, etc.) Spawns
independent training script processes, one
with each of the configurations.
If fewer CPUs/GPUs are available than the
number of configs in run_configs, waits for
processes to finish, then launches more.
Configs may take one of three forms:
o File path to a config file
o JSON string with all the config info
o A NeuralNetConfig instance
Use world_map.json to know how many, and which
GPUs this machine is to use.
Each copy of the training script is told:
o RANK # The copy's sequence number, which is
# Unique within this machine (but not
# currently across machines, as in in
# distributed data parallel (DDP)
o LOCAL_RANK # Which of this machine's GPU to use (0-origin)
o WORLD_SIZE # How many GPUs are used on all machines together
o GPUS_USED_THIS_MACHINE # Number of GPUs *used* on this
# machine, according to the world_map.
# (As opposed to number of GPUs that
# exist on this machine.)
:param run_configs: list of configurations. Each config
may either be a JSON string, the file name of
a config file, or a NeuralNetConfig instance
:type run_configs: [str | NeuralNetConfig]
:return 0 for success of all processes, else 1
:rtype int
'''
gpu_ids_to_use = self.gpu_landscape[self.hostname]['gpu_device_ids']
cpu_only = len(gpu_ids_to_use) == 0
self.gpu_manager = GPUManager(gpu_ids_to_use)
for config in run_configs:
# Get next available GPU ID, waiting
# for one to free up, if necessary:
local_rank = self.gpu_manager.obtain_gpu()
# Create a command that is fit for passing to
# Popen; it will start one training script
# process. The conditional expression accounts for
# machine with no GPU (which will run on CPU):
cmd = self.training_script_start_cmd(local_rank, config)
# Copy stdin, and give the copy to the subprocess.
# This enables the subprocess to ask user whether
# to save training state in case of a cnt-C:
newstdin = os.fdopen(os.dup(sys.stdin.fileno()))
# Spawn one training script. Use psutil's
# Popen instead of subprocess.Popen to get
# the wait_procs() method on the resulting
# process instances:
process = psutil.Popen(
cmd,
stdin=newstdin,
stdout=None, # Script inherits this launch
stderr=None # ... script's stdout/stderr
)
if cpu_only:
process.wait()
# CPU op is for debugging only;
# Rebel right away if something
# went wrong:
if process.returncode != 0:
print("CPU job ran with errors; see log")
return
continue
# Associate process instance with
# the configuration it was to run.
self.gpu_manager.process_register(
RunInfo(local_rank, process, config, cmd))
# Launched all configurations; wait for
# the last of them to be done:
if cpu_only:
print("CPU job(s) ran OK")
return
# Ask for GPUs until we accounted
# for all that we were allowed to
# use; that will be indication that
# all processes finished:
for _i in len(gpu_ids_to_use):
self.gpu_manager.obtain_gpu()
if not self.quiet:
print(f"Node {self.hostname} {os.path.basename(sys.argv[0])}: " \
f"Processed {len(run_configs)} configurations")
failed_processes = self.gpu_manager.failures()
if len(failed_processes) > 0:
print(
f"Failures: {len(failed_processes)} (Check log for error entries):"
)
for failed_proc in failed_processes:
failed_config = self.gpu_manager.process_info(failed_proc)
train_script = self.training_script
msg = (f"Training script {train_script}: {str(failed_config)}")
print(msg)
#------------------------------------
# training_script_start_cmd
#-------------------
def training_script_start_cmd(self, local_rank, config):
'''
From provided information, creates a legal
command string for starting the training script.
:param local_rank: GPU identifier (between 0 and
num of GPUs in this machine)
:type local_rank: int
:param config: additional information in a config instance,
or a path to a configuration file
:type config: {NeuralNetConfig | str}
'''
# Build the shell command line,
# starting with 'python -u':
cmd = [sys.executable, "-u", f"{self.training_script}"]
# Add the 'secret' args that tell the training
# script all the communication parameters:
cmd.extend([
f"--LOCAL_RANK={local_rank}",
f"--WORLD_SIZE={self.WORLD_SIZE}",
])
# Finally, the obligatory non-option arg
# to the training script: the configuration.
# Could be a file, a json string, or a
# NeuralNetConfig instance:
if isinstance(config, NeuralNetConfig):
# Turn into a JSON str for communicating
# to the script:
config_arg = config.to_json()
self.log.info(f"\nLAUNCHING TRAINING: " +\
f"{NeuralNetConfig.json_human_readable(config_arg)}")
else:
config_arg = config
self.log.info(f"\nLAUNCHING TRAINING from file: {config_arg}")
cmd.append(config_arg)
#self.log.debug(f"****** Launch: the cmd is {cmd}")
return cmd
# ------------------- Utils --------------
#------------------------------------
# handle_cnt_c
#-------------------
def handle_cnt_c(self):
'''
Given a list of process instances,
Send SIGINT (cnt-C) to them:
:param procs:
:type procs:
'''
if self.cnt_c_received:
# Just quit after a second
# cnt-c:
print(
f"Hard quit. May wish to check for stray {self.training_script} processes"
)
sys.exit(1)
self.cnt_c_received = True
for process in self.gpu_manager.process_list():
# If process is no longer running,
# forget about it:
if process.poll is not None:
# Process dead:
continue
process.send_signal(signal.SIGTERM)
process.wait()
#------------------------------------
# am_master_node
#-------------------
def am_master_node(self):
'''
This method allows this script to stay somewhat
close to the Distributed Data Parallel sibling
launch_birds_parallel(). For this script,
though, every process is its own master.
'''
return True
#------------------------------------
# is_json_str
#-------------------
def is_json_str(self, str_to_check):
'''
Very primitive test whether a passed-in
string is (legal) JSON or not.
:param str_to_check: string to examine
:type str_to_check: str
:return True/False
:rtype bool
'''
try:
json5.loads(str_to_check)
except JSONError:
return False
return True
