def materialize_model(self, model_path, gpu_to_use=0):
model_fname = os.path.basename(model_path)
# Extract model properties
# from the model filename:
self.model_props = FileUtils.parse_filename(model_fname)
model = NetUtils.get_net(
self.model_props['net_name'],
num_classes=self.model_props['num_classes'],
pretrained=False,
freeze=0,
to_grayscale=self.model_props['to_grayscale']
)
try:
if torch.cuda.is_available():
self.model.load_state_dict(torch.load(self.model_path))
FileUtils.to_device(model, 'gpu', gpu_to_use)
else:
self.model.load_state_dict(torch.load(
model_path,
map_location=torch.device('cpu')
))
except RuntimeError as e:
emsg = repr(e)
if emsg.find("size mismatch for conv1") > -1:
emsg += " Maybe model was trained with to_grayscale=False, but local net created for grayscale?"
raise RuntimeError(emsg) from e
return model
def test_densenet(self):
num_pretrained_layers = 6
#trainer = BirdTrainer(self.tst_config)
model = NetUtils.get_net(
'densenet161',
num_classes=self.num_classes, # num_classes
num_layers_to_retain=num_pretrained_layers,
to_grayscale=True)
self.assertEqual(model.state_dict()['features.conv0.weight'].shape,
torch.Size([96, 1, 7, 7]))
self.assertEqual(model.classifier.out_features, 4)
def test_resnet(self):
num_pretrained_layers = 6
model = NetUtils.get_net(
'resnet18',
num_classes=self.num_classes, # num_classes
num_layers_to_retain=num_pretrained_layers,
to_grayscale=True)
self.assertEqual(model.state_dict()['conv1.weight'].shape,
torch.Size([64, 1, 7, 7]))
self.assertEqual(model.fc.out_features, 4)
def _instantiate_model(self, run_path_str=None, config=None):
'''
Returns a model based on information in
the config structure, or the info encoded
in the run_path_str file name.
One of run_path_str or config must be non-None.
If both are non-None, uses config.
File paths that encode run parameters look like
this horror:
model_2021-03-11T10_59_02_net_resnet18_pretrain_0_lr_0.01_opt_SGD_bs_64_ks_7_folds_0_gray_True_classes_10.pth
:param run_path_str: a path name associated with
a model.
:type run_path_str:
:param config: run configuration structure
:type config: NeuralNetConfig
:return: a model
:rtype: torch.nn.module
'''
if config is None:
# Get a dict with info
# in a standard (horrible) file name:
fname_props = FileUtils.parse_filename(run_path_str)
else:
fname_props = config.Training
data_root = config.Paths.root_train_test_data
class_names = FileUtils.find_class_names(data_root)
fname_props['classes'] = len(class_names)
fname_props['pretrain'] = config.Training.getint('freeze', 0)
model = NetUtils.get_net(net_name=fname_props['net_name'],
num_classes=fname_props['classes'],
freeze=fname_props['pretrain'],
to_grayscale=fname_props['to_grayscale']
)
return model
def initialize_model(self):
self.model = NetUtils.get_net(self.net_name,
num_classes=self.num_classes,
pretrained=self.pretrained,
freeze=self.freeze,
to_grayscale=self.to_grayscale)
self.log.debug(
f"Before any gpu push: \n{'none--on CPU' if self.fastest_device.type == 'cpu' else torch.cuda.memory_summary()}"
)
FileUtils.to_device(self.model, 'gpu')
self.log.debug(
f"Before after model push: \n{'none--on CPU' if self.fastest_device.type == 'cpu' else torch.cuda.memory_summary()}"
)
self.opt_name = self.config.Training.get('optimizer',
'Adam') # Default
self.optimizer = self.get_optimizer(self.opt_name, self.model, self.lr)
self.loss_fn = nn.CrossEntropyLoss()
self.scheduler = optim.lr_scheduler.CosineAnnealingLR(
self.optimizer, self.min_epochs)
def __init__(self, config_info, debugging=False):
'''
Constructor
'''
self.log = LoggingService()
if debugging:
self.log.logging_level = DEBUG
self.curr_dir = os.path.dirname(os.path.abspath(__file__))
try:
self.config = self.initialize_config_struct(config_info)
except Exception as e:
msg = f"During config init: {repr(e)}"
self.log.err(msg)
raise RuntimeError(msg) from e
try:
self.root_train_test_data = self.config.getpath(
'Paths', 'root_train_test_data', relative_to=self.curr_dir)
except ValueError as e:
raise ValueError(
"Config file must contain an entry 'root_train_test_data' in section 'Paths'"
) from e
self.batch_size = self.config.getint('Training', 'batch_size')
self.kernel_size = self.config.getint('Training', 'kernel_size')
self.min_epochs = self.config.Training.getint('min_epochs')
self.max_epochs = self.config.Training.getint('max_epochs')
self.lr = self.config.Training.getfloat('lr')
self.net_name = self.config.Training.net_name
self.pretrained = self.config.Training.getboolean('pretrained', False)
self.freeze = self.config.Training.getint('freeze', 0)
self.to_grayscale = self.config.Training.getboolean(
'to_grayscale', True)
self.set_seed(42)
self.log.info("Parameter summary:")
self.log.info(f"network {self.net_name}")
self.log.info(f"pretrained {self.pretrained}")
if self.pretrained:
self.log.info(f"freeze {self.freeze}")
self.log.info(f"min epochs {self.min_epochs}")
self.log.info(f"max epochs {self.max_epochs}")
self.log.info(f"batch_size {self.batch_size}")
self.fastest_device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
self.num_classes = self.find_num_classes(self.root_train_test_data)
self.model = NetUtils.get_net(self.net_name,
num_classes=self.num_classes,
pretrained=self.pretrained,
freeze=self.freeze,
to_grayscale=self.to_grayscale)
self.log.debug(
f"Before any gpu push: \n{'none--on CPU' if self.fastest_device.type == 'cpu' else torch.cuda.memory_summary()}"
)
FileUtils.to_device(self.model, 'gpu')
self.log.debug(
f"Before after model push: \n{'none--on CPU' if self.fastest_device.type == 'cpu' else torch.cuda.memory_summary()}"
)
# No cross validation:
self.folds = 0
self.opt_name = self.config.Training.get('optimizer',
'Adam') # Default
self.optimizer = self.get_optimizer(self.opt_name, self.model, self.lr)
self.loss_fn = nn.CrossEntropyLoss()
self.scheduler = optim.lr_scheduler.CosineAnnealingLR(
self.optimizer, self.min_epochs)
sample_width = self.config.getint('Training', 'sample_width', 400)
sample_height = self.config.getint('Training', 'sample_height', 400)
self.train_loader, self.val_loader = self.get_dataloader(
sample_width, sample_height)
self.class_names = self.train_loader.dataset.classes
log_dir = os.path.join(self.curr_dir, 'runs')
raw_data_dir = os.path.join(self.curr_dir, 'runs_raw_results')
self.setup_tensorboard(log_dir, raw_data_dir=raw_data_dir)
# Log a few example spectrograms to tensorboard;
# one per class:
TensorBoardPlotter.write_img_grid(
self.writer,
self.root_train_test_data,
len(self.class_names), # Num of train examples
)
# All ResultTally instances are
# collected here (two per epoch, for
# for all training loop runs, and one
# for all val loop runs:
self.step_results = ResultCollection()
self.log.debug(
f"Just before train: \n{'none--on CPU' if self.fastest_device.type == 'cpu' else torch.cuda.memory_summary()}"
)
try:
final_epoch = self.train()
self.visualize_final_epoch_results(final_epoch)
finally:
self.close_tensorboard()
def prep_model_inference(self, model_path):
'''
1. Parses model_path into its components, and
creates a dict: self.model_props, which
contains the network type, grayscale or not,
whether pretrained, etc.
2. Creates self.csv_writer to write results measures
into csv files. The destination file is determined
as follows:
<script_dir>/runs_raw_inferences/inf_csv_results_<datetime>/<model-props-derived-fname>.csv
3. Creates self.writer(), a tensorboard writer with destination dir:
<script_dir>/runs_inferences/inf_results_<datetime>
4. Creates an ImageFolder classed dataset to self.samples_path
5. Creates a shuffling DataLoader
6. Initializes self.num_classes and self.class_names
7. Creates self.model from the passed-in model_path name
:param model_path: path to model that will be used for
inference by this instance of Inferencer
:type model_path: str
'''
model_fname = os.path.basename(model_path)
# Extract model properties
# from the model filename:
self.model_props = FileUtils.parse_filename(model_fname)
csv_results_root = os.path.join(self.curr_dir, 'runs_raw_inferences')
#self.csv_dir = os.path.join(csv_results_root, f"inf_csv_results_{uuid.uuid4().hex}")
ts = FileUtils.file_timestamp()
self.csv_dir = os.path.join(csv_results_root, f"inf_csv_results_{ts}")
os.makedirs(self.csv_dir, exist_ok=True)
csv_file_nm = FileUtils.construct_filename(self.model_props,
prefix='inf',
suffix='.csv',
incl_date=True)
csv_path = os.path.join(self.csv_dir, csv_file_nm)
self.csv_writer = CSVWriterCloseable(csv_path)
ts = FileUtils.file_timestamp()
tensorboard_root = os.path.join(self.curr_dir, 'runs_inferences')
tensorboard_dest = os.path.join(tensorboard_root, f"inf_results_{ts}")
#f"inf_results_{ts}{uuid.uuid4().hex}")
os.makedirs(tensorboard_dest, exist_ok=True)
self.writer = SummaryWriterPlus(log_dir=tensorboard_dest)
dataset = SingleRootImageDataset(
self.samples_path, to_grayscale=self.model_props['to_grayscale'])
# Make reproducible:
Utils.set_seed(42)
#********Utils.set_seed(56)
self.loader = DataLoader(dataset,
batch_size=self.batch_size,
shuffle=True,
drop_last=True)
self.class_names = dataset.class_names()
self.num_classes = len(self.class_names)
# Get the right type of model,
# Don't bother getting it pretrained,
# of freezing it, b/c we will overwrite
# the weights:
self.model = NetUtils.get_net(
self.model_props['net_name'],
num_classes=self.num_classes,
pretrained=False,
freeze=0,
to_grayscale=self.model_props['to_grayscale'])
self.log.info(f"Tensorboard info written to {tensorboard_dest}")
self.log.info(f"Result measurement CSV file(s) written to {csv_path}")