def prepare_training_generators(self): self._train_generator = VOCGenerator(batch_size=self.batch_size, shuffle=True, label_encoder=self.input_encoder, dct=True, transforms=self.train_transformations, images_path=self.train_sets) self._train_generator.prepare_dataset() self._validation_generator = VOCGenerator(batch_size=self.batch_size, shuffle=True, label_encoder=self.input_encoder, dct=True, transforms=self.validation_transformations, images_path=self.validation_sets) self._validation_generator.prepare_dataset(exclude_difficult=True) self.validation_steps = len(self._validation_generator)
def prepare_testing_generator(self): self._test_generator = VOCGenerator( batch_size=1, shuffle=False, label_encoder=self.input_encoder, transforms=self.test_transformations, images_path=self.test_sets) self._test_generator.prepare_dataset()
class TrainingConfiguration(object): def __init__(self): # Variables to hold the description of the experiment self.config_description = "This is the template config file." # System dependent variable self._workers = 5 self._multiprocessing = True # Variables for comet.ml self._project_name = "jpeg_deep" self._workspace = "ssd" # Network variables self._weights = "/dlocal/home/2017018/bdegue01/weights/jpeg_deep/classification_dct/lcrfa_y/classification_dct_jpeg-deep_46lWNDTTTHMqAvNj88wqA2HPofUMhHva/checkpoints/epoch-71_loss-0.9617_val_loss-1.6794.h5" self._network = SSD300_resnet(backbone="lcrfa_y", dct=True, image_shape=(38, 38)) # Training variables self._epochs = 240 self._batch_size = 32 self._steps_per_epoch = 1000 self.optimizer_parameters = {"lr": 0.001, "momentum": 0.9} self._optimizer = SGD(**self.optimizer_parameters) self._loss = SSDLoss(neg_pos_ratio=3, alpha=1.0).compute_loss self._metrics = None dataset_path = environ["DATASET_PATH"] images_2007_path = join(dataset_path, "VOC2007/JPEGImages") images_2012_path = join(dataset_path, "VOC2012/JPEGImages") self.train_sets = [(images_2007_path, join(dataset_path, "VOC2007/ImageSets/Main/train.txt")), (images_2012_path, join(dataset_path, "VOC2012/ImageSets/Main/train.txt"))] self.validation_sets = [(images_2007_path, join(dataset_path, "VOC2007/ImageSets/Main/val.txt")), (images_2012_path, join(dataset_path, "VOC2012/ImageSets/Main/val.txt"))] self.test_sets = [(images_2007_path, join(dataset_path, "VOC2007/ImageSets/Main/test.txt"))] # Keras stuff self.model_checkpoint = None self.reduce_lr_on_plateau = ReduceLROnPlateau(patience=5, verbose=1) self.terminate_on_nan = TerminateOnNaN() self.early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=15) self._callbacks = [ self.reduce_lr_on_plateau, self.early_stopping, self.terminate_on_nan ] self.input_encoder = SSDInputEncoder() self.train_transformations = [SSDDataAugmentation()] self.validation_transformations = [ ConvertTo3Channels(), Resize(height=300, width=300) ] self.test_transformations = [ ConvertTo3Channels(), Resize(height=300, width=300) ] self._train_generator = None self._validation_generator = None self._test_generator = None self._horovod = None self._displayer = DisplayerObjects() def prepare_runtime_checkpoints(self, directories_dir): log_dir = directories_dir["log_dir"] checkpoints_dir = directories_dir["checkpoints_dir"] self._callbacks.append( ModelCheckpoint(filepath=join( checkpoints_dir, "epoch-{epoch:02d}_loss-{loss:.4f}_val_loss-{val_loss:.4f}.h5" ), save_best_only=True)) self._callbacks.append(TensorBoard(log_dir)) def prepare_horovod(self, hvd): self._horovod = hvd self.optimizer_parameters[ "lr"] = self.optimizer_parameters["lr"] * hvd.size() self._optimizer = SGD(**self.optimizer_parameters) self._optimizer = hvd.DistributedOptimizer(self._optimizer) self._steps_per_epoch = self._steps_per_epoch // hvd.size() self._validation_steps = 3 * self._validation_steps // hvd.size() self._callbacks = [ hvd.callbacks.BroadcastGlobalVariablesCallback(0), # Note: This callback must be in the list before the ReduceLROnPlateau, # TensorBoard or other metrics-based callbacks. hvd.callbacks.MetricAverageCallback(), # Horovod: using `lr = 1.0 * hvd.size()` from the very beginning leads to worse final # accuracy. Scale the learning rate `lr = 1.0` ---> `lr = 1.0 * hvd.size()` during # the first five epochs. See https://arxiv.org/abs/1706.02677 for details. hvd.callbacks.LearningRateWarmupCallback(warmup_epochs=5, verbose=1), # Reduce the learning rate if training plateaues. self.reduce_lr_on_plateau, self.terminate_on_nan, self.early_stopping ] def prepare_for_inference(self): K.clear_session() self._network = SSD300_resnet(backbone="lcrfa_y", dct=True, image_shape=(38, 38), mode="inference") def prepare_evaluator(self): self._evaluator = PascalEvaluator() def prepare_testing_generator(self): self._test_generator = VOCGenerator( batch_size=self.batch_size, shuffle=False, label_encoder=self.input_encoder, dct=True, only_y=True, transforms=self.test_transformations, images_path=self.test_sets) self._test_generator.prepare_dataset() def prepare_training_generators(self): self._train_generator = VOCGenerator( batch_size=self.batch_size, shuffle=True, label_encoder=self.input_encoder, dct=True, only_y=True, transforms=self.train_transformations, images_path=self.train_sets) self._train_generator.prepare_dataset() self._validation_generator = VOCGenerator( batch_size=self.batch_size, shuffle=True, label_encoder=self.input_encoder, dct=True, only_y=True, transforms=self.validation_transformations, images_path=self.validation_sets) self._validation_generator.prepare_dataset(exclude_difficult=True) self.validation_steps = len(self._validation_generator) @property def train_generator(self): return self._train_generator @property def validation_generator(self): return self._validation_generator @property def test_generator(self): return self._test_generator @property def batch_size(self): return self._batch_size @property def gpus(self): return self._gpus @property def evaluator(self): return self._evaluator @property def loss(self): return self._loss @property def displayer(self): return self._displayer @property def metrics(self): return self._metrics @property def multiprocessing(self): return self._multiprocessing @property def network(self): return self._network @property def optimizer(self): return self._optimizer @property def steps_per_epoch(self): return self._steps_per_epoch @property def weights(self): return self._weights @weights.setter def weights(self, value): self._weights = value @property def workers(self): return self._workers @property def epochs(self): return self._epochs @property def callbacks(self): return self._callbacks @property def project_name(self): return self._project_name @property def workspace(self): return self._workspace