def generate_predictions(self, model, data): self.test_image_augmentations = {} if 'zca_whitening' in self.image_augmentations.keys(): if self.image_augmentations['zca_whitening']: self.test_image_augmentations['zca_whitening'] = True if self.img_norm == 'mean_and_std': self.test_image_augmentations['featurewise_center'] = True self.test_image_augmentations[ 'featurewise_std_normalization'] = True test_datagen = ImageDataGenerator( preprocessing_function=self.preprocess_input, **self.test_image_augmentations) if self.img_norm == 'mean_and_std': if self.greyscale: test_datagen.mean = np.array(self.img_mean_pt, dtype=np.float32) test_datagen.std = np.array(self.img_std_pt, dtype=np.float32) else: test_datagen.mean = np.array(self.img_mean_pt, dtype=np.float32).reshape( (1, 1, 3)) # ordering: [R, G, B] test_datagen.std = np.array(self.img_std_pt, dtype=np.float32).reshape( (1, 1, 3)) # ordering: [R, G, B] # Create Data Generators if self.greyscale: valid_generator = test_datagen.flow_from_dataframe( dataframe=data, directory=None, x_col=self.x_name, y_col=self.y_name, batch_size=self.batch_size, seed=self.random_state, shuffle=False, class_mode="other", target_size=(self.model_img_width, self.model_img_height), color_mode='grayscale') else: valid_generator = test_datagen.flow_from_dataframe( dataframe=data, directory=None, x_col=self.x_name, y_col=self.y_name, batch_size=self.batch_size, seed=self.random_state, shuffle=False, class_mode="other", target_size=(self.model_img_width, self.model_img_height)) # Predict if self.norm_labels: print(f"mean: {self.mean_pt} std: {self.std_pt}") return model.predict_generator( valid_generator, steps=np.ceil(len(data) / self.batch_size), verbose=1) * self.std_pt + self.mean_pt else: return model.predict_generator(valid_generator, steps=np.ceil( len(data) / self.batch_size), verbose=1)
model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy']) # this is the augmentation we will use for training train_datagen = ImageDataGenerator( rescale=1. / 255, featurewise_center=True, featurewise_std_normalization=True, horizontal_flip=True, vertical_flip=True, rotation_range=15, width_shift_range=0.1, height_shift_range=0.1) train_datagen.mean = GetCifar10Mean() train_datagen.std = GetCifar10STD() test_datagen = ImageDataGenerator(rescale=1. / 255, featurewise_center=True, featurewise_std_normalization=True) test_datagen.mean = GetCifar10Mean() test_datagen.std = GetCifar10STD() train_generator = train_datagen.flow_from_directory( train_data_dir, target_size=(img_width, img_height), color_mode='rgb', batch_size=batch_size, class_mode='categorical') validation_generator = test_datagen.flow_from_directory(
def train_model(self, model, train_data, test_data): # If normalized labels are needed, extract it if self.norm_labels: train_data[self.y_name] = (train_data[self.y_name] - self.mean_pt) / self.std_pt test_data[self.y_name] = (test_data[self.y_name] - self.mean_pt) / self.std_pt if self.lf_setting == 'weighted_mape': self.weight = train_data[self.y_name].max() # Create Data Generators self.test_image_augmentations = {} if 'zca_whitening' in self.image_augmentations.keys(): if self.image_augmentations['zca_whitening']: self.test_image_augmentations['zca_whitening'] = True if self.img_norm == 'mean_and_std': self.image_augmentations['featurewise_center'] = True self.image_augmentations['featurewise_std_normalization'] = True self.test_image_augmentations['featurewise_center'] = True self.test_image_augmentations[ 'featurewise_std_normalization'] = True datagen = ImageDataGenerator( preprocessing_function=self.preprocess_input, **self.image_augmentations) test_datagen = ImageDataGenerator( preprocessing_function=self.preprocess_input, **self.test_image_augmentations) if self.img_norm == 'mean_and_std': if self.greyscale: datagen.mean = np.array(self.img_mean_pt, dtype=np.float32) datagen.std = np.array(self.img_std_pt, dtype=np.float32) test_datagen.mean = np.array(self.img_mean_pt, dtype=np.float32) test_datagen.std = np.array(self.img_std_pt, dtype=np.float32) else: datagen.mean = np.array(self.img_mean_pt, dtype=np.float32).reshape( (1, 1, 3)) # ordering: [R, G, B] datagen.std = np.array(self.img_std_pt, dtype=np.float32).reshape( (1, 1, 3)) # ordering: [R, G, B] test_datagen.mean = np.array(self.img_mean_pt, dtype=np.float32).reshape( (1, 1, 3)) # ordering: [R, G, B] test_datagen.std = np.array(self.img_std_pt, dtype=np.float32).reshape( (1, 1, 3)) # ordering: [R, G, B] # Set-up callback callbacks_list = [ ModelCheckpoint(self.model_pth, verbose=1, period=5), ModelCheckpoint(self.model_best_pth, verbose=1, save_best_only=True), CSVLogger(os.path.join(self.output_pth, 'hist.csv')) ] # Create Data Generators if self.greyscale: train_generator = datagen.flow_from_dataframe( dataframe=train_data, directory=None, x_col=self.x_name, y_col=self.y_name, batch_size=self.batch_size, seed=self.random_state, shuffle=True, class_mode="raw", target_size=(self.model_img_width, self.model_img_height), color_mode='grayscale') valid_generator = test_datagen.flow_from_dataframe( dataframe=test_data, directory=None, x_col=self.x_name, y_col=self.y_name, batch_size=self.batch_size, seed=self.random_state, shuffle=False, class_mode="raw", target_size=(self.model_img_width, self.model_img_height), color_mode='grayscale') else: train_generator = datagen.flow_from_dataframe( dataframe=train_data, directory=None, x_col=self.x_name, y_col=self.y_name, batch_size=self.batch_size, seed=self.random_state, shuffle=True, class_mode="raw", target_size=(self.model_img_width, self.model_img_height)) valid_generator = test_datagen.flow_from_dataframe( dataframe=test_data, directory=None, x_col=self.x_name, y_col=self.y_name, batch_size=self.batch_size, seed=self.random_state, shuffle=False, class_mode="raw", target_size=(self.model_img_width, self.model_img_height)) H = model.fit_generator( generator=train_generator, steps_per_epoch=len(train_data) // self.batch_size, validation_data=valid_generator, validation_steps=len(test_data) // self.batch_size, epochs=self.epochs, verbose=1, callbacks=callbacks_list, workers=12) return H, model