def generate_predictions(self, model, data):
self.test_image_augmentations = {}
if 'zca_whitening' in self.image_augmentations.keys():
if self.image_augmentations['zca_whitening']:
self.test_image_augmentations['zca_whitening'] = True
if self.img_norm == 'mean_and_std':
self.test_image_augmentations['featurewise_center'] = True
self.test_image_augmentations[
'featurewise_std_normalization'] = True
test_datagen = ImageDataGenerator(
preprocessing_function=self.preprocess_input,
**self.test_image_augmentations)
if self.img_norm == 'mean_and_std':
if self.greyscale:
test_datagen.mean = np.array(self.img_mean_pt,
dtype=np.float32)
test_datagen.std = np.array(self.img_std_pt, dtype=np.float32)
else:
test_datagen.mean = np.array(self.img_mean_pt,
dtype=np.float32).reshape(
(1, 1,
3)) # ordering: [R, G, B]
test_datagen.std = np.array(self.img_std_pt,
dtype=np.float32).reshape(
(1, 1,
3)) # ordering: [R, G, B]
# Create Data Generators
if self.greyscale:
valid_generator = test_datagen.flow_from_dataframe(
dataframe=data,
directory=None,
x_col=self.x_name,
y_col=self.y_name,
batch_size=self.batch_size,
seed=self.random_state,
shuffle=False,
class_mode="other",
target_size=(self.model_img_width, self.model_img_height),
color_mode='grayscale')
else:
valid_generator = test_datagen.flow_from_dataframe(
dataframe=data,
directory=None,
x_col=self.x_name,
y_col=self.y_name,
batch_size=self.batch_size,
seed=self.random_state,
shuffle=False,
class_mode="other",
target_size=(self.model_img_width, self.model_img_height))
# Predict
if self.norm_labels:
print(f"mean: {self.mean_pt} std: {self.std_pt}")
return model.predict_generator(
valid_generator,
steps=np.ceil(len(data) / self.batch_size),
verbose=1) * self.std_pt + self.mean_pt
else:
return model.predict_generator(valid_generator,
steps=np.ceil(
len(data) / self.batch_size),
verbose=1)
model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'])
# this is the augmentation we will use for training
train_datagen = ImageDataGenerator(
rescale=1. / 255,
featurewise_center=True,
featurewise_std_normalization=True,
horizontal_flip=True,
vertical_flip=True,
rotation_range=15,
width_shift_range=0.1,
height_shift_range=0.1)
train_datagen.mean = GetCifar10Mean()
train_datagen.std = GetCifar10STD()
test_datagen = ImageDataGenerator(rescale=1. / 255,
featurewise_center=True,
featurewise_std_normalization=True)
test_datagen.mean = GetCifar10Mean()
test_datagen.std = GetCifar10STD()
train_generator = train_datagen.flow_from_directory(
train_data_dir,
target_size=(img_width, img_height),
color_mode='rgb',
batch_size=batch_size,
class_mode='categorical')
validation_generator = test_datagen.flow_from_directory(
def train_model(self, model, train_data, test_data):
# If normalized labels are needed, extract it
if self.norm_labels:
train_data[self.y_name] = (train_data[self.y_name] -
self.mean_pt) / self.std_pt
test_data[self.y_name] = (test_data[self.y_name] -
self.mean_pt) / self.std_pt
if self.lf_setting == 'weighted_mape':
self.weight = train_data[self.y_name].max()
# Create Data Generators
self.test_image_augmentations = {}
if 'zca_whitening' in self.image_augmentations.keys():
if self.image_augmentations['zca_whitening']:
self.test_image_augmentations['zca_whitening'] = True
if self.img_norm == 'mean_and_std':
self.image_augmentations['featurewise_center'] = True
self.image_augmentations['featurewise_std_normalization'] = True
self.test_image_augmentations['featurewise_center'] = True
self.test_image_augmentations[
'featurewise_std_normalization'] = True
datagen = ImageDataGenerator(
preprocessing_function=self.preprocess_input,
**self.image_augmentations)
test_datagen = ImageDataGenerator(
preprocessing_function=self.preprocess_input,
**self.test_image_augmentations)
if self.img_norm == 'mean_and_std':
if self.greyscale:
datagen.mean = np.array(self.img_mean_pt, dtype=np.float32)
datagen.std = np.array(self.img_std_pt, dtype=np.float32)
test_datagen.mean = np.array(self.img_mean_pt,
dtype=np.float32)
test_datagen.std = np.array(self.img_std_pt, dtype=np.float32)
else:
datagen.mean = np.array(self.img_mean_pt,
dtype=np.float32).reshape(
(1, 1, 3)) # ordering: [R, G, B]
datagen.std = np.array(self.img_std_pt,
dtype=np.float32).reshape(
(1, 1, 3)) # ordering: [R, G, B]
test_datagen.mean = np.array(self.img_mean_pt,
dtype=np.float32).reshape(
(1, 1,
3)) # ordering: [R, G, B]
test_datagen.std = np.array(self.img_std_pt,
dtype=np.float32).reshape(
(1, 1,
3)) # ordering: [R, G, B]
# Set-up callback
callbacks_list = [
ModelCheckpoint(self.model_pth, verbose=1, period=5),
ModelCheckpoint(self.model_best_pth,
verbose=1,
save_best_only=True),
CSVLogger(os.path.join(self.output_pth, 'hist.csv'))
]
# Create Data Generators
if self.greyscale:
train_generator = datagen.flow_from_dataframe(
dataframe=train_data,
directory=None,
x_col=self.x_name,
y_col=self.y_name,
batch_size=self.batch_size,
seed=self.random_state,
shuffle=True,
class_mode="raw",
target_size=(self.model_img_width, self.model_img_height),
color_mode='grayscale')
valid_generator = test_datagen.flow_from_dataframe(
dataframe=test_data,
directory=None,
x_col=self.x_name,
y_col=self.y_name,
batch_size=self.batch_size,
seed=self.random_state,
shuffle=False,
class_mode="raw",
target_size=(self.model_img_width, self.model_img_height),
color_mode='grayscale')
else:
train_generator = datagen.flow_from_dataframe(
dataframe=train_data,
directory=None,
x_col=self.x_name,
y_col=self.y_name,
batch_size=self.batch_size,
seed=self.random_state,
shuffle=True,
class_mode="raw",
target_size=(self.model_img_width, self.model_img_height))
valid_generator = test_datagen.flow_from_dataframe(
dataframe=test_data,
directory=None,
x_col=self.x_name,
y_col=self.y_name,
batch_size=self.batch_size,
seed=self.random_state,
shuffle=False,
class_mode="raw",
target_size=(self.model_img_width, self.model_img_height))
H = model.fit_generator(
generator=train_generator,
steps_per_epoch=len(train_data) // self.batch_size,
validation_data=valid_generator,
validation_steps=len(test_data) // self.batch_size,
epochs=self.epochs,
verbose=1,
callbacks=callbacks_list,
workers=12)
return H, model
