def _load_and_preprocess_data(self, images, labels):
"""Loads and preprocesses data
Records `input_shape`, `data`, and `num_classes` into `self
Args:
images (numpy.array/str): array with shape (n_images, dim1, dim2 , channel_size), or a string with name of keras-dataset (cifar10, fashion_mnsit)
labels (numpy.array): labels of images, array with shape (n_images) where each element is an integer from 0 to number of classes
"""
if isinstance(images, str):
X, y, self.input_shape = DataOp.load(images)
else:
X, y = images, labels
self.input_shape = X.shape[1:]
self.data = DataOp.preprocess(X, y, self.config["train_set_size"])
self.num_classes = DataOp.find_num_classes(self.data)
def fit(self, data, augmented_data=None, epochs=None):
"""Fits the model with given data and augmented-data
Args:
data (dict): should have keys 'X_train' and 'y_train'
augmented_data (dict): should have keys 'X_train' and 'y_train'. If none, augmented-data not used
epochs (int):
Returns:
dict: history of training
"""
if epochs is None:
epochs = self.config["child_epochs"]
if augmented_data is None:
X_train = data["X_train"]
y_train = data["y_train"]
else:
X_train = np.concatenate([data["X_train"], augmented_data["X_train"]])
y_train = np.concatenate([data["y_train"], augmented_data["y_train"]])
X_val, y_val = DataOp.sample_validation_set(data)
record = self.model.fit(
x=X_train,
y=y_train,
batch_size=self.config["child_batch_size"],
epochs=epochs,
validation_data=(X_val, y_val),
shuffle=True,
verbose=2,
)
return record.history
def run_model(dataset_name, num_classes, epochs, batch_size, policies_path):
data, input_shape = DataOp.load(dataset_name)
data = DataOp.preprocess_normal(data)
wrn_28_10 = ChildCNN(
model_name="wrn_28_10",
input_shape=input_shape,
batch_size=batch_size,
num_classes=num_classes,
pre_augmentation_weights_path="initial_model_weights.h5",
logging=logging,
)
if policies_path == "dont_augment":
policy_str = "non_augmented"
else:
policy_str = "augmented"
csv_logger = CSVLogger(
f"{EXPERIMENT_FOLDER_PATH}/wrn_28_10_training_on_{dataset_name}_{policy_str}.csv"
)
if policies_path == "dont_augment":
history = wrn_28_10.fit_normal(data,
epochs=epochs,
csv_logger=csv_logger)
print(f"Reached validation accuracy is {history['val_acc'][-1]}")
else:
datagen = deepaugment_image_generator(
data["X_train"],
data["y_train"],
policies_path,
batch_size=batch_size,
augment_chance=0.8,
)
print("fitting the model")
history = wrn_28_10.fit_with_generator(
datagen,
data["X_val"],
data["y_val"],
train_data_size=len(data["X_train"]),
epochs=epochs,
csv_logger=csv_logger,
)
print(f"Reached validation accuracy is {history['val_acc'][-1]}")
def main():
X, y, input_shape = DataOp.load("cifar10")
run_full_model(
X,
y,
test_proportion=0.1,
model="basiccnn",
epochs=200,
batch_size=32,
policies_path="../../reports/best_policies/25_policies_cifar10.csv")
def main():
X, y, input_shape = DataOp.load("cifar10")
run_full_model(
X,
y,
test_proportion=0.1,
model="wrn_28_10",
epochs=100,
batch_size=32,
policies_path=
"/home/acb11354uz/B4researchMain/B4ResearchDeepaugment/reports/best_policies/top20_policies_cifar10_exp_2019-02-08_03-54_3000_iters.csv"
)
def main():
X, y, input_shape = DataOp.load("cifar10")
run_full_model(
X,
y,
test_proportion=0.1,
model="wrn_40_2",
epochs=100,
batch_size=32,
policies_path=
"./deepaugment/reports/best_policies/top20_policies_cifar10_exp_2019-02-08_03-54_3000_iters.csv"
)
import numpy as np
import os
from keras.preprocessing import image
from sklearn.model_selection import train_test_split
import sys
from os.path import dirname, realpath
file_path = realpath(__file__)
dir_of_file = dirname(file_path)
parent_dir_of_file = dirname(dir_of_file)
sys.path.insert(0, parent_dir_of_file)
from run_full_model import run_full_model
from build_features import DataOp
X, y, input_shape = DataOp.load("cifar10")
run_full_model(X,
y,
test_proportion=0.1,
epochs=200,
batch_size=32,
policies_path="random")
def run_full_model(images,
labels,
test_proportion=0.1,
model="wrn_28_10",
epochs=200,
batch_size=64,
policies_path="dont_augment"):
data = {}
data["X_train"], data["X_val"], data["y_train"], data[
"y_val"] = train_test_split(images,
labels,
test_size=test_proportion,
shuffle=True)
data = DataOp.preprocess_normal(data)
input_shape = data["X_train"][0].shape
num_classes = data["y_train"].shape[1]
cnn_config = {
"model": model,
"weights": "imagenet",
"input_shape": input_shape,
"child_batch_size": batch_size,
"pre_augmentation_weights_path": "initial_model_weights.h5",
"logging": logging
}
full_model = ChildCNN(input_shape=input_shape,
num_classes=num_classes,
config=cnn_config)
if policies_path == "dont_augment":
policy_str = "non_augmented"
else:
policy_str = "augmented"
csv_logger = CSVLogger(
f"{EXPERIMENT_FOLDER_PATH}/wrn_28_10_training_on_{policy_str}.csv")
if policies_path == "dont_augment":
history = full_model.fit_normal(data,
epochs=epochs,
csv_logger=csv_logger)
print(f"Reached validation accuracy is {history['val_acc'][-1]}")
else:
datagen = deepaugment_image_generator(
data["X_train"],
data["y_train"],
policies_path,
batch_size=batch_size,
augment_chance=0.8,
)
file = open('datagen.txt', 'w')
file.write(datagen)
file.close()
print("fitting the model")
history = full_model.fit_with_generator(
datagen,
data["X_val"],
data["y_val"],
train_data_size=len(data["X_train"]),
epochs=epochs,
csv_logger=csv_logger,
)
print(f"Reached validation accuracy is {history['val_acc'][-1]}")