def load_lsa16(data_dir, config, splits):
"""
Load lsa16 dataset.
Args:
data_dir (str): path of the directory with 'splits', 'data' subdirs.
config (dict): general dict with program settings.
splits (list): list of strings 'train'|'val'|'test'
Returns (dict): dictionary with keys as splits and values as tf.Dataset
"""
DATASET_NAME = "lsa16"
DATASET_PATH = "/develop/data/lsa16/data"
data = hd.load(DATASET_NAME, DATASET_PATH)
features = data[0]
classes = data[1]['y']
uniqueClasses = np.unique(classes)
x_train, x_test, y_train, y_test = train_test_split_balanced(features,
classes,
train_size=config['data.train_size'],
test_size=config['data.test_size'])
x_train, x_test = x_train / 255.0, x_test / 255.0
_, amountPerTrain = np.unique(y_train, return_counts=True)
_, amountPerTest = np.unique(y_test, return_counts=True)
train_datagen_args = dict(featurewise_center=True,
featurewise_std_normalization=True,
rotation_range=config['data.rotation_range'],
width_shift_range=config['data.width_shift_range'],
height_shift_range=config['data.height_shift_range'],
horizontal_flip=config['data.horizontal_flip'],
fill_mode='constant',
cval=0)
train_datagen = ImageDataGenerator(train_datagen_args)
train_datagen.fit(x_train)
test_datagen_args = dict(featurewise_center=True,
featurewise_std_normalization=True,
fill_mode='constant',
cval=0)
test_datagen = ImageDataGenerator(test_datagen_args)
test_datagen.fit(x_train)
w, h, c = list(map(int, config['model.x_dim'].split(',')))
ret = {}
for split in splits:
# n_way (number of classes per episode)
if split in ['val', 'test']:
n_way = config['data.test_way']
else:
n_way = config['data.train_way']
# n_support (number of support examples per class)
if split in ['val', 'test']:
n_support = config['data.test_support']
else:
n_support = config['data.train_support']
# n_query (number of query examples per class)
if split in ['val', 'test']:
n_query = config['data.test_query']
else:
n_query = config['data.train_query']
if split in ['val', 'test']:
y = y_test
x = x_test
dg = train_datagen
dg_args = train_datagen_args
else:
y = y_train
x = x_train
dg = test_datagen
dg_args = test_datagen_args
amountPerClass = amountPerTest if split in ['val', 'test'] else amountPerTrain
i = np.argsort(y)
x = x[i, :, :, :]
if config['model.type'] in ['processed']:
for index in i:
x[index, :, :, :] = dg.apply_transform(x[index], dg_args)
data = np.reshape(x, (len(uniqueClasses), amountPerClass[0], w, h, c))
data_loader = DataLoader(data,
n_classes=len(uniqueClasses),
n_way=n_way,
n_support=n_support,
n_query=n_query,
x_dim=(w, h, c))
ret[split] = data_loader
return ret
# if idx >= 1:
# break
print("folder :", folder)
base_dir = TRAIN_DIR + '/' + folder + '/' # 각 폴더 디렉토리 'C:/data/LPD_competition/train/0'
print("basedir :", base_dir)
img_lst = natsorted(os.listdir(base_dir)) # 각 폴더 안에 있는 jpg 파일 리스트
print(img_lst)
for i, f in enumerate(img_lst): # 각 폴더에 접근해서
# if i >= 1:
# break
print("f :", f)
img_dir = base_dir + f # 'C:/data/LPD_competition/train/0/0.jpg'
print("img_dir :", img_dir)
img = np.expand_dims(image.load_img(img_dir,
target_size=(DIMENSION,
DIMENSION)),
axis=0) # 각 이미지를 불어온다
print(img, "\n", img.shape)
# cv_img = cv.imread(img_dir)
# cv.imshow("whatever", cv_img)
# cv.waitKey(0)
# cv.destroyAllWindows()
train_datagen.fit(img) #image gen
for x, val in zip(
train_datagen.flow(x=img,
save_to_dir=base_dir,
save_prefix='aug',
shuffle=False), range(1)):
pass
print("base dir :", base_dir)
def get_data(problem, shards, rank, data_augmentation_level, n_batch_train,
n_batch_test, n_batch_init, resolution):
if problem == 'mnist':
from tensorflow.keras.datasets import mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
y_train = np.reshape(y_train, [-1])
y_test = np.reshape(y_test, [-1])
# Pad with zeros to make 32x32
x_train = np.lib.pad(x_train, ((0, 0), (2, 2), (2, 2)), 'minimum')
# Pad with zeros to make 32x23
x_test = np.lib.pad(x_test, ((0, 0), (2, 2), (2, 2)), 'minimum')
# x_train = np.tile(np.reshape(x_train, (-1, 32, 32, 1)), (1, 1, 1, 3))
# x_test = np.tile(np.reshape(x_test, (-1, 32, 32, 1)), (1, 1, 1, 3))
x_train = np.reshape(x_train, (-1, 32, 32, 1))
x_test = np.reshape(x_test, (-1, 32, 32, 1))
elif problem == 'cifar10':
from tensorflow.keras.datasets import cifar10
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
y_train = np.reshape(y_train, [-1])
y_test = np.reshape(y_test, [-1])
else:
raise Exception()
print('n_train:', x_train.shape[0], 'n_test:', x_test.shape[0])
# Shard before any shuffling
x_train, y_train = shard((x_train, y_train), shards, rank)
x_test, y_test = shard((x_test, y_test), shards, rank)
print('n_shard_train:', x_train.shape[0], 'n_shard_test:', x_test.shape[0])
from tensorflow.keras.preprocessing.image import ImageDataGenerator
datagen_test = ImageDataGenerator()
if data_augmentation_level == 0:
datagen_train = ImageDataGenerator()
else:
if problem == 'mnist':
datagen_train = ImageDataGenerator(width_shift_range=0.1,
height_shift_range=0.1)
elif problem == 'cifar10':
if data_augmentation_level == 1:
datagen_train = ImageDataGenerator(width_shift_range=0.1,
height_shift_range=0.1)
elif data_augmentation_level == 2:
datagen_train = ImageDataGenerator(
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
rotation_range=15, # degrees rotation
zoom_range=0.1,
shear_range=0.02,
)
else:
raise Exception()
else:
raise Exception()
datagen_train.fit(x_train)
datagen_test.fit(x_test)
train_flow = datagen_train.flow(x_train, y_train, n_batch_train)
test_flow = datagen_test.flow(x_test, y_test, n_batch_test, shuffle=False)
def make_iterator(flow, resolution):
def iterator():
x_full, y = flow.next()
x_full = x_full.astype(np.float32)
x = downsample(x_full, resolution)
x = x_to_uint8(x)
return x, y
return iterator
#init_iterator = make_iterator(train_flow, resolution)
train_iterator = make_iterator(train_flow, resolution)
test_iterator = make_iterator(test_flow, resolution)
# Get data for initialization
data_init = make_batch(train_iterator, n_batch_train, n_batch_init)
return train_iterator, test_iterator, data_init
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range=10, # randomly rotate images in the range (degrees, 0 to 180)
zoom_range = 0.2,
shear_range = 0.2,
width_shift_range=0.2, # randomly shift images horizontally (fraction of total width)
height_shift_range=0.2, # randomly shift images vertically (fraction of total height)
horizontal_flip=True, # randomly flip images
vertical_flip=False)
#model.fit_generator(datagen.flow(X_train, y_train,batch_size=32), steps_per_epoch=len(X_train) / 32, epochs=13, validation_data=val_datagen.flow(X_test, y_test, batch_size=32), validation_steps = len(y_train)/32, callbacks=[earlyStopping], shuffle=True)
datagen.fit(X_train)
val_datagen = ImageDataGenerator(rescale=1./255)
# create base model from pretrained
base_model = tf.keras.applications.MobileNetV2(input_shape=(96,96,3),
include_top=False,
weights='imagenet')
# freeze the conv base_model
base_model.trainable = False
#base_model.summary()
print(X.shape)
### dataset
(train_x, train_y), (valid_x, valid_y) = tf.keras.datasets.cifar10.load_data()
train_x = train_x.astype('float')
train_y = np.squeeze(train_y)
valid_x = valid_x.astype('float')
valid_y = np.squeeze(valid_y)
print('train_x shape : ', train_x.shape)
print('train_y shape : ', train_y.shape)
print('valid_x shape : ', valid_x.shape)
print('valid_y shape : ', valid_y.shape)
train_datagen = ImageDataGenerator(horizontal_flip=True,
width_shift_range=0.1,
height_shift_range=0.1)
train_datagen.fit(train_x)
valid_datagen = ImageDataGenerator()
model = xenqore.apps.VGGNet13(mode=0,
network_config=network_config,
layer_config=layers_config,
act_config=activations_config,
saved_model='',
classes=network_config.classes,
input_shape=train_x.shape[1:])
model.summary()
model.compile(optimizer=tf.keras.optimizers.Adam(
learning_rate=network_config.initial_lr),
model.add(Flatten())
model.add(Dense(units=512, activation="relu") )
model.add(Dropout(0.5))
model.add(Dense(units=10,activation='softmax'))
"""
model.summary()
#data augmentation
datagen = ImageDataGenerator(
rotation_range=20,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True
)
datagen.fit(train_feature_normalize)
#training
batch_size = 64
#opt_rms = keras.optimizers.rmsprop(lr=0.001,decay=1e-6)
#model.compile(loss='categorical_crossentropy',optimizer='adam' , metrics=['accuracy'])
opt_rms = keras.optimizers.rmsprop(lr=0.001,decay=1e-6)
model.compile(loss='categorical_crossentropy',optimizer=opt_rms , metrics=['accuracy'])
try:
model.load_weights("Cifar10_CNN_model_3.weight")
print("載入模型參數成功，繼續訓練模型!")
except :
print("載入模型參數失敗，開始訓練一個新模型!")
model.compile(optimizer='Adam', loss='categorical_crossentropy',
metrics=['accuracy'])
# featurewise_center = False, featurewise_std_normalization=False,
img = ImageDataGenerator(
rotation_range=25,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=0.1,
zoom_range=0.1,
horizontal_flip=True,
vertical_flip=False,
dtype=np.float32)
img.fit(x_train)
# call back
callbacks = []
file_path = model_save_path
modelCheckpoint = ModelCheckpoint(
file_path,
monitor='val_accuracy',
save_best_only=True,
mode='max',
save_weights_only=False)
callbacks.append(modelCheckpoint)
def main():
print("Load and normalize Cifar-10 dataset")
(X_train, y_train), (X_test, y_test) = load_and_normalize_cifar10()
model_file_path, recipe_file_path = download_model_and_recipe(root_dir)
print("Load pretrained model")
base_model = tf.keras.models.load_model(model_file_path)
base_model.summary()
scores = base_model.evaluate(X_test, y_test, verbose=1)
print("Test loss:", scores[0])
print("Test accuracy:", scores[1])
optimizer = tf.keras.optimizers.Adam()
steps_per_epoch = math.ceil(len(X_train) / batch_size)
# Enhance the model and optimizer for pruning using the manager
loggers = TensorBoardLogger(log_dir=log_dir, update_freq=update_freq)
manager = ScheduledModifierManager.from_yaml(recipe_file_path)
model_for_pruning, optimizer, callbacks = manager.modify(base_model,
optimizer,
steps_per_epoch,
loggers=loggers)
callbacks.append(LossesAndMetricsLoggingCallback(loggers))
callbacks.append(model_checkpoint_callback())
# Compile the enhanced model
model_for_pruning.compile(
loss=tf.keras.losses.categorical_crossentropy,
optimizer=optimizer,
metrics=["accuracy"],
run_eagerly=True,
)
# Run training and pruning, with or without data augmentation.
if not data_augmentation:
print("Not using data augmentation.")
model_for_pruning.fit(
X_train,
y_train,
batch_size=batch_size,
validation_data=(X_test, y_test),
shuffle=True,
callbacks=callbacks,
epochs=manager.max_epochs,
)
else:
print("Using real-time data augmentation.")
# This will do preprocessing and realtime data augmentation:
datagen = ImageDataGenerator(
# set input mean to 0 over the dataset
featurewise_center=False,
# set each sample mean to 0
samplewise_center=False,
# divide inputs by std of dataset
featurewise_std_normalization=False,
# divide each input by its std
samplewise_std_normalization=False,
# apply ZCA whitening
zca_whitening=False,
# epsilon for ZCA whitening
zca_epsilon=1e-06,
# randomly rotate images in the range (deg 0 to 180)
rotation_range=0,
# randomly shift images horizontally
width_shift_range=0.1,
# randomly shift images vertically
height_shift_range=0.1,
# set range for random shear
shear_range=0.0,
# set range for random zoom
zoom_range=0.0,
# set range for random channel shifts
channel_shift_range=0.0,
# set mode for filling points outside the input boundaries
fill_mode="nearest",
# value used for fill_mode = "constant"
cval=0.0,
# randomly flip images
horizontal_flip=True,
# randomly flip images
vertical_flip=False,
# set rescaling factor (applied before any other transformation)
rescale=None,
# set function that will be applied on each input
preprocessing_function=None,
# image data format, either "channels_first" or "channels_last"
data_format=None,
# fraction of images reserved for validation (strictly between 0 and 1)
validation_split=0.0,
)
# Compute quantities required for featurewise normalization
# (std, mean, and principal components if ZCA whitening is applied).
datagen.fit(X_train)
# Fit the model on the batches generated by datagen.flow().
model_for_pruning.fit_generator(
datagen.flow(X_train, y_train, batch_size=batch_size),
validation_data=(X_test, y_test),
epochs=manager.max_epochs,
verbose=1,
workers=4,
callbacks=callbacks,
)
# Erase pruning masks and export to ONNX model
pruned_model = manager.finalize(model_for_pruning)
exporter = ModelExporter(pruned_model, output_dir=pruned_model_dir)
onnx_model_name = "pruned_resnet20_v1.onnx"
exporter.export_onnx(name=onnx_model_name)
print("Model exported to {}".format(
os.path.join(pruned_model_dir, onnx_model_name)))
if data_augmentation:
# Data augmentation
data_gen_args = dict(rotation_range=90,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=5,
zoom_range=0.1,
fill_mode='nearest',
horizontal_flip=True,
vertical_flip=True)
image_datagen = ImageDataGenerator(**data_gen_args)
segmentation_datagen = ImageDataGenerator(**data_gen_args)
# Provide the same seed and keyword arguments to the fit and flow methods
seed = 1
image_datagen.fit(dataset_train.x, augment=True, seed=seed)
segmentation_datagen.fit(dataset_train.y, augment=True, seed=seed)
# Combine generators into one which yields image and masks
# Use option save_to_dir='img_flow' to save augmented images
train_generator = zip(
image_datagen.flow(dataset_train.x,
batch_size=batch_size,
seed=seed,
sample_weight=sample_weight),
segmentation_datagen.flow(dataset_train.y,
batch_size=batch_size,
seed=seed,
sample_weight=sample_weight))
# Fit the model with a generator
def run(logger, options, session_log_file, logits_dir, models_dir):
logger.info(cfg.student_train_spacer + "GENERIC MULTISTAGE" +
cfg.student_train_spacer)
with open(session_log_file, "w") as f:
f.write("begin test: " + datetime.datetime.now().isoformat() + "\n")
f.close()
# load configuration file
configuration = import_config(options.config_file_path)
teacher_name = configuration['teacher_name']
epochs = configuration['epochs']
temperatures = configuration['temp_config']
alphas = configuration['alpha_config']
order_combinations = configuration['size_combinations']
# loading training data
X_train, Y_train, X_test, Y_test = load_dataset.load_dataset_by_name(
logger, cfg.dataset)
# mean subtraction regularization
if cfg.subtract_pixel_mean is True:
x_train_mean = np.mean(X_train, axis=0)
X_train -= x_train_mean
X_test -= x_train_mean
if cfg.use_fit_generator_student is True or cfg.use_fit_generator_teacher is True:
# data generator for on the fly training data manipulation
datagen = ImageDataGenerator(
# set input mean to 0 over the dataset
featurewise_center=False,
# set each sample mean to 0
samplewise_center=False,
# divide inputs by std of dataset
featurewise_std_normalization=False,
# divide each input by its std
samplewise_std_normalization=False,
# apply ZCA whitening
zca_whitening=False,
# epsilon for ZCA whitening
zca_epsilon=1e-06,
# randomly rotate images in the range (deg 0 to 180)
rotation_range=0,
# randomly shift images horizontally
width_shift_range=0.1,
# randomly shift images vertically
height_shift_range=0.1,
# set range for random shear
shear_range=0.,
# set range for random zoom
zoom_range=0.,
# set range for random channel shifts
channel_shift_range=0.,
# set mode for filling points outside the input boundaries
fill_mode='nearest',
# value used for fill_mode = "constant"
cval=0.,
# randomly flip images
horizontal_flip=True,
# randomly flip images
vertical_flip=False,
# set rescaling factor (applied before any other transformation)
rescale=None,
# set function that will be applied on each input
preprocessing_function=None,
# image data format, either "channels_first" or "channels_last"
data_format=None,
# fraction of images reserved for validation (strictly between 0 and 1)
validation_split=0.0)
datagen.fit(X_train)
try:
for order in order_combinations:
for temp in temperatures:
# clearing all saved teacher logits
for alpha in alphas:
tf.keras.backend.clear_session(
) # must clear the current session to free memory!
K.clear_session(
) # must clear the current session to free memory!
logger.info(
"Clearing tensorflow/keras backend session and de-allocating remaining models..."
)
model = None
previousModel = None
if teacher_name is not None:
ssm = model_loader(logger, options.teacherModel)
previousModel = ssm.get_loaded_model()
teacher_name = options.teacherModel
# creating experiment1 metadata
experiment_result = {
"experiment_results": []
} # empty space for our experiment1's data
experiment_metadata = create_meta(cfg.dataset,
teacher_name, epochs,
temp, alpha, order)
experiment_result['metadata'] = experiment_metadata
# performing experiment on given size, alpha, and temperature combination
for net_size in order:
model = None
# perform KD if there is a previously trained model to work with
if previousModel is not None:
model = knowledge_distillation_models.get_model(
cfg.dataset,
cfg.dataset_num_classes,
X_train,
net_size,
)
logger.info(
"loading soft targets for student training...")
print("previous model to load logits for: %s" %
str(previousModel))
teacher_train_logits, teacher_test_logits = get_pretrained_teacher_logits(
logits_dir, previousModel, alpha, cfg.dataset,
order)
Y_train_new, Y_test_new = teacher_utils.convert_logits_to_soft_targets(
temp, teacher_train_logits,
teacher_test_logits, Y_train, Y_test)
# # TODO remove next three lines
# file_name = "/home/blakete/" + temp + "_" + previousModel + "_training_labels.npy"
# filehandler = open(file_name, 'wb')
# pickle.dump(Y_train_new, filehandler)
# pickle.dump(Y_test_new, filehandler)
if Y_train_new is None or Y_test_new is None:
logger.info(
"soft targets not loaded correctly!")
else:
logger.info("completed")
# filehandler = open("mnist_10_soft_targets.pkl", 'wb')
# pickle.dump(Y_train_new, filehandler)
# pickle.dump(Y_test_new, filehandler)
model = helper_util.apply_knowledge_distillation_modifications(
logger, model, temp)
# model = multi_gpu_model(model, gpus=4)
optimizer = get_optimizer(
cfg.student_optimizer)
model.compile(
optimizer=optimizer,
loss=lambda y_true, y_pred: helper_util.
knowledge_distillation_loss(
logger, y_true, y_pred, alpha),
metrics=[helper_util.acc])
logger.info(
"training model...\norder:%s\nsize:%d\ntemp:%d\nalpha:%f"
% (order, net_size, temp, alpha))
callbacks = [
EarlyStopping(monitor='val_acc',
patience=50,
min_delta=0.00007),
# ReduceLROnPlateau(monitor='val_acc', factor=0.1, patience=4, min_lr=0.0001),
ModelCheckpoint(cfg.checkpoint_path,
monitor='val_acc',
verbose=1,
save_best_only=True,
mode='max')
]
if cfg.use_fit_generator_student is True:
model.fit(datagen.flow(
X_train,
Y_train_new,
batch_size=cfg.student_batch_size),
validation_data=(X_test,
Y_test_new),
epochs=epochs,
verbose=1,
callbacks=callbacks)
else:
model.fit(
X_train,
Y_train_new,
batch_size=cfg.student_batch_size,
epochs=epochs,
verbose=1,
callbacks=callbacks,
validation_data=(X_test, Y_test_new))
# model = helper_util.revert_knowledge_distillation_modifications(logger, model)
del model
# train_score, val_score = helper_util.calculate_unweighted_score(logger, model, X_train, Y_train,
# X_test, Y_test)
model = knowledge_distillation_models.get_model(
cfg.dataset,
cfg.dataset_num_classes,
X_train,
net_size,
)
# model.summary()
# load best model from checkpoint for evaluation
model.load_weights(cfg.checkpoint_path)
optimizer = get_optimizer(
cfg.student_optimizer)
model.compile(
optimizer=optimizer,
loss=
logloss, # the same as the custom loss function
metrics=['accuracy'])
train_score = model.evaluate(X_train,
Y_train,
verbose=0)
val_score = model.evaluate(X_test,
Y_test,
verbose=0)
result = create_result(net_size, temp, alpha,
train_score, val_score)
logger.info(result)
experiment_result["experiment_results"].append(
result)
# # remove checkpoint of best model for new checkpoint
# os.remove(cfg.checkpoint_path)
# save the trained model the saved model directory
saved_model(logger, cfg.dataset, net_size,
alpha, val_score, order, model,
models_dir)
if order.index(net_size) < len(order) - 1:
# save soft targets
logger.info(
"creating student training data...")
Y_train_new, Y_test_new = teacher_utils.createStudentTrainingData(
model, temp, X_train, Y_train, X_test,
Y_test)
save_pretrained_teacher_logits(
logits_dir, net_size, alpha,
Y_train_new, Y_test_new, cfg.dataset,
order)
logger.info("done.")
else:
logger.info(
"skipping creation of student training data, we are @ target model..."
)
# clear soft targets
Y_train_new = None
Y_test_new = None
# set model to current net size to preserve in previousModel
model = net_size
# if no previously trained model, train the network
else:
# load the already created soft targets
Y_train_new = None
Y_test_new = None
val_score = None
teacher_train_logits, teacher_test_logits = get_pretrained_teacher_logits(
logits_dir, net_size, alpha, cfg.dataset,
order)
# train network if not previously created logits
if teacher_train_logits is None or teacher_test_logits is None:
if os.path.isfile(cfg.checkpoint_path):
logger.info("removing previous checkpoint")
os.remove(cfg.checkpoint_path
) # remove previous checkpoint
logger.info(
"training teacher model...\norder:%s\nsize:%d\ntemp:%d\nalpha:%f"
% (order, net_size, temp, alpha))
model = knowledge_distillation_models.get_model(
cfg.dataset,
cfg.dataset_num_classes,
X_train,
net_size,
)
# model.summary()
optimizer = get_optimizer(
cfg.start_teacher_optimizer)
model.compile(
optimizer=optimizer,
loss=
logloss, # the same as the custom loss function
metrics=['accuracy'])
# train network and save model with bet validation accuracy to cfg.checkpoint_path
callbacks = [
EarlyStopping(monitor='val_acc',
patience=50,
min_delta=0.00007),
# ReduceLROnPlateau(monitor='val_acc', factor=0.1, patience=4, min_lr=0.0001),
ModelCheckpoint(cfg.checkpoint_path,
monitor='val_acc',
verbose=1,
save_best_only=True,
mode='max')
]
if cfg.use_fit_generator_teacher is True:
model.fit(datagen.flow(
X_train,
Y_train,
batch_size=cfg.student_batch_size),
validation_data=(X_test, Y_test),
epochs=epochs,
verbose=1,
callbacks=callbacks)
else:
model.fit(
X_train,
Y_train,
validation_data=(X_test, Y_test),
batch_size=cfg.student_batch_size,
epochs=epochs,
verbose=1,
callbacks=callbacks)
# load best model from checkpoint for evaluation
del model
model = knowledge_distillation_models.get_model(
cfg.dataset,
cfg.dataset_num_classes,
X_train,
net_size,
)
model.load_weights(cfg.checkpoint_path)
optimizer = get_optimizer(
cfg.start_teacher_optimizer)
model.compile(
optimizer=optimizer,
loss=
logloss, # the same as the custom loss function
metrics=['accuracy'])
# evaluate network
train_score, val_score = helper_util.calculate_unweighted_score(
logger, model, X_train, Y_train, X_test,
Y_test)
# save evaluation
result = create_result(net_size, None, None,
train_score, val_score)
logger.info(result)
experiment_result["experiment_results"].append(
result)
if len(order) != 1:
logger.info(
"creating student training data...")
teacher_train_logits, teacher_test_logits = teacher_utils.createStudentTrainingData(
model, temp, X_train, Y_train, X_test,
Y_test)
save_pretrained_teacher_logits(
logits_dir, net_size, alpha,
teacher_train_logits,
teacher_test_logits, cfg.dataset,
order)
logger.info(
"done creating student training data.")
# save the trained model the saved model directory
saved_model(logger, cfg.dataset, net_size,
alpha, val_score, order, model,
models_dir)
# # remove checkpoint of best model for new checkpoint
# os.remove(cfg.checkpoint_path)
else:
model = net_size
# temporarily serialize model to load as teacher in following KD training to avoid errors
del previousModel # free memory
previousModel = net_size # previously trained model becomes teacher
# appending experiment result to log file
if os.path.isfile(session_log_file):
open_type = 'a'
else:
open_type = 'w'
with open(session_log_file, open_type) as f:
f.write(json.dumps(experiment_result))
f.write("\n")
f.close()
# printing the results of training
logger.info(cfg.student_train_spacer)
# free model variables for next configuration iteration
del model
del previousModel
logger.info('-- COMPLETE')
except Exception:
traceback.print_exc()
error = traceback.format_exc()
# error.upper()
logging.error('Error encountered: %s' % error, exc_info=True)
#model.summary()
# Compile the model
opt = keras.optimizers.Adam(learning_rate=0.00001)
ls = keras.losses.binary_crossentropy
model.compile(loss=ls, optimizer=opt, metrics=['accuracy'])
my_callbacks=[
callbacks.EarlyStopping(patience = 15, monitor='val_loss', mode = 'min'),
callbacks.ModelCheckpoint(filepath='Task2_0707_AlexNet4_ash.h5', save_best_only=True)
]
###### Training
x_tr, x_val, y_tr, y_val = train_test_split(x_train_value, y_train, test_size = 0.2, random_state = 1)
aug = ImageDataGenerator(vertical_flip=True, horizontal_flip=True, rotation_range=90)
aug.fit(x_tr)
history = model.fit_generator(aug.flow(x_tr, y_tr, batch_size=64, shuffle=True),
epochs = 50, verbose = 1,
validation_data = (x_val, y_val),
callbacks = my_callbacks)
###### Evaluation
from sklearn.metrics import accuracy_score
from sklearn.metrics import roc_auc_score
from sklearn.metrics import confusion_matrix
from sklearn.preprocessing import binarize
import matplotlib.pyplot as plt
plt.figure(figsize=(10,8)).suptitle('Loss Plot', fontsize = 20)
plt.plot(history.history['loss'], label = 'train')
''' Images_load_length = Images.shape[0] train_images = Images[0:np.int(Images_load_length*train_ratio),:,:,:] test_images = Images[np.int(Images_load_length*train_ratio) :,:,:,:] train_labels = Labels[0:np.int( Images_load_length*train_ratio )] test_labels = Labels[np.int(Images_load_length*train_ratio) :] ''' # ============================================================================== # Data augment using keras data generator # ============================================================================== # datagen = ImageDataGenerator(horizontal_flip=True , vertical_flip=True , zoom_range = [1,1.2] , rotation_range=10, width_shift_range=0.1,height_shift_range=0.1) datagen = ImageDataGenerator(horizontal_flip=True , vertical_flip=True ) datagen.fit(train_images) # ============================================================================== # Define the neural network hierachy and classes # ============================================================================== model = Eff_net_B0_simplest(Net_file_path , 'All' , Checkpoint_network) # ============================================================================== # Define the learning variable and fit the model # ============================================================================== callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=True, min_delta=0.0001) model.summary() model.compile(optimizer=Adam(lr=0.0001),loss = 'binary_crossentropy', metrics=['accuracy']) history = model.fit_generator(datagen.flow(train_images, train_labels, batch_size=32, shuffle=True) , epochs=EPOCHS, callbacks = [callback], validation_data=(test_images, test_labels), verbose = 1)
def MobileNet(data, settings, width_multiplier=1.0, q=None):
model_type = 'MobileNetv2'
batch_size = settings[0]
epochs = settings[1]
data_augmentation = settings[2]
weight_precision = settings[5]
activations_prescision = settings[6]
x_train = data[0]
y_train = data[1]
x_test = data[2]
y_test = data[3]
x_train_mean = data[4]
input_shape = data[5]
num_classes = data[6]
depth = 26
k = settings[3]
"""MobileNetv2
This function defines a MobileNetv2 architectures.
# Arguments
input_shape: An integer or tuple/list of 3 integers, shape
of input tensor.
k: Integer, number of classes.
# Returns
MobileNetv2 model.
"""
global nlay
nlay = -1
inputs = Input(shape=input_shape)
x = _conv_block(inputs, roundup(int(32 * width_multiplier)), (3, 3), strides=(2, 2), use_bias=False)
nlay += 1
fix = 0
if width_multiplier - 1.3 < 0.01:
fix = -2
x = _inverted_residual_block(x, roundup(int(16 * width_multiplier)), (3, 3), t=1, strides=1, n=1)
x = _inverted_residual_block(x, roundup(int(24 * width_multiplier)), (3, 3), t=6, strides=2, n=2)
x = _inverted_residual_block(x, roundup(int(32 * width_multiplier)), (3, 3), t=6, strides=2, n=3)
x = _inverted_residual_block(x, roundup(int(64 * width_multiplier) + fix), (3, 3), t=6, strides=2, n=4)
x = _inverted_residual_block(x, roundup(int(96 * width_multiplier)), (3, 3), t=6, strides=1, n=3)
x = _inverted_residual_block(x, roundup(int(160 * width_multiplier)), (3, 3), t=6, strides=2, n=3)
x = _inverted_residual_block(x, roundup(int(320 * width_multiplier)), (3, 3), t=6, strides=1, n=1)
last_conv_size = max(1280, int(1280 * width_multiplier))
x = _conv_block(x, last_conv_size, (1, 1), strides=(1, 1), use_bias=False)
x = GlobalAveragePooling2D()(x)
x = Reshape((1, 1, last_conv_size))(x)
x = Dropout(0.3, name='Dropout')(x)
x = Conv2D(k, (1, 1), padding='same', name='logits', use_bias=True)(x)
x = Activation('softmax', name='softmax')(x)
outputs = Reshape((k,), name='out')(x)
model = Model(inputs=inputs, outputs=outputs)
#Quantize
# a/w: 8/8, 16/16, 16/8
if q == True:
model = QUANTIZER.apply_quantization(model, pruning_policy=None, weight_precision=weight_precision, activation_precision=activations_prescision,
activation_margin=None)
# plot_model(model, to_file='MobileNetv2.png', show_shapes=True)
#--------------------------------------------------------------------------------
#Get Output
num_layers = 156
all_layers = list()
for layer_index in range(1, num_layers):
all_layers.append(model.get_layer(name=None, index=layer_index).output)
intermediate_layer_model_input = model.input
intermediate_layer_model = Model(inputs=intermediate_layer_model_input, outputs=all_layers)
#---------------------------------------------------------------------------------
model.compile(loss='categorical_crossentropy',
optimizer=Adam(learning_rate=Settings.lr_schedule(0)),
metrics=['accuracy'])
model.summary()
print(model_type)
# Prepare model model saving directory.
save_dir = os.path.join(os.getcwd(), 'saved_models')
model_name = 'cifar10_%s_model.{epoch:03d}.h5' % model_type
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
filepath = os.path.join(save_dir, model_name)
# Prepare callbacks for model saving and for learning rate adjustment.
checkpoint = ModelCheckpoint(filepath=filepath,
monitor='val_acc',
verbose=1,
save_best_only=True)
lr_scheduler = LearningRateScheduler(Settings.lr_schedule)
lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1),
cooldown=0,
patience=5,
min_lr=0.5e-6)
callbacks = [checkpoint, lr_reducer, lr_scheduler]
# Run training, with or without data augmentation.
if not data_augmentation:
print('Not using data augmentation.')
model.fit(x_train, y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(x_test, y_test),
shuffle=True,
callbacks=callbacks)
else:
print('Using real-time data augmentation.')
# This will do preprocessing and realtime data augmentation:
datagen = ImageDataGenerator(
# set input mean to 0 over the dataset
featurewise_center=False,
# set each sample mean to 0
samplewise_center=False,
# divide inputs by std of dataset
featurewise_std_normalization=False,
# divide each input by its std
samplewise_std_normalization=False,
# apply ZCA whitening
zca_whitening=False,
# epsilon for ZCA whitening
zca_epsilon=1e-06,
# randomly rotate images in the range (deg 0 to 180)
rotation_range=0,
# randomly shift images horizontally
width_shift_range=0.1,
# randomly shift images vertically
height_shift_range=0.1,
# set range for random shear
shear_range=0.,
# set range for random zoom
zoom_range=0.,
# set range for random channel shifts
channel_shift_range=0.,
# set mode for filling points outside the input boundaries
fill_mode='nearest',
# value used for fill_mode = "constant"
cval=0.,
# randomly flip images
horizontal_flip=True,
# randomly flip images
vertical_flip=False,
# set rescaling factor (applied before any other transformation)
rescale=None,
# set function that will be applied on each input
preprocessing_function=None,
# image data format, either "channels_first" or "channels_last"
data_format=None,
# fraction of images reserved for validation (strictly between 0 and 1)
validation_split=0.0)
# Compute quantities required for featurewise normalization
# (std, mean, and principal components if ZCA whitening is applied).
datagen.fit(x_train)
# Fit the model on the batches generated by datagen.flow().
model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size),
validation_data=(x_test, y_test),
epochs=epochs, verbose=1, workers=4,
callbacks=callbacks)
#----------------------------------------------------------
'''
data = x_test
num_batches = data.shape[0] // batch_size
for batch_idx in range(num_batches):
start = batch_idx * batch_size
end = start + batch_size
intermediate_output = intermediate_layer_model.predict(data[start:end])
print("Intermediate result batch {}/{} done".format(batch_idx, num_batches))
print("Got intermediate, a random entry: {}".format(intermediate_output[0][0]))
'''
# Score trained model.
scores = model.evaluate(x_test, y_test, verbose=1)
print('Test loss:', scores[0])
print('Test accuracy:', scores[1])
#model.save("mobilenet_model.h5")
return model
cifar10 = tf.keras.datasets.cifar10
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0
imagetrain = ImageDataGenerator(rescale=1. / 1.,
rotation_range=45,
width_shift_range=.15,
height_shift_range=.15,
horizontal_flip=False,
zoom_range=0.5)
#x_train, x_test = x_train / 255.0, x_test / 255.0
imagetrain.fit(x_train)
class Baseline(Model):
def __init__(self):
super(Baseline, self).__init__()
self.c1 = Conv2D(filters=6, kernel_size=(5, 5), padding='same') # 卷积层
self.b1 = BatchNormalization() # BN层
self.a1 = Activation('relu') # 激活层
self.p1 = MaxPool2D(pool_size=(2, 2), strides=2, padding='same') # 池化层
self.d1 = Dropout(0.2) # dropout层
self.flatten = Flatten()
self.f1 = Dense(128, activation='relu')
self.d2 = Dropout(0.2)
self.f2 = Dense(10, activation='softmax')
def runExample(numEpochs, batchSize, dataAugmentation, directoryName, modelName):
"This function is based off of the cifar10_cnn.py example shown in class"
"Baseline test for this examples was: (numEpochs=100, NUM_CLASSES=10, batchSize=32, "
"dataAugmentation=True)"
"Create the save directory for this specific test/example"
saveDir = os.path.join(SAVE_DIR, directoryName)
"Set num_predictions to 20"
num_predictions = 20
"Create tensorboard callback"
tensorboard_callback = keras.callbacks.TensorBoard(log_dir=saveDir, histogram_freq=1, profile_batch = 10000000000)
# The data, split between train and test sets:
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
# Convert class vectors to binary class matrices.
y_train = keras.utils.to_categorical(y_train, NUM_CLASSES)
y_test = keras.utils.to_categorical(y_test, NUM_CLASSES)
model = Sequential()
model.add(Conv2D(32, (3, 3), padding='same', input_shape=x_train.shape[1:]))
model.add(Activation('relu'))
model.add(Conv2D(32, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Conv2D(64, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(Conv2D(64, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(NUM_CLASSES))
model.add(Activation('softmax'))
# initiate RMSprop optimizer
opt = keras.optimizers.RMSprop(learning_rate=0.0001, decay=1e-6)
# Let's train the model using RMSprop
model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
if not dataAugmentation:
print('Not using data augmentation.')
model.fit(x_train, y_train,
batch_size=batchSize,
epochs=numEpochs,
validation_data=(x_test, y_test),
shuffle=True,
callbacks=[tensorboard_callback]) # "Added tensorboard callback"
else:
print('Using real-time data augmentation.')
# This will do preprocessing and realtime data augmentation:
datagen = ImageDataGenerator(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
zca_epsilon=1e-06, # epsilon for ZCA whitening
rotation_range=0, # randomly rotate images in the range (degrees, 0 to 180)
# randomly shift images horizontally (fraction of total width)
width_shift_range=0.1,
# randomly shift images vertically (fraction of total height)
height_shift_range=0.1,
shear_range=0., # set range for random shear
zoom_range=0., # set range for random zoom
channel_shift_range=0., # set range for random channel shifts
# set mode for filling points outside the input boundaries
fill_mode='nearest',
cval=0., # value used for fill_mode = "constant"
horizontal_flip=True, # randomly flip images
vertical_flip=False, # randomly flip images
# set rescaling factor (applied before any other transformation)
rescale=None,
# set function that will be applied on each input
preprocessing_function=None,
# image data format, either "channels_first" or "channels_last"
data_format=None,
# fraction of images reserved for validation (strictly between 0 and 1)
validation_split=0.0)
# Compute quantities required for feature-wise normalization
# (std, mean, and principal components if ZCA whitening is applied).
datagen.fit(x_train)
# Fit the model on the batches generated by datagen.flow().
"Add the tensorboard callback"
model.fit_generator(datagen.flow(x_train, y_train,
batch_size=batchSize),
epochs=numEpochs,
validation_data=(x_test, y_test),
workers=4,
callbacks=[tensorboard_callback])
# Save model and weights
if not os.path.isdir(saveDir):
os.makedirs(saveDir)
model_path = os.path.join(saveDir, modelName)
model.save(model_path)
print('Saved trained model at %s ' % model_path)
# Score trained model.
scores = model.evaluate(x_test, y_test, verbose=1)
print('Test loss:', scores[0])
print('Test accuracy:', scores[1])
print()
def train(self):
# mirrored_strategy = tf.distribute.MirroredStrategy()
mirrored_strategy = tf.contrib.distribute.MirroredStrategy()
with mirrored_strategy.scope():
model = Sequential()
weight_decay = self.weight_decay
model.add(
Conv2D(64, (3, 3),
padding='same',
input_shape=self.x_shape,
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(Dropout(0.3))
model.add(
Conv2D(64, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(
Conv2D(128, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(Dropout(0.4))
model.add(
Conv2D(128, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(
Conv2D(256, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(Dropout(0.4))
model.add(
Conv2D(256, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(Dropout(0.4))
model.add(
Conv2D(256, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(
Conv2D(512, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(Dropout(0.4))
model.add(
Conv2D(512, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(Dropout(0.4))
model.add(
Conv2D(512, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(
Conv2D(512, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(Dropout(0.4))
model.add(
Conv2D(512, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(Dropout(0.4))
model.add(
Conv2D(512, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(
Dense(512, kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(self.num_classes))
model.add(Activation('softmax'))
# optimization details
learning_rate = 0.1
lr_decay = 1e-6
sgd = optimizers.SGD(lr=learning_rate,
decay=lr_decay,
momentum=0.9,
nesterov=True)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
# training parameters
batch_size = 128
maxepoches = 250
learning_rate = 0.1
lr_decay = 1e-6
lr_drop = 20
# The data, shuffled and split between train and test sets:
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train, x_test = self.normalize(x_train, x_test)
y_train = keras.utils.to_categorical(y_train, self.num_classes)
y_test = keras.utils.to_categorical(y_test, self.num_classes)
def lr_scheduler(epoch):
return learning_rate * (0.5**(epoch // lr_drop))
reduce_lr = keras.callbacks.LearningRateScheduler(lr_scheduler)
# data augmentation
datagen = ImageDataGenerator(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=
False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range=
15, # randomly rotate images in the range (degrees, 0 to 180)
width_shift_range=
0.1, # randomly shift images horizontally (fraction of total width)
height_shift_range=
0.1, # randomly shift images vertically (fraction of total height)
horizontal_flip=True, # randomly flip images
vertical_flip=False) # randomly flip images
# (std, mean, and principal components if ZCA whitening is applied).
datagen.fit(x_train)
# training process in a for loop with learning rate drop every 25 epoches.
'''
historytemp = model.fit_generator(datagen.flow(x_train, y_train,
batch_size=batch_size),
steps_per_epoch=x_train.shape[0] // batch_size,
epochs=maxepoches,
validation_data=(x_test, y_test), callbacks=[reduce_lr], verbose=2)
'''
for e in range(maxepoches):
print('Epoch', e)
batches = 0
for x_batch, y_batch in datagen.flow(x_train,
y_train,
batch_size=batch_size):
model.fit(x_batch, y_batch)
batches += 1
if batches >= len(x_train) / batch_size:
# we need to break the loop by hand because
# the generator loops indefinitely
break
model.save_weights('cifar10vgg.h5')
return model
fashion_mnist = tf.keras.datasets.fashion_mnist # 导入数据集
(x_train, y_train), (x_test, y_test) = fashion_mnist.load_data() # 分出训练集和测试集
x_test = x_test / 255.
x_train = x_train.reshape(x_train.shape[0], 28, 28, 1)
x_test = x_test.reshape(x_test.shape[0], 28, 28, 1)
image_gen_train = ImageDataGenerator(
rescale=1. / 255, #归至0～1
rotation_range=45, #随机45度旋转
width_shift_range=.15, #宽度偏移
height_shift_range=.15, #高度偏移
horizontal_flip=True, #水平翻转
zoom_range=0.5 #将图像随机缩放到50％
)
image_gen_train.fit(x_train)
model = tf.keras.models.Sequential([
Conv2D(input_shape=(28, 28, 1),
filters=32,
kernel_size=(5, 5),
padding='same'), # 卷积层
BatchNormalization(), # BN层
Activation('relu'), # 激活层
MaxPool2D(pool_size=(2, 2), strides=2, padding='same'), # 池化层
Dropout(0.2), # dropout层
Conv2D(64, kernel_size=(5, 5), padding='same'),
BatchNormalization(),
Activation('relu'),
MaxPool2D(pool_size=(2, 2), strides=2, padding='same'),
Dropout(0.2),
def train_and_save_network(attr=None):
if attr is None: attr=sys.argv[1]
assert (attr is not None)
subject = f'cnn.py {attr} {datetime.datetime.now()}'
subprocess.run('> nohup.out', shell=True)
os.environ['TF_ENABLE_AUTO_MIXED_PRECISION'] = '1'
GPU = get_available_device()
print('* * * device to be used:%s * * *'%GPU)
globals().update(get_data(attr=attr))
model = create_model(x_train, y_train)
callback = CallbackMailer(subject)
if not data_augmentation:
print('Not using data augmentation.')
with tf.device(GPU):
hist = model.fit(x_train, y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(x_valid, y_valid),
shuffle=True,
callbacks = [callback])
else:
print('Using real-time data augmentation.')
# preprocessing and realtime data augmentation
datagen = ImageDataGenerator(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
zca_epsilon=1e-06, # epsilon for ZCA whitening
rotation_range=0, # randomly rotate images in the range (degrees, 0 to 180)
# randomly shift images horizontally (fraction of total width)
width_shift_range=0.1,
# randomly shift images vertically (fraction of total height)
height_shift_range=0.1,
shear_range=0., # set range for random shear
zoom_range=0., # set range for random zoom
channel_shift_range=0., # set range for random channel shifts
# set mode for filling points outside the input boundaries
fill_mode='nearest',
cval=0., # value used for fill_mode = "constant"
horizontal_flip=True, # randomly flip images
vertical_flip=False, # randomly flip images
# set rescaling factor (applied before any other transformation)
rescale=None,
# set function that will be applied on each input
preprocessing_function=None,
# image data format, either "channels_first" or "channels_last"
data_format=None,
# fraction of images reserved for validation (strictly between 0 and 1)
validation_split=0.0)
# Compute quantities required for feature-wise normalization
# (std, mean, and principal components if ZCA whitening is applied).
datagen.fit(x_train)
with tf.device(GPU):
# Fit the model on the batches generated by datagen.flow().
hist = model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size),
epochs=epochs,
validation_data=(x_valid, y_valid))
# Save model and weights
print('saving model... ',end='',flush=True)
model_path = raw_model_path.format(attr=attr)
model.save(model_path)
print('done! >>> Saved trained model at {}'.format(model_path))
write_model_run(hist, model, attr, callback, subject)
# Score trained model.
scores = np.array(model.evaluate(x_test, y_test, verbose=1))#df.loc['test'].values
print('Test loss:', scores[0])
print('Test accuracy:', scores[1])
mail(
subject = subject,
message = "cnn.py is finished running: loss, accuracy = {}".format(scores.tolist())
)
dataset = load_dataset(validation_size=validation_size,
test_size=test_size,
split=False,
img_size=[img_rows, img_cols])
datagen_train = ImageDataGenerator(featurewise_center=True,
featurewise_std_normalization=True,
rotation_range=20,
width_shift_range=0.2,
height_shift_range=0.2,
horizontal_flip=True)
datagen_test = ImageDataGenerator(featurewise_center=True,
featurewise_std_normalization=True)
datagen_train.fit(dataset['x_train'])
datagen_test.fit(dataset['x_train'])
def prot_div(left, right):
if right == 0:
return 0
else:
return left / right
def if_func(condition, state1, state2):
if condition:
return state1
else:
return state2
print('Weight for class 0: {:.2f}'.format(weight_for_0))
print('Weight for class 1: {:.2f}'.format(weight_for_1))
print("#######################\n"
"# DATA AUGMENTATION #\n"
"#######################\n")
datagen = ImageDataGenerator(horizontal_flip=hflips,
vertical_flip=vflips,
rotation_range=rotation_range,
shear_range=k_shear_range,
width_shift_range=k_w_shift_r,
height_shift_range=k_h_shift_r,
fill_mode='reflect')
datagen.fit(x_train, augment=True, seed=seed)
print("#################################\n"
"# BUILD AND TRAIN THE NETWORK #\n"
"#################################\n")
optim = tf.keras.optimizers.Adam(learning_rate=learning_rate_value)
# Create model
model = create_oztel_model(optimizer=optim, loss=loss)
# Check the network created
model.summary(line_length=150)
os.makedirs(char_dir, exist_ok=True)
model_name = os.path.join(char_dir, "model_plot_" + job_identifier + ".png")
plot_model(model, to_file=model_name, show_shapes=True, show_layer_names=True)
def ResNetTrainer(network, data, mypath, batchsize, maxepochs):
datagen = ImageDataGenerator(
# set input mean to 0 over the dataset
featurewise_center=False,
# set each sample mean to 0
samplewise_center=False,
# divide inputs by std of dataset
featurewise_std_normalization=False,
# divide each input by its std
samplewise_std_normalization=False,
# apply ZCA whitening
zca_whitening=False,
# epsilon for ZCA whitening
zca_epsilon=1e-06,
# randomly rotate images in the range (deg 0 to 180)
rotation_range=0,
# randomly shift images horizontally
width_shift_range=0.1,
# randomly shift images vertically
height_shift_range=0.1,
# set range for random shear
shear_range=0.,
# set range for random zoom
zoom_range=0.,
# set range for random channel shifts
channel_shift_range=0.,
# set mode for filling points outside the input boundaries
fill_mode='nearest',
# value used for fill_mode = "constant"
cval=0.,
# randomly flip images
horizontal_flip=True,
# randomly flip images
vertical_flip=False,
# set rescaling factor (applied before any other transformation)
rescale=None,
# set function that will be applied on each input
preprocessing_function=None,
# image data format, either "channels_first" or "channels_last"
data_format=None,
# fraction of images reserved for validation (strictly between 0 and 1)
validation_split=0.0)
# Compute quantities required for featurewise normalization
# (std, mean, and principal components if ZCA whitening is applied).
Xtrain, Ytrain, Xval, Yval, Xtest, Ytest, nclasses = data
datagen.fit(Xtrain)
epoch = 0
print("\nEvaluate network with no training:")
TrainL0, TrainA0 = network.evaluate(Xtrain,
Ytrain,
batch_size=200,
verbose=2)
ValL0, ValA0 = network.evaluate(Xval, Yval, batch_size=200, verbose=2)
TestL0, TestA0 = network.evaluate(Xtest, Ytest, batch_size=200, verbose=2)
TrainLoss = np.asarray([TrainL0])
TrainAccuracy = np.asarray([TrainA0])
TestLoss = np.asarray([TestL0])
TestAccuracy = np.asarray([TestA0])
ValLoss = np.asarray([ValL0])
ValAccuracy = np.asarray([ValA0])
mask_perlayer, remaining, total = getpercentages(network)
RemainingWeights = np.asarray([remaining])
maxtrainacc = 0
maxvalacc = 0
maxtestacc = 0
lr = 1e-3
while epoch < maxepochs:
start_time = time.time()
loss, metric = network.metrics_names
if epoch == 80:
lr = 1e-4
kb.set_value(network.optimizer.lr, lr)
if epoch == 120:
lr = 1e-5
kb.set_value(network.optimizer.lr, lr)
if epoch == 160:
lr = 1e-6
kb.set_value(network.optimizer.lr, lr)
print('Standard learning rate: ', lr)
fit_history = network.fit_generator(datagen.flow(Xtrain,
Ytrain,
batch_size=batchsize),
validation_data=(Xtest, Ytest),
epochs=1,
verbose=1,
workers=1,
shuffle=True)
TrainLoss = np.append(TrainLoss, fit_history.history[loss])
ValLoss = np.append(ValLoss, fit_history.history['val_loss'])
TrainAccuracy = np.append(TrainAccuracy, fit_history.history[metric])
ValAccuracy = np.append(ValAccuracy,
fit_history.history['val_' + metric])
maxtrainacc = max(maxtrainacc, TrainAccuracy[-1])
maxtestacc = max(maxtestacc, ValAccuracy[-1])
maxvalacc = max(maxvalacc, ValAccuracy[-1])
print("\nepoch {}/{}".format(epoch + 1, maxepochs))
print("batchsize =", batchsize)
print("trn loss = {:.7f}".format(TrainLoss[-1]))
print("val loss = {:.7f}".format(ValLoss[-1]))
print("tst loss = {:.7f}".format(TestLoss[-1]))
print("trn {} = {:.7f}, best {:.7f}".format(metric,
TrainAccuracy[-1],
maxtrainacc))
print("val {} = {:.7f}, best {:.7f}".format(metric, ValAccuracy[-1],
maxvalacc))
print("tst {} = {:.7f}, best {:.7f}".format(metric, ValAccuracy[-1],
maxtestacc))
mask_perlayer, remaining, total = getpercentages(network)
RemainingWeights = np.append(RemainingWeights, remaining)
epoch += 1
print("Output:", mypath)
Logs = {
"trainLoss": TrainLoss,
"testLoss": ValLoss,
"valLoss": ValLoss,
"trainAccuracy": TrainAccuracy,
"testAccuracy": ValAccuracy,
"valAccuracy": ValAccuracy,
"remainingWeights": RemainingWeights
}
file = open(mypath + "TrainLogs.pkl", "wb")
pickle.dump(Logs, file)
file.close()
print("Execution time: {:.3f} seconds".format(time.time() -
start_time))
print("=" * (len(mypath) + 8))
file = open(mypath + "Masks.pkl", "wb")
pickle.dump(getmasks(network), file)
file.close()
W = []
P = 1
for l in range(1, len(network.layers)):
w = network.layers[l].get_weights()
# m = network.layers[l].get_mask()
if isinstance(w, list):
W.append([])
continue
# print(w.shape)
print("maxw=", np.max(w), "minw=", np.min(w))
# print("maxm=", np.max(m), "minm=", np.min(m))
# print(np.max(m), np.min(m))
P *= np.max(w)
W.append(w)
# print(w)
print("product of all layer's weights:", P)
file = open(mypath + "Weights.pkl", "wb")
pickle.dump(W, file)
file.close()
return Logs
X_train = X_train.reshape(X_train.shape[0], X_train.shape[1], X_train.shape[2],
1)
X_validation = X_validation.reshape(X_validation.shape[0],
X_validation.shape[1],
X_validation.shape[2], 1)
X_test = X_test.reshape(X_test.shape[0], X_test.shape[1], X_test.shape[2], 1)
############################### AUGMENTATAION OF IMAGES: TO MAKEIT MORE GENERIC
dataGen = ImageDataGenerator(
width_shift_range=
0.1, # 0.1 = 10% IF MORE THAN 1 E.G 10 THEN IT REFFERS TO NO. OF PIXELS EG 10 PIXELS
height_shift_range=0.1,
zoom_range=0.2, # 0.2 MEANS CAN GO FROM 0.8 TO 1.2
shear_range=0.1, # MAGNITUDE OF SHEAR ANGLE
rotation_range=10) # DEGREES
dataGen.fit(X_train)
batches = dataGen.flow(
X_train, y_train, batch_size=20
) # REQUESTING DATA GENRATOR TO GENERATE IMAGES BATCH SIZE = NO. OF IMAGES CREAED EACH TIME ITS CALLED
X_batch, y_batch = next(batches)
# TO SHOW AGMENTED IMAGE SAMPLES
fig, axs = plt.subplots(1, 15, figsize=(20, 5))
fig.tight_layout()
for i in range(15):
axs[i].imshow(X_batch[i].reshape(imageDimesion[0], imageDimesion[1]))
axs[i].axis('off')
plt.show()
y_train = to_categorical(y_train, noOfClasses)
!ls
X = cv2.imread('ferrari-spider-indian-theluxecafe.jpg')
X = cv2.cvtColor(X, cv2.COLOR_BGR2RGB)
plt.imshow(X)
print(X.shape)
IMAGE_SIZE = X.shape
X = np.expand_dims(X, axis=0)
print(X.shape)
y = np.ndarray([1])
print(y.shape)
from tensorflow.keras.preprocessing.image import ImageDataGenerator
datagen = ImageDataGenerator(zoom_range = [1,6])
datagen.fit(X)
batch_size = 10
X_batch = datagen.flow(X, batch_size=batch_size)
print(X_batch)
show(X_batch, batch_size)
#history = model.fit(cnnInput, y_train_hot_encoded, batch_size=256, epochs=50, shuffle=True, validation_split=0.1)
model.summary()
# Data augmentation
# this is the augmentation configuration we will use for training
train_datagen = ImageDataGenerator(width_shift_range=0.2,
height_shift_range=0.2,
zoom_range=[.6, 1],
vertical_flip=True,
horizontal_flip=True)
train_generator = train_datagen.flow(cnnInput, y_train, batch_size=64, seed=11)
valid_datagen = ImageDataGenerator()
valid_generator = valid_datagen.flow(cnnValidation,
y_valid,
batch_size=64,
seed=11)
train_datagen.fit(cnnInput)
valid_datagen.fit(cnnValidation)
model.fit_generator(train_datagen.flow(cnnInput,
y_train_hot_encoded,
batch_size=64),
shuffle=True,
validation_data=valid_datagen.flow(cnnValidation,
y_test_hot_encoded,
batch_size=64),
class_weight=class_weights,
epochs=20)
# aug_image_batch, aug_label_batch = next(augmented_data_gen)
val_batch, val_label_batch = next(val_data_gen)
X_train = image_batch
y_train = label_batch
X_valid = val_batch
y_valid = val_label_batch
#data augmentation
train_datagen = ImageDataGenerator(
rotation_range=15,
width_shift_range=0.15,
height_shift_range=0.15,
shear_range=0.15,
zoom_range=0.15,
horizontal_flip=True,
)
train_datagen.fit(X_train)
batch_size = 128
epochs = 50
early_stopping = EarlyStopping(
monitor='val_accuracy',
min_delta=0.00005,
patience=11,
verbose=1,
restore_best_weights=True,
)
lr_scheduler = ReduceLROnPlateau(
monitor='val_accuracy',
factor=0.5,
def get_augmented(
X_train,
Y_train,
X_val=None,
Y_val=None,
batch_size=32,
seed=0,
data_gen_args=dict(
rotation_range=10.0,
# width_shift_range=0.02,
height_shift_range=0.02,
shear_range=5,
# zoom_range=0.3,
horizontal_flip=True,
vertical_flip=False,
fill_mode="constant",
),
):
"""[summary]
Args:
X_train (numpy.ndarray): [description]
Y_train (numpy.ndarray): [description]
X_val (numpy.ndarray, optional): [description]. Defaults to None.
Y_val (numpy.ndarray, optional): [description]. Defaults to None.
batch_size (int, optional): [description]. Defaults to 32.
seed (int, optional): [description]. Defaults to 0.
data_gen_args ([type], optional): [description]. Defaults to dict(rotation_range=10.0,# width_shift_range=0.02,height_shift_range=0.02,shear_range=5,# zoom_range=0.3,horizontal_flip=True,vertical_flip=False,fill_mode="constant",).
Returns:
[type]: [description]
"""
# Train data, provide the same seed and keyword arguments to the fit and flow methods
X_datagen = ImageDataGenerator(**data_gen_args)
Y_datagen = ImageDataGenerator(**data_gen_args)
X_datagen.fit(X_train, augment=True, seed=seed)
Y_datagen.fit(Y_train, augment=True, seed=seed)
X_train_augmented = X_datagen.flow(X_train,
batch_size=batch_size,
shuffle=True,
seed=seed)
Y_train_augmented = Y_datagen.flow(Y_train,
batch_size=batch_size,
shuffle=True,
seed=seed)
train_generator = zip(X_train_augmented, Y_train_augmented)
if not (X_val is None) and not (Y_val is None):
# Validation data, no data augmentation, but we create a generator anyway
X_datagen_val = ImageDataGenerator(**data_gen_args)
Y_datagen_val = ImageDataGenerator(**data_gen_args)
X_datagen_val.fit(X_val, augment=True, seed=seed)
Y_datagen_val.fit(Y_val, augment=True, seed=seed)
X_val_augmented = X_datagen_val.flow(X_val,
batch_size=batch_size,
shuffle=True,
seed=seed)
Y_val_augmented = Y_datagen_val.flow(Y_val,
batch_size=batch_size,
shuffle=True,
seed=seed)
# combine generators into one which yields image and masks
val_generator = zip(X_val_augmented, Y_val_augmented)
return train_generator, val_generator
else:
return train_generator
def train():
print('Loading and preprocessing train data...')
print('-' * 30)
imgs_train, imgs_mask_train = load_train_data()
print('Loaded train images: ', imgs_train.shape, imgs_mask_train.shape)
print('-' * 30)
# Normalization of the train set #1 and #3
imgs_train = imgs_train.astype('float32')
mean = np.mean(imgs_train) # mean for data centering
std = np.std(imgs_train) # std for data normalization
imgs_train -= mean
imgs_train /= std
imgs_mask_train = imgs_mask_train.astype('float32')
# Normalization of the train set #2 and #4
# imgs_train = rescale_intensity(imgs_train, in_range=(0, 1))
# imgs_train = imgs_train.astype(np.float32)
# imgs_mask_train = imgs_mask_train.astype(np.float32)
print('Train test split')
X_train, X_test, y_train, y_test = train_test_split(imgs_train,
imgs_mask_train,
test_size=0.1)
print('-' * 30)
print('Data Augmentation Start')
data_gen_args = dict(shear_range=0.1,
rotation_range=20,
width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=0.3,
fill_mode='constant',
horizontal_flip=True,
vertical_flip=True,
cval=0)
image_datagen = ImageDataGenerator(**data_gen_args)
mask_datagen = ImageDataGenerator(**data_gen_args)
seed = 1
image_datagen.fit(X_train, augment=True, seed=seed)
mask_datagen.fit(y_train, augment=True, seed=seed)
image_generator = image_datagen.flow(X_train, batch_size=BATCH_SIZE)
mask_generator = mask_datagen.flow(y_train, batch_size=BATCH_SIZE)
train = zip(image_generator, mask_generator)
# val = zip(X_test, y_test)
print('-' * 30)
print('Data Augmentation End')
print('-' * 30)
print('Creating and compiling model...')
print('-' * 30)
model = unet()
#Saving the weights and the loss of the best predictions we obtained
model_checkpoint = ModelCheckpoint(
'/data/flavio/anatiel/models/dissertacao/unet_exp1_100epc_lungseg_32bits_augment_best.h5',
monitor='val_loss',
save_best_only=True,
mode="min")
print('Fitting model...')
print('-' * 30)
history = model.fit(
train,
batch_size=BATCH_SIZE,
epochs=EPOCHS,
verbose=1,
shuffle=True,
validation_data=(X_test, y_test),
# validation_split=0.1,
steps_per_epoch=X_train.shape[0],
callbacks=[model_checkpoint])
model.save(
'/data/flavio/anatiel/models/dissertacao/unet_exp1_100epc_lungseg_32bits_augment_last.h5'
)
# convert the history.history dict to a pandas DataFrame:
hist_df = pd.DataFrame(history.history)
# save to json:
hist_json_file = '/data/flavio/anatiel/models/dissertacao/unet_exp1_100epc_lungseg_32bits_augment_history.json'
with open(hist_json_file, mode='w') as f:
hist_df.to_json(f)
print("history saved")
plt.plot(history.history['dice_coef'])
plt.plot(history.history['val_dice_coef'])
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('Model dice coeff')
plt.ylabel('Dice coeff')
plt.xlabel('Epoch')
plt.legend(['Train', 'Val', 'Loss', 'Val Loss'], loc='upper left')
# save plot to file
plt.savefig(
'/data/flavio/anatiel/models/dissertacao/unet_exp1_100epc_lungseg_32bits_augment_plot.png'
)
def prepare_data(trainX):
#data augmentation
datagen = ImageDataGenerator(width_shift_range=0.1, \
rotation_range=45, height_shift_range=0.1, horizontal_flip=True)
datagen.fit(trainX)
return datagen
# 7. example
(x_train, y_train), (x_test, y_test) = tf.keras.datasets.cifar10.load_data()
y_train = tf.keras.utils.to_categorical(y_train, num_classes=10)
y_test = tf.keras.utils.to_categorical(y_test, num_classes=10)
datagen = ImageDataGenerator(featurewise_center=True,
featurewise_std_normalization=True,
rotation_range=20,
width_shift_range=0.2,
height_shift_range=0.2,
horizontal_flip=True,
vertical_flip=True,
fill_mode="constant",
cval=0)
datagen.fit(x_train)
# simple cnn
inputs = tf.keras.Input(shape=(32, 32, 3))
x = tf.keras.layers.Conv2D(filters=16,
kernel_size=(3, 3),
activation='relu',
input_shape=(32, 32, 3),
strides=(1, 1),
name='Conv2D_layer1')(inputs)
x = (tf.keras.layers.MaxPooling2D((2, 2), name='Maxpooling1_2D'))(x)
x = tf.keras.layers.Conv2D(filters=16,
kernel_size=(3, 3),
activation='relu',
strides=(1, 1),
name='Conv2D_layer2')(x)
def get_dataset(config):
"""Get dataset, either from ``.npz`` files or ``keras.ImageDataGenerator``.
Returns Dictionaries with keys ``x_test`` and ``y_test`` if data set was
loaded in ``.npz`` format, or with ``dataflow`` key if data will be loaded
from ``.jpg``, ``.png``, or ``.bmp`` files by a
``keras.ImageDataGenerator``.
Parameters
----------
config: configparser.ConfigParser
Settings.
Returns
-------
normset: dict
Used to normalized the network parameters.
testset: dict
Used to test the networks.
"""
testset = {}
normset = try_get_normset_from_scalefacs(config)
dataset_path = config.get('paths', 'dataset_path')
is_testset_needed = config.getboolean('tools', 'evaluate_ann') or \
config.getboolean('tools', 'simulate') or \
config.getboolean('loihi', 'normalize_thresholds', fallback=False)
is_normset_needed = config.getboolean('tools', 'normalize') and \
normset is None
# _______________________________ Keras __________________________________#
try:
keras_dataset = config.get('input', 'keras_dataset')
if keras_dataset:
from keras_rewiring.utilities.load_dataset \
import load_and_preprocess_dataset
num_to_test = config.getint('simulation', 'num_to_test')
data = load_and_preprocess_dataset(keras_dataset)
x_test, y_test = data['test']
testset = {
'x_test': x_test[:num_to_test],
'y_test': y_test[:num_to_test]}
if is_normset_needed:
normset = {'x_norm': x_test}
return normset, testset
except (NoOptionError, ImportError) as e:
print("Warning:", e)
# ________________________________ npz ___________________________________#
if config.get('input', 'dataset_format') == 'npz':
print("Loading data set from '.npz' files in {}.\n".format(
dataset_path))
if is_testset_needed:
num_to_test = config.getint('simulation', 'num_to_test')
testset = {
'x_test': load_npz(dataset_path, 'x_test.npz')[:num_to_test],
'y_test': load_npz(dataset_path, 'y_test.npz')[:num_to_test]}
assert testset, "Test set empty."
if is_normset_needed:
normset = {'x_norm': load_npz(dataset_path, 'x_norm.npz')}
assert normset, "Normalization set empty."
# ________________________________ jpg ___________________________________#
elif config.get('input', 'dataset_format') in {'jpg', 'png'}:
from tensorflow.keras.preprocessing.image import ImageDataGenerator
print("Loading data set from ImageDataGenerator, using images in "
"{}.\n".format(dataset_path))
# Transform str to dict
datagen_kwargs = eval(config.get('input', 'datagen_kwargs'))
dataflow_kwargs = eval(config.get('input', 'dataflow_kwargs'))
# Get class labels
class_idx_path = config.get('paths', 'class_idx_path')
if class_idx_path != '':
class_idx = json.load(open(os.path.abspath(class_idx_path)))
dataflow_kwargs['classes'] = \
[class_idx[str(idx)][0] for idx in range(len(class_idx))]
# Get proprocessing function
if 'preprocessing_function' in datagen_kwargs:
from snntoolbox.utils.utils import import_helpers
helpers = import_helpers(datagen_kwargs['preprocessing_function'],
config)
datagen_kwargs['preprocessing_function'] = \
helpers.preprocessing_function
dataflow_kwargs['directory'] = dataset_path
if 'batch_size' not in dataflow_kwargs:
dataflow_kwargs['batch_size'] = config.getint('simulation',
'batch_size')
datagen = ImageDataGenerator(**datagen_kwargs)
if (datagen.featurewise_center or datagen.featurewise_std_normalization
or datagen.zca_whitening):
# Compute quantities required for featurewise normalization
# (std, mean, and principal components if ZCA whitening is applied)
rs = datagen_kwargs.get('rescale', None)
x_orig = ImageDataGenerator(rescale=rs).flow_from_directory(
**dataflow_kwargs).next()[0]
datagen.fit(x_orig)
if is_normset_needed:
shuffle = dataflow_kwargs.get('shuffle')
dataflow_kwargs['shuffle'] = True
normset = {
'dataflow': datagen.flow_from_directory(**dataflow_kwargs)}
dataflow_kwargs['shuffle'] = shuffle
assert normset, "Normalization set empty."
if is_testset_needed:
testset = {
'dataflow': datagen.flow_from_directory(**dataflow_kwargs)}
assert testset, "Test set empty."
# _______________________________ aedat __________________________________#
elif config.get('input', 'dataset_format') == 'aedat':
if is_normset_needed:
normset = {'x_norm': load_npz(dataset_path, 'x_norm.npz')}
assert normset, "Normalization set empty."
testset = {}
return normset, testset
