os.environ["CUDA_VISIBLE_DEVICES"] = params.gpu
import tensorflow as tf
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = params.mem
set_session(tf.Session(config=config))
model = ResnetBuilder.build_resnet_cifar10(params.ly, use_bn=params.bn)
sgd = SGD(momentum=0.9)
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
dir_name = "e" + datetime.now().strftime("%m%d-%H-%M-%S") + \
"l" + str(params.ly) + \
"_ing"
csv_logger = CSVLogger(os.path.join(dir_name, 'log.csv'))
lr_scheduler = LearningRateScheduler(lambda x: 0.05 if x < 60 else 0.03 if x < 80 else 0.005)
board = TensorBoard(log_dir=dir_name, histogram_freq=0, write_graph=False, write_images=False)
checker = ModelCheckpoint(filepath=os.path.join(dir_name, "weights.{epoch:02d}-{val_loss:.2f}.hdf5"), period=20)
if not os.path.exists(dir_name): os.mkdir(dir_name)
with open(os.path.join(dir_name, 'config'), 'w') as wf:
wf.write("-----" + str(params) + dir_name)
callbacks = [lr_scheduler, csv_logger, board, checker]
datagen_train = ImageDataGenerator(
width_shift_range=params.shift,
height_shift_range=params.shift,
)
datagen_test = ImageDataGenerator()
model.fit_generator(datagen_train.flow(x_train, y_train, batch_size=params.bs),
steps_per_epoch=x_train.shape[0] / params.bs,
validation_data=datagen_test.flow(x_test, y_test, batch_size=params.bs),
validation_steps=x_test.shape[0] / params.bs,
def train_model_from_images(network, model_train_params,
train_data_dir, validation_data_dir,
verbose=0, tb_logs=False, csv_log=False,
early_stopping=False,
save_to_s3=False):
""" Train and validate the Convolutional Neural Network (CNN) model
by reading data from image folders. """
# dimensions of our images.
img_width, img_height = network.input_dim[1:]
loss = model_train_params['loss']
optimizer = model_train_params['optimizer']
metrics = model_train_params['metrics']
batch_size = model_train_params['batch_size']
nb_epochs = model_train_params['nb_epochs']
nb_train_samples = model_train_params['nb_train_samples']
nb_validation_samples = model_train_params['nb_validation_samples']
train_datagen = ImageDataGenerator(rescale=1.0/255)
train_generator = \
train_datagen.flow_from_directory(train_data_dir,
target_size=(img_width, img_height),
batch_size=100,
classes=['0', '1', '2', '3', '4',
'5', '6', '7', '8', '9'],
class_mode='categorical', seed=131)
total_samples = 30000
image_set = np.empty([0, 3, img_width, img_height])
label_set = np.empty([0, 10])
for i in range(total_samples/batch_size):
(X, y) = train_generator.next()
image_set = np.concatenate((image_set, X), axis=0)
label_set = np.concatenate((label_set, y), axis=0)
# this is the augmentation configuration we will use for training
train_datagen = ImageDataGenerator(rescale=1.0/255,
samplewise_center=False,
featurewise_center=True)
train_datagen.fit(image_set)
# this is the augmentation configuration we will use for testing:
# only rescaling
val_datagen = ImageDataGenerator(rescale=1.0/255,
samplewise_center=False,
featurewise_center=True)
val_datagen.fit(image_set)
train_generator = \
train_datagen.flow_from_directory(train_data_dir,
target_size=(img_width, img_height),
batch_size=batch_size,
classes=['0', '1', '2', '3', '4',
'5', '6', '7', '8', '9'],
class_mode='categorical')
validation_generator = \
val_datagen.flow_from_directory(validation_data_dir,
target_size=(img_width, img_height),
batch_size=batch_size,
classes=['0', '1', '2', '3', '4',
'5', '6', '7', '8', '9'],
class_mode='categorical')
if optimizer == 'sgd':
optimizer = SGD(lr=model_train_params['lr'],
momentum=model_train_params['momentum'],
decay=model_train_params['decay'],
nesterov=model_train_params['nesterov'])
elif optimizer == 'adam':
optimizer = Adam(lr=model_train_params['lr'])
network.model.compile(loss=loss,
metrics=metrics,
optimizer=optimizer)
callbacks = []
if tb_logs:
tb_callback = TensorBoard(log_dir='./logs', histogram_freq=1,
write_graph=True)
callbacks = [tb_callback]
if early_stopping:
early_stopping = EarlyStopping(monitor='val_acc',
patience=2, min_delta=0.005,
verbose=0, mode='auto')
callbacks.append(early_stopping)
if csv_log:
csv_filename = (network.name + '_lr_' + str(model_train_params['lr']) +
'_l2weightdecay_' + str(model_train_params['decay']) + '.csv')
csv_logger_callback = CSVLogger(filename=csv_filename, append=True)
callbacks.append(csv_logger_callback)
if save_to_s3:
s3 = boto3.resource('s3')
else:
s3 = None
def scheduler(epoch):
init_lr = model_train_params['lr']
return init_lr * (0.9**epoch)
change_lr = LearningRateScheduler(scheduler)
callbacks.append(change_lr)
# model_checkpoint_cb = ModelCheckpoint2S3(filepath=network.name + '_' + time.strftime("%x") + '.h5',
# monitor='val_loss',
# save_best_only=True,
# verbose=0,
# s3resource=s3,
# s3filename=network.name + '_' + time.strftime("%x") + '.h5')
model_checkpoint_cb = ModelCheckpoint(filepath=network.name + '.{epoch:02d}-{val_loss:.2f}.hdf5', monitor='val_loss')
callbacks.append(model_checkpoint_cb)
loss_history = LossHistory(filepath=network.name + '_lr_' + str(model_train_params['lr']) + '_' + time.strftime("%x").replace("/", "_") + "_trainloss.csv",
s3resource=s3, s3filename=network.name + '_lr_' + str(model_train_params['lr']) + '_' + time.strftime("%x").replace("/", "_") + "_trainloss.csv")
callbacks.append(loss_history)
history = network.model.fit_generator(train_generator,
samples_per_epoch=nb_train_samples,
nb_epoch=nb_epochs,
validation_data=validation_generator,
nb_val_samples=nb_validation_samples,
callbacks=callbacks,
verbose=1)
return history
#data augmentation
datagen = ImageDataGenerator(
rotation_range=15,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
)
datagen.fit(x_train)
#training
batch_size = 64
opt_rms = keras.optimizers.rmsprop(lr=0.001, decay=1e-6)
model.compile(loss='categorical_crossentropy',
optimizer=opt_rms,
metrics=['accuracy'])
model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size),\
steps_per_epoch=x_train.shape[0] // batch_size,epochs=125,\
verbose=1,validation_data=(x_test,y_test),callbacks=[LearningRateScheduler(lr_schedule)])
#save to disk
model_json = model.to_json()
with open('model.json', 'w') as json_file:
json_file.write(model_json)
model.save_weights('model.h5')
#testing
scores = model.evaluate(x_test, y_test, batch_size=128, verbose=1)
print('\nTest result: %.3f loss: %.3f' % (scores[1] * 100, scores[0]))
time_loss_weights[0] = 0
prednet = PredNet(stack_sizes, R_stack_sizes,
A_filt_sizes, Ahat_filt_sizes, R_filt_sizes,
output_mode='error', return_sequences=True)
inputs = Input(shape=(nt,) + input_shape)
errors = prednet(inputs) # errors will be (batch_size, nt, nb_layers)
errors_by_time = TimeDistributed(Dense(1, trainable=False), weights=[layer_loss_weights, np.zeros(1)], trainable=False)(errors) # calculate weighted error by layer
errors_by_time = Flatten()(errors_by_time) # will be (batch_size, nt)
final_errors = Dense(1, weights=[time_loss_weights, np.zeros(1)], trainable=False)(errors_by_time) # weight errors by time
model = Model(inputs=inputs, outputs=final_errors)
model.compile(loss='mean_absolute_error', optimizer='adam')
train_generator = SequenceGenerator(train_file, train_sources, nt, batch_size=batch_size, shuffle=True)
val_generator = SequenceGenerator(val_file, val_sources, nt, batch_size=batch_size, N_seq=N_seq_val)
lr_schedule = lambda epoch: 0.001 if epoch < 75 else 0.0001 # start with lr of 0.001 and then drop to 0.0001 after 75 epochs
callbacks = [LearningRateScheduler(lr_schedule)]
if save_model:
if not os.path.exists(WEIGHTS_DIR): os.mkdir(WEIGHTS_DIR)
callbacks.append(ModelCheckpoint(filepath=weights_file, monitor='val_loss', save_best_only=True))
history = model.fit_generator(train_generator, samples_per_epoch / batch_size, nb_epoch, callbacks=callbacks, validation_data=val_generator, validation_steps=N_seq_val / batch_size)
#history=model.fit(train,
if save_model:
json_string = model.to_json()
with open(json_file, "w") as f:
f.write(json_string)
def train():
KITTI_train_gen = KITTILoader(subset='training')
dim_avg, dim_cnt = KITTI_train_gen.get_average_dimension()
####
#dim_avg = {"Car": np.array([1.64473343, 1.68501193, 4.19985967])}
new_data = orientation_confidence_flip(KITTI_train_gen.image_data, dim_avg)
model = nn.network()
#model.load_weights('model00000006.hdf5')
early_stop = callbacks.EarlyStopping(monitor='val_loss',
min_delta=0.001,
patience=10,
mode='min',
verbose=1)
checkpoint = callbacks.ModelCheckpoint('model{epoch:08d}.hdf5',
monitor='val_loss',
verbose=1,
save_best_only=False,
mode='min',
period=1)
tensorboard = callbacks.TensorBoard(log_dir='logs/',
histogram_freq=0,
write_graph=True,
write_images=False)
all_examples = len(new_data)
trv_split = int(cfg().split * all_examples) # train val split
train_gen = data_gen(new_data[:trv_split])
valid_gen = data_gen(new_data[trv_split:all_examples])
print("READY FOR TRAINING")
train_num = int(np.ceil(trv_split / cfg().batch_size))
valid_num = int(np.ceil((all_examples - trv_split) / cfg().batch_size))
#gen_flow = gen_flow_for_two_inputs(X_train, X_angle_train, y_train)
# choose the minimizer to be sgd
# minimizer = optimizer.SGD(lr=0.0001, momentum = 0.9)
minimizer = Adam(lr=0.0001)
#minimizer = SGD(lr=0.0001)
# multi task learning
model.compile(
optimizer=minimizer, #minimizer,
loss={
'dimensions': 'mean_squared_error',
'orientation': orientation_loss,
'confidence': 'categorical_crossentropy'
},
loss_weights={
'dimensions': 2.,
'orientation': 1.,
'confidence': 4.
})
print("####################################################")
print(K.get_value(model.optimizer.lr))
# Tambahan aing
def scheduler(epoch):
if epoch % 50 == 0 and epoch != 0:
lr = K.get_value(model.optimizer.lr)
K.set_value(model.optimizer.lr, lr * .8)
print("lr changed to {}".format(lr * .8))
print("lr = ", K.get_value(model.optimizer.lr))
return K.get_value(model.optimizer.lr)
lr_sched = LearningRateScheduler(scheduler)
#print(asdadsad)
# d:0.0088 o:0.0042, c:0.0098
# steps_per_epoch=train_num,
# validation_steps=valid_num,
# callbacks=[early_stop, checkpoint, tensorboard],
model.fit_generator(generator=train_gen,
steps_per_epoch=train_num,
epochs=500,
verbose=1,
validation_data=valid_gen,
validation_steps=valid_num,
shuffle=True,
callbacks=[checkpoint, tensorboard, lr_sched],
max_queue_size=3)
batch_size = 5
epochs = 40
model_type = "large_full"
#### Train ####
# Callbacks
checkpoint = ModelCheckpoint('output/icnet_' + model_type +
'_{epoch:03d}_{categorical_accuracy:.3f}.h5',
mode='max')
tensorboard = TensorBoard(
batch_size=batch_size,
log_dir="./logs/ICNet/" + model_type +
"/{}/".format(strftime("%Y-%m-%d-%H-%M-%S", gmtime())))
lr_decay = LearningRateScheduler(PolyDecay(0.01, 0.9, epochs).scheduler)
# Generators
train_generator = utils.generator(df=utils.load_train_data(),
batch_size=batch_size,
resize_shape=(configs.img_width,
configs.img_height),
crop_shape=(configs.img_width,
configs.img_height),
n_classes=34,
training=True)
val_generator = utils.generator(df=utils.load_val_data(configs.val_label_path),
batch_size=1,
resize_shape=(configs.img_width,
configs.img_height),
#opt = keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=decay_rate)
opt = keras.optimizers.Adam(lr=0.3, beta_1=0.3, beta_2=0.999, epsilon=1e-08)
model.compile(optimizer=opt,loss='categorical_crossentropy',metrics=['accuracy',f1])
# hyperparameter tuning
filepath=" Result binary weights-improvement-{epoch:02d}-{val_accuracy:.2f}.hdf5"
checkpoint = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')
#early = keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=1e-4, patience=10)
reduce1 = keras.callbacks.ReduceLROnPlateau(monitor='val_accuracy', verbose=1,factor=0.33, patience=2,min_lr=0.000001 )
lrate = LearningRateScheduler(step_decay, verbose=1)
#callbacks_list = [checkpoint,early,reduce1]
#callbacks_list = [checkpoint,reduce1,lrate]
callbacks_list = [checkpoint,reduce1]
#callbacks_list = [checkpoint,lrate]
# data augmentation
train_datagen = ImageDataGenerator(
def train_():
X, y = read_train_data()
train_img, test_img, train_mask, test_mask = train_test_split(
X, y, test_size=0.2, random_state=1)
train_img, val_img, train_mask, val_mask = train_test_split(train_img,
train_mask,
test_size=0.2,
random_state=1)
print(train_img.shape, train_mask.shape)
print('-' * 30)
print('UNET FOR MASK SEGMENTATION.')
print('-' * 30)
model = get_unet(IMG_WIDTH=256, IMG_HEIGHT=256, IMG_CHANNELS=1)
model_checkpoint = ModelCheckpoint(model_name + ".hdf5",
monitor='loss',
save_best_only=False)
model.summary()
model_json = model.to_json()
with open("{}.json".format(model_name), "w") as json_file:
json_file.write(model_json)
print('...Fitting model...')
print('-' * 30)
change_lr = LearningRateScheduler(step_decay)
tensorboard = TensorBoard(log_dir="logs/{}".format(model_name))
datagen = ImageDataGenerator(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=
False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range=
30, # randomly rotate images in the range (degrees, 0 to 180)
width_shift_range=
0.3, # randomly shift images horizontally (fraction of total width)
height_shift_range=
0.3, # randomly shift images vertically (fraction of total height)
zoom_range=0.3,
shear_range=0.,
horizontal_flip=True, # randomly flip images
vertical_flip=True, # randomly flip images
fill_mode='constant',
dim_ordering='tf')
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=[dice_coef])
# Fit the model on the batches generated by datagen.flow().
model.fit_generator(
datagen.flow(train_img, train_mask, batch_size=8),
validation_data=(val_img, val_mask),
samples_per_epoch=train_img.shape[0],
nb_epoch=10,
callbacks=[model_checkpoint, change_lr, tensorboard],
)
score = model.evaluate(test_img, test_mask, batch_size=16)
model.save_weights('seg_weight.h5')
# f.write(s)
model_check = ModelCheckpoint(model_name, save_best_only=True, verbose=1)
def get_lr(epoch):
if epoch < 50:
return 0.8e-3
elif epoch < 100:
return 5e-4
elif epoch < 300:
return 1e-4
else:
return 5e-5
lr = LearningRateScheduler(get_lr)
rms = RMSprop(lr=1e-5)
print ('compiling...')
model.compile(optimizer=rms, loss='mape', metrics=[])
model.fit(X_train, Y_train, batch_size=128, nb_epoch=100, sample_weight=None,
validation_data=(x_val, y_val), callbacks=[model_check, lr])
# model.load_weights(model_name)
# pred = test_pred.reshape((5, 9, 66))
# with codecs.open('result/test_3937.csv', 'w') as f:
# for id, date in enumerate([22, 24, 26, 28, 30]):
# for it, time_idx in enumerate(predict_idx):
# if (date, time_idx) in [(24, 46), (28, 46)]:
# pass
# else:
def NNModel(input_array: np.ndarray,
output_array: np.ndarray,
n_layers: int,
n_neurons: int,
model_name: str,
normal_out: bool = True,
nn_type: str = "normal",
scalar: str = "stardardize") -> float:
print("Starting: " + model_name)
if normal_out:
if nn_type == "normal":
folder_name = "Heston"
elif nn_type == "tanh":
folder_name = "HestonTanh"
else:
folder_name = "HestonMix"
else:
if nn_type == "normal":
folder_name = "Heston_non_normal"
elif nn_type == "tanh":
folder_name = "Heston_non_normal_tanh"
else:
folder_name = "Heston_non_normal_mix"
if not os.path.exists("Models2/" + folder_name):
os.makedirs("Models2/" + folder_name)
# checking file name
no = 0
for i in range(1, 100):
saveString = "Models2/" + folder_name + "/" + model_name + "_" + str(
i) + ".h5"
no = i
if os.path.isfile(saveString) == False:
break
model_path = "Models2/" + folder_name + "/" + model_name + "_" + str(
no) + ".h5"
X_train, X_test, Y_train, Y_test = train_test_split(input_array,
output_array,
test_size=0.3,
random_state=42)
if scalar == "stardardize":
norm_features = StandardScaler()
else:
norm_features = MinMaxScaler()
if normal_out:
norm_labels = StandardScaler()
Y_train = norm_labels.fit_transform(Y_train)
Y_test = norm_labels.transform(Y_test)
X_train_norm = norm_features.fit_transform(X_train)
X_test_norm = norm_features.transform(X_test)
if nn_type == "normal":
model = nng.NN_generator(n_layers, n_neurons,
np.shape(input_array)[1],
np.shape(output_array)[1])
elif nn_type == "tanh":
model = nng.NN_generator_tanh(n_layers, n_neurons,
np.shape(input_array)[1],
np.shape(output_array)[1])
else:
model = nng.NN_generator_mix(n_layers, n_neurons,
np.shape(input_array)[1],
np.shape(output_array)[1])
adam = Adam()
model.compile(
loss='mean_squared_error', #mean squared error
optimizer=adam)
callbacks_list = [
LearningRateScheduler(lr_schedule, verbose=0),
ModelCheckpoint(model_path, monitor="val_loss", save_best_only=True)
]
start_time = time.time()
model.fit(X_train_norm,
Y_train,
epochs=100,
batch_size=1024,
verbose=0,
callbacks=callbacks_list,
validation_split=0.1,
shuffle=True)
stop_time = time.time()
score = model.evaluate(X_test_norm, Y_test, verbose=2)
# Saving model
model.save(model_path)
# Saving normalization parameters
joblib.dump(
norm_features, "Models2/" + folder_name + "/" + model_name +
"_norm_features_" + str(no) + ".pkl")
if normal_out:
joblib.dump(
norm_labels, "Models2/" + folder_name + "/" + model_name +
"_norm_labels_" + str(no) + ".pkl")
# Appending test score to file
with open("Models2/" + folder_name + "/HestonModels.txt",
"a") as output_file:
output_file.write("\n")
output_file.write(model_name + " has a score of: " + str(score) +
", and took a total time of: " +
str(stop_time - start_time))
print("Stopping: " + model_name)
return score
def NNModelNext(data_set: list,
folder: str,
model_name: str,
n_layers: int,
n_neurons: int,
nn_type: str,
output_scaling: str,
input_scaling: str,
include_loss: bool = False) -> float:
model_save = "Models5/" + folder + "/" + model_name + "_" + str(
n_layers) + "_" + str(n_neurons) + ".h5"
model_path = "Models5/" + folder + "/"
if not os.path.exists(model_path):
os.makedirs(model_path)
X_train = data_set[0]
X_test = data_set[1]
Y_train = data_set[2]
Y_test = data_set[3]
if input_scaling == "standardize":
norm_features = StandardScaler()
normal_in = True
elif input_scaling == "normalize":
norm_features = MinMaxScaler()
normal_in = True
else:
normal_in = False
if output_scaling == "standardize":
norm_labels = StandardScaler()
normal_out = True
elif output_scaling == "normalize":
norm_labels = MinMaxScaler()
normal_out = True
else:
normal_out = False
if normal_in:
X_train = norm_features.fit_transform(X_train)
X_test = norm_features.transform(X_test)
### saving feature normalization if it doesn't exists.
joblib.dump(norm_features, model_path + "norm_feature.pkl")
if normal_out:
Y_train = norm_labels.fit_transform(Y_train)
Y_test = norm_labels.transform(Y_test)
joblib.dump(norm_labels, model_path + "norm_labels.pkl")
if nn_type == "normal":
model = nng.NN_generator(n_layers, n_neurons,
np.shape(X_train)[1],
np.shape(Y_train)[1])
elif nn_type == "tanh":
model = nng.NN_generator_tanh(n_layers, n_neurons,
np.shape(X_train)[1],
np.shape(Y_train)[1])
elif nn_type == "mix":
model = nng.NN_generator_mix(n_layers, n_neurons,
np.shape(X_train)[1],
np.shape(Y_train)[1])
elif nn_type == "regularization":
model = nng.NN_generator_regul(n_layers, n_neurons,
np.shape(X_train)[1],
np.shape(Y_train)[1])
elif nn_type == "dropput":
model = nng.NN_generator_dropout(n_layers, n_neurons,
np.shape(X_train)[1],
np.shape(Y_train)[1])
else:
model = nng.NN_generator_mix_noise(n_layers, n_neurons,
np.shape(X_train)[1],
np.shape(Y_train)[1])
adam = Adam()
model.compile(
loss='mean_squared_error', #mean squared error
optimizer=adam)
callbacks_list = [
LearningRateScheduler(lr_schedule, verbose=0),
ModelCheckpoint(model_save, monitor="val_loss", save_best_only=True)
]
if (model_name.find("mat") != -1):
n_batch_size = 1024 * 5
elif (model_name.find("single") != -1):
n_batch_size = 1024 * 25
elif np.shape(X_train)[0] > 140000:
n_batch_size = 1024 * 2
else:
n_batch_size = 1024
start_time = time.time()
loss = model.fit(X_train,
Y_train,
epochs=100,
batch_size=n_batch_size,
verbose=0,
callbacks=callbacks_list,
validation_split=0.1,
shuffle=True)
stop_time = time.time()
score = model.evaluate(X_test, Y_test, verbose=2)
if score > 0.7: #if overfitting, save that model
print("overfit, saving overfit model")
model.save(model_save)
if not os.path.exists(model_path + "/HestonModels.txt"):
with open(model_path + "/HestonModels.txt", "w") as output_file:
pass
# Appending test score to file
with open(model_path + "/HestonModels.txt", "a") as output_file:
output_file.write("\n")
output_file.write(model_save + " has a score of: " + str(score) +
", and took a total time of: " +
str(stop_time - start_time))
print("Done with: ", model_save)
if include_loss:
return loss, model
else:
return score
nt,
batch_size=batch_size,
N_seq=N_seq_val)
lr_schedule = lambda epoch: 0.001 if epoch < 100 else 0.0001 # start with lr of 0.001 and then drop to 0.0001 after 75 epochs
from keras.callbacks import Callback
class ImageCallback(Callback):
not_print = 'False'
def on_epoch_end(self, epoch, logs):
print("EPOCH_CHECK", epoch, logs['loss'])
callbacks = [LearningRateScheduler(lr_schedule), ImageCallback()]
save_model = 'True'
if save_model:
if not os.path.exists(WEIGHTS_DIR): os.mkdir(WEIGHTS_DIR)
callbacks.append(
ModelCheckpoint(filepath=weights_file,
monitor='loss',
save_best_only=True))
#callbacks.append(ModelCheckpoint(filepath=weights_file2, monitor='val_loss', save_best_only=True))
history = model.fit_generator(train_generator,
samples_per_epoch / batch_size,
nb_epoch,
callbacks=callbacks,
verbose=0,
initial_lrate = 0.01
save_weights = True
# epoch是10的整数倍时，学习率重置初始值，连续两个整十数之间呈指数下降
def Exponentialdecay(epoch):
drop = 0.9
yushu = epoch % 100
if (yushu == 0):
lrate = initial_lrate
else:
lrate = initial_lrate * math.pow(drop, yushu)
return lrate
mylrate = LearningRateScheduler(Exponentialdecay)
optimal = SGD(lr=initial_lrate, momentum=0.9, decay=0.0, nesterov=False)
# Adam(lr = 0.002)
if (activation == 'sigmoid'):
UnetModel.compile(optimizer=optimal,
loss='binary_crossentropy',
metrics=['accuracy']) # sgd,rmsprop ['accuracy']
else:
UnetModel.compile(optimizer=optimal,
loss='categorical_crossentropy',
metrics=['accuracy'])
model_checkpoint = ModelCheckpoint(pretrained_weights,
monitor='acc',
verbose=1,
model = create_model()
if os.path.exists(WEIGHTS):
model.load_weights(WEIGHTS)
if MODE == 1:
model_checkpoint = ModelCheckpoint(filepath=WEIGHTS,
monitor='loss',
verbose=1,
save_best_only=True)
reduce_lr = ReduceLROnPlateau(monitor='loss', factor=0.5, patience=10)
scheduler = LearningRateScheduler(learning_scheduler)
model.fit(
Y_channel,
UV_channel,
batch_size=BATCH_SIZE,
epochs=EPOCHS,
verbose=1,
# validation_split=0.1,
callbacks=[scheduler])
model.save_weights(WEIGHTS)
elif MODE == 2:
for i in range(0, 1000):
y = Y_channel[i]
"airplane", "automobile", "bird", "cat", "deer", "dog", "frog", "horse",
"ship", "truck"
]
# construct the image generator for data augmentation
aug = ImageDataGenerator(width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
fill_mode="nearest")
# construct the set of callbacks
figPath = os.path.sep.join([args["output"], "{}.png".format(os.getpid())])
jsonPath = os.path.sep.join([args["output"], "{}.json".format(os.getpid())])
callbacks = [
TrainingMonitor(figPath, jsonPath=jsonPath),
LearningRateScheduler(poly_decay)
]
# initialize the optimizer and model
print("[INFO] compiling model...")
opt = SGD(lr=INIT_LR, momentum=0.9)
model = MiniGoogLeNet.build(width=32, height=32, depth=3, classes=10)
model.compile(loss="categorical_crossentropy",
optimizer=opt,
metrics=["accuracy"])
# train the network
print("[INFO] training network...")
model.fit_generator(aug.flow(trainX, trainY, batch_size=64),
validation_data=(testX, testY),
steps_per_epoch=len(trainX) // 64,
def ResNet(trainset,
valset,
n=3,
version=1,
batch_size=32,
epochs=200,
data_augmentation=True,
save_dir='./save_models'):
if not trainset.with_labels:
raise ModelTrainingError('The training dataset does not have labels.')
if not valset.with_labels:
raise ModelTrainingError(
'The validation dataset does not have labels.')
# Training parameters
#batch_size = 32 # orig paper trained all networks with batch_size=128
#epochs = 200
# Subtracting pixel mean improves accuracy
# Model parameter
# ----------------------------------------------------------------------------
# | | 200-epoch | Orig Paper| 200-epoch | Orig Paper| sec/epoch
# Model | n | ResNet v1 | ResNet v1 | ResNet v2 | ResNet v2 | GTX1080Ti
# |v1(v2)| %Accuracy | %Accuracy | %Accuracy | %Accuracy | v1 (v2)
# ----------------------------------------------------------------------------
# ResNet20 | 3 (2)| 92.16 | 91.25 | ----- | ----- | 35 (---)
# ResNet32 | 5(NA)| 92.46 | 92.49 | NA | NA | 50 ( NA)
# ResNet44 | 7(NA)| 92.50 | 92.83 | NA | NA | 70 ( NA)
# ResNet56 | 9 (6)| 92.71 | 93.03 | 93.01 | NA | 90 (100)
# ResNet110 |18(12)| 92.65 | 93.39+-.16| 93.15 | 93.63 | 165(180)
# ResNet164 |27(18)| ----- | 94.07 | ----- | 94.54 | ---(---)
# ResNet1001| (111)| ----- | 92.39 | ----- | 95.08+-.14| ---(---)
# ---------------------------------------------------------------------------
# n = 3
# Model version
# Orig paper: version = 1 (ResNet v1), Improved ResNet: version = 2 (ResNet v2)
# version = 1
# Computed depth from supplied model parameter n
if version == 1:
depth = n * 6 + 2
elif version == 2:
depth = n * 9 + 2
# Model name, depth and version
model_type = 'ResNet%dv%d' % (depth, version)
# Load the data.
if trainset.in_memory:
x_train, y_train = trainset.preprocess(trainset.images,
trainset.labels, trainset.mean)
else:
pass
if valset.in_memory:
x_val, y_val = valset.preprocess(valset.images, valset.labels,
trainset.mean)
else:
pass
# Input image dimensions.
input_shape = x_train.shape[1:]
# If subtract pixel mean is enabled
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_val.shape[0], 'test samples')
print('y_train shape:', y_train.shape)
if version == 2:
model = resnet_v2(input_shape=input_shape, depth=depth)
else:
model = resnet_v1(input_shape=input_shape, depth=depth)
model.compile(loss='categorical_crossentropy',
optimizer=Adam(learning_rate=lr_schedule(0)),
metrics=['accuracy'])
model.summary()
print(model_type)
# Prepare model model saving directory.
save_dir = os.path.join(os.getcwd(), 'saved_models')
model_name = 'cifar10_%s_model.{epoch:03d}.h5' % model_type
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
filepath = os.path.join(save_dir, model_name)
# Prepare callbacks for model saving and for learning rate adjustment.
checkpoint = ModelCheckpoint(filepath=filepath,
monitor='val_acc',
verbose=1,
save_best_only=True)
lr_scheduler = LearningRateScheduler(lr_schedule)
lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1),
cooldown=0,
patience=5,
min_lr=0.5e-6)
callbacks = [checkpoint, lr_reducer, lr_scheduler]
# Run training, with or without data augmentation.
# Run training, with or without data augmentation.
if not data_augmentation:
print('Not using data augmentation.')
model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(x_val, y_val),
shuffle=True,
callbacks=callbacks)
else:
print('Using real-time data augmentation.')
# This will do preprocessing and realtime data augmentation:
datagen = ImageDataGenerator(
# set input mean to 0 over the dataset
featurewise_center=False,
# set each sample mean to 0
samplewise_center=False,
# divide inputs by std of dataset
featurewise_std_normalization=False,
# divide each input by its std
samplewise_std_normalization=False,
# apply ZCA whitening
zca_whitening=False,
# epsilon for ZCA whitening
zca_epsilon=1e-06,
# randomly rotate images in the range (deg 0 to 180)
rotation_range=0,
# randomly shift images horizontally
width_shift_range=0.1,
# randomly shift images vertically
height_shift_range=0.1,
# set range for random shear
shear_range=0.,
# set range for random zoom
zoom_range=0.,
# set range for random channel shifts
channel_shift_range=0.,
# set mode for filling points outside the input boundaries
fill_mode='nearest',
# value used for fill_mode = "constant"
cval=0.,
# randomly flip.py images
horizontal_flip=True,
# randomly flip.py images
vertical_flip=False,
# set rescaling factor (applied before any other transformation)
rescale=None,
# set function that will be applied on each input
preprocessing_function=None,
# image data format, either "channels_first" or "channels_last"
data_format=None,
# fraction of images reserved for validation (strictly between 0 and 1)
validation_split=0.0)
# Compute quantities required for featurewise normalization
# (std, mean, and principal components if ZCA whitening is applied).
datagen.fit(x_train)
# Fit the model on the batches generated by datagen.flow().
model.fit_generator(datagen.flow(x_train,
y_train,
batch_size=batch_size),
validation_data=(x_val, y_val),
epochs=epochs,
verbose=1,
workers=4,
callbacks=callbacks)
# Score trained model.
scores = model.evaluate(x_val, y_val, verbose=1)
print('Test loss:', scores[0])
print('Test accuracy:', scores[1])
return model
def train2(self):
clean = []
adv = []
query = os.path.join("/home/natalia/clever", "**", "*.jpg")
clean.extend(glob.glob(query, recursive=True))
query = os.path.join("/home/natalia/adv", "**", "*.jpg")
adv.extend(glob.glob(query, recursive=True))
clean_set = []
adv_set = []
for c in clean:
img = cv2.imread(c)
clean_set.append(img)
for a in adv:
img = cv2.imread(a)
adv_set.append(img)
y_clean = [[0] for _ in range(len(clean_set))]
y_adv = [[1] for _ in range(len(adv_set))]
shuffle(clean_set)
shuffle(adv_set)
x_train = clean_set[:45000] + adv_set[:45000]
y_train = y_clean[:45000] + y_adv[:45000]
x_test = clean_set[45000:]
adv_test = adv_set[45000:]
for i, e in enumerate(x_test):
cv2.imwrite("/home/natalia/test/clean_{}.jpg".format(str(i)), e)
for i, e in enumerate(adv_test):
cv2.imwrite("/home/natalia/test/adv_{}.jpg".format(str(i)), e)
y_test = y_clean[45000:]
y_train = keras.utils.to_categorical(y_train, 2)
y_test = keras.utils.to_categorical(y_test, 2)
x_train = np.array(x_train)
y_train = np.array(y_train)
x_test = np.array(x_test)
y_test = np.array(y_test)
x_train, x_test = self.color_preprocessing(x_train, x_test)
print("Loaded dataset")
model = self.build_detector()
adam = optimizers.Adam(lr=0.0001, beta_1=0.99, beta_2=0.999)
model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=['accuracy'])
change_lr = LearningRateScheduler(self.scheduler)
checkpoint = ModelCheckpoint(self.model_filename,
monitor='val_loss',
verbose=0,
save_best_only=True,
mode='auto')
cbks = [change_lr, checkpoint]
datagen = ImageDataGenerator(horizontal_flip=True,
width_shift_range=0.125,
height_shift_range=0.125,
fill_mode='constant',
cval=0.)
datagen.fit(x_train)
model.fit_generator(datagen.flow(x_train,
y_train,
batch_size=self.batch_size),
steps_per_epoch=self.iterations,
epochs=200,
callbacks=cbks,
validation_data=(x_test, y_test))
model.save(
"/home/natalia/repos/cleverhans/my_tests/models/resnet_detector.h5"
)
if not args.no_weights:
print('Loading weights from ', os.path.abspath(args.weights))
model.load_weights(args.weights, by_name=True)
# Define model callbacks.
model_checkpoint = ModelCheckpoint(filepath='snapshot/fcn8s_resnet_epoch-{epoch:02d}_loss-{loss:.3f}_val_loss-{val_loss:.3f}.h5',
monitor='loss',
verbose=1,
save_best_only=False,
save_weights_only=False,
mode='auto',
period=10)
csv_logger = CSVLogger(filename='log/fcn8s_resnet_training_log.csv',
separator=',',
append=True)
learning_rate_scheduler = LearningRateScheduler(schedule=lr_schedule, verbose=1)
terminate_on_nan = TerminateOnNaN()
callbacks = [model_checkpoint,
csv_logger,
learning_rate_scheduler,
terminate_on_nan]
train_generator = DataGenerator(split='train', batch_size=args.batch_size)
val_generator = DataGenerator(split='minival', batch_size=1, shuffle=False)
history = model.fit_generator(generator=train_generator,
epochs=args.epochs,
callbacks=callbacks,
workers=1,
validation_data=val_generator,
initial_epoch=args.init_epoch)
print(model_type)
# Prepare model model saving directory.
save_dir = os.path.join(os.getcwd(), 'saved_models')
model_name = 'cifar10_%s_model.{epoch:03d}.h5' % model_type
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
filepath = os.path.join(save_dir, model_name)
# Prepare callbacks for model saving and for learning rate adjustment.
checkpoint = ModelCheckpoint(filepath=filepath,
monitor='val_acc',
verbose=1,
save_best_only=True)
lr_scheduler = LearningRateScheduler(lr_schedule)
lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1),
cooldown=0,
patience=5,
min_lr=0.5e-6)
callbacks = [checkpoint, lr_reducer, lr_scheduler]
history = model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(x_test, y_test),
shuffle=True,
callbacks=callbacks)
def model_train(model_name, train_data, valid_data, trainTemporal,
validTemporal, topo_data, type):
# set callbacks
csv_logger = CSVLogger(PATH + '/' + MODELNAME + '_' + type + '.log')
checkpointer_path = PATH + '/' + MODELNAME + '_' + type + '.h5'
checkpointer = ModelCheckpoint(filepath=checkpointer_path,
verbose=1,
save_best_only=True)
early_stopping = EarlyStopping(monitor='val_loss',
patience=10,
verbose=1,
mode='auto')
LearnRate = LearningRateScheduler(lambda epoch: LR)
# data generator
train_generator = data_generator(train_data, trainTemporal, topo_data,
BATCHSIZE, TIMESTEP, model_name)
val_generator = data_generator(valid_data, validTemporal, topo_data,
BATCHSIZE, TIMESTEP, model_name)
sep = (sum([len(x) for x in train_data]) -
TIMESTEP * len(train_data)) * train_data[0].shape[1] // BATCHSIZE
val_sep = (sum([len(x) for x in valid_data]) - TIMESTEP *
len(valid_data)) * valid_data[0].shape[1] // BATCHSIZE
# train model
model = get_model_structure(model_name)
# model = multi_gpu_model(model, gpus=2) # gpu parallel
model.compile(loss=LOSS, optimizer=OPTIMIZER)
model.fit_generator(
train_generator,
steps_per_epoch=sep,
epochs=EPOCH,
validation_data=val_generator,
validation_steps=val_sep,
callbacks=[csv_logger, checkpointer, LearnRate, early_stopping])
# compute mse
val_nolabel_generator = test_generator(valid_data, validTemporal,
topo_data, BATCHSIZE, TIMESTEP)
val_predY = model.predict_generator(val_nolabel_generator, steps=val_sep)
valY = get_test_true(valid_data, TIMESTEP, model_name)
# mse
scaled_valY = np.reshape(valY,
((sum([len(x) for x in valid_data]) -
TIMESTEP * len(valid_data)), HEIGHT, WIDTH))
scaled_predValY = np.reshape(val_predY,
((sum([len(x) for x in valid_data]) -
TIMESTEP * len(valid_data)), HEIGHT, WIDTH))
print('val scale shape: ', scaled_predValY.shape)
val_scale_MSE = np.mean((scaled_valY - scaled_predValY)**2)
print("Model val scaled MSE", val_scale_MSE)
# rescale mse
val_rescale_MSE = val_scale_MSE * MAX_VALUE**2
print("Model val rescaled MSE", val_rescale_MSE)
# write record
with open(PATH + '/' + MODELNAME + '_prediction_scores.txt', 'a') as wf:
wf.write('train flow {} start time: {}\n'.format(type, StartTime))
wf.write('train flow {} end time: {}\n'.format(
type,
datetime.datetime.now().strftime('%Y%m%d_%H%M%S')))
wf.write("Keras MSE on flow {} trainData, {}\n".format(
type, val_scale_MSE))
wf.write("Rescaled MSE on flow {} trainData, {}\n".format(
type, val_rescale_MSE))
return val_scale_MSE, val_rescale_MSE
def get_lrate(epoch, lr):
''' learning rate
drop = 0.5
epochs_drop = 2.0
lrate = lr * math.pow(drop,
math.floor((1+epoch)/epochs_drop))
'''
# try jumping out of local min shall there exists
# if random.random() < .1:
# return lr * 2
return min(.005, .25 / (epoch**1.25 + 1))
loss_type = 'mae'
lr = LearningRateScheduler(get_lrate, verbose=0)
# model.compile(optimizer=RMSprop(lr=.001), loss=loss_type)
model.compile(optimizer=Adam(lr=.01), loss=loss_type)
# model.compile(optimizer='adam', loss=loss_type)
stoppu = EarlyStopping(min_delta=1e-8, patience=5, verbose=1, mode='min')
btfu = ModelCheckpoint('vox_' + loss_type + '.hdf5',
save_best_only=False,
period=1)
'''
# %% MIREX
data_dir = '/home/tran.ngo.quang.ngoc/Downloads/MIR-1K_for_MIREX/Wavfile/'
sample_rate, fmat = wavread(data_dir+'abjones_1_01.wav')
# sample_rate, fmat = wavread(data_dir+'abjones_2_02.wav')
master = fmat[:,0] * .5 + fmat[:,1] * .5
model.summary()
epochs = 25
initial_lrate = 0.01
def decay(epoch, steps=100):
initial_lrate = 0.01
drop = 0.96
epochs_drop = 8
lrate = initial_lrate * math.pow(drop, math.floor((1+epoch)/epochs_drop))
return lrate
sgd = SGD(lr=initial_lrate, momentum=0.9, nesterov=False)
lr_sc = LearningRateScheduler(decay, verbose=1)
model.compile(loss=['categorical_crossentropy', 'categorical_crossentropy',
'categorical_crossentropy'], loss_weights=[1, 0.3, 0.3], optimizer=sgd, metrics=['accuracy'])
history = model.fit(X_train, [y_train, y_train, y_train], validation_data=(
X_test, [y_test, y_test, y_test]), epochs=epochs, batch_size=256, callbacks=[lr_sc])
if __name__ == "__main__":
start()
valid_set_x, valid_set_y = dataset[1]
test_set_x, test_set_y = dataset[2]
print(train_set_x.shape)
print(train_set_y.shape)
print(test_set_x.shape)
print(test_set_y.shape)
train_set_x = train_set_x.reshape(-1, 32, 32, 1)
test_set_x = test_set_x.reshape(-1, 32, 32, 1)
# build network
print(model.summary())
# set callback
#tb_cb = TensorBoard(log_dir='./lenet', histogram_freq=0)
change_lr = LearningRateScheduler(scheduler)
ckpt = ModelCheckpoint('./output/ckpt.h5',
save_weights_only=True,
save_best_only=False,
mode='auto',
period=10)
cbks = [change_lr, ckpt]
# start train
model.fit(train_set_x,
train_set_y,
batch_size=128,
epochs=200,
callbacks=cbks,
validation_data=(test_set_x, test_set_y),
shuffle=True)
print('Learning rate: ', lr)
return lr
##############################################################
##############################################################
input_shape = list(x_train.shape[1:])
n_classes = y_train.shape[1]
n_train = x_train.shape[0]
clean_index = np.array([(y_train_noisy[i, :] == y_train[i, :]).all()
for i in range(n_train)])
# For tracking only, unused during training
noisy_index = np.array([not i for i in clean_index])
INCV_lr_callback = LearningRateScheduler(INCV_lr_schedule)
# Define optimizer and compile model
optimizer = optimizers.Adam(lr=INCV_lr_schedule(0),
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0)
model = create_model(input_shape=input_shape,
classes=n_classes,
name='INCV_ResNet32',
architecture='ResNet32')
model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'])
weights_initial = model.get_weights()
# Print model architecture
def objective_function(params):
nb_epochs = 200
start_time = time.time()
depth = 28
width = 2
batch_size = 128
B_min = 2
B_max = 6
lr_initial_hp = float(np.exp(params[0]))
decay_rate_factor_hp = float(np.exp(params[1]))
l2_regular_weight_hp = float(np.exp(params[2]))
momentum_hp = float(params[3])
model = wide_resnet(depth, width, l2_regular_weight_hp)(dset.shape,
dset.output_size)
model.compile(loss="categorical_crossentropy",
optimizer=SGD(lr=lr_initial_hp, momentum=momentum_hp),
metrics=["accuracy"])
# Create the data augmentation generator
datagen = ImageDataGenerator(
# set input mean to 0 over the dataset
featurewise_center=False,
# set each sample mean to 0
samplewise_center=False,
# divide inputs by std of dataset
featurewise_std_normalization=False,
# divide each input by its std
samplewise_std_normalization=False,
# apply ZCA whitening
zca_whitening=False,
# randomly rotate images in the range (deg 0 to 180)
rotation_range=0,
# randomly shift images horizontally
width_shift_range=0.1,
# randomly shift images vertically
height_shift_range=0.1,
# randomly flip images
horizontal_flip=True,
# randomly flip images
vertical_flip=False)
datagen.fit(x_train)
schedule = StepDecay(initAlpha=lr_initial_hp,
factor=decay_rate_factor_hp,
dropEvery=40)
callbacks = [LearningRateScheduler(schedule)]
# Train the model
global initial
if initial < n_init_num:
B = B_max
else:
B = np.random.uniform(low=B_min, high=B_max)
initial = initial + 1
ImportanceTraining(model, presample=float(B)).fit_generator(
datagen.flow(x_train, y_train, batch_size=batch_size),
validation_data=(x_test, y_test),
epochs=200,
verbose=1,
batch_size=batch_size,
callbacks=callbacks,
steps_per_epoch=int(np.ceil(float(len(x_train)) / batch_size)))
loss, score = model.evaluate(x_test, y_test, verbose=1)
c = time.time() - start_time
print('Loss:', loss)
print('Test error:', 1.0 - score)
global obj_track
obj_track.append(1 - score)
global loss_track
loss_track.append(loss)
print("Obj Track: ", obj_track)
print("Loss Track: ", loss_track)
global iter_num
iter_num = iter_num + 1
print("Iter num:", iter_num)
global best_obj
best_obj = min(best_obj, 1 - score)
global best_loss
best_loss = min(best_loss, loss)
print("Best Error: ", best_obj)
print("Best Loss:", best_loss)
print("#######################")
end_time = time.time() - start_time
print("Time to run this hp:", end_time)
print("#######################")
return 1.0 - score
# Change `figPath` and `jsonPath` to correctly use the `FIG_PATH`
# and `JSON_PATH` in the `tiny_imagenet_config`?
# construct the set of callbacks
figPath = os.path.sep.join([args["output"], "{}.png".format(os.getpid())])
jsonPath = os.path.sep.join([args["output"], "{}.json".format(os.getpid())])
fname = os.path.sep.join(
[args["weights"], "weights-{epoch:03d}-{val_acc:.4f}.hdf5"])
checkpoint = ModelCheckpoint(fname,
monitor="val_acc",
mode="max",
save_best_only=True,
verbose=1)
callbacks = [
TrainingMonitor(figPath, jsonPath=jsonPath),
LearningRateScheduler(poly_decay), checkpoint
]
# initialize the optimizer and model (ResNet-56)
print("[INFO] compiling model...")
model = ResNet.build(64,
64,
3,
2, (3, 4, 6), (64, 128, 256, 512),
reg=0.0005,
dataset="tiny_imagenet")
opt = SGD(lr=INIT_LR, momentum=0.9)
model.compile(loss="binary_crossentropy", optimizer=opt, metrics=["accuracy"])
# train the network
print("[INFO] training network...")
# CSVLogger to save the training results in a csv file
csv_logger = CSVLogger(out_dir + 'csv_log.csv', separator=';')
def lr_scheduler(epoch, lr):
if epoch == 15:
return lr
elif epoch == 25:
return lr * .1
elif epoch == 35:
return lr * .1
else:
return lr
lrs = LearningRateScheduler(lr_scheduler)
# Callback to terminate on NaN loss (so terminate on error)
NanLoss = TerminateOnNaN()
callbacks = [checkpointer, csv_logger, NanLoss, lrs, es]
# Train model
model.fit_generator(adSeq,
steps_per_epoch=None,
epochs= 80,
shuffle=True,
callbacks=callbacks,
verbose=1,
validation_data=adSeq_val)
'{}'.format(exp_suffix) + '/')
nb_epochs = epochs
nb_cycles = 1
init_lr = 0.0005
def _cosine_anneal_schedule(t):
cos_inner = np.pi * (t % (nb_epochs // nb_cycles))
cos_inner /= nb_epochs // nb_cycles
cos_out = np.cos(cos_inner) + 1
return float(init_lr / 2 * cos_out)
lr_schedule = LearningRateScheduler(_cosine_anneal_schedule, verbose=True)
callbacks_list = [lr_schedule, tensorboard]
# warm up model
model = create_model(input_shape=(SIZE, SIZE, 3), n_out=28)
POS_WEIGHT = 10 # multiplier for positive targets, needs to be tuned
import tensorflow as tf
import keras.backend.tensorflow_backend as tfb
def weighted_binary_crossentropy(target, output):
"""
Weighted binary crossentropy between an output tensor
and a target tensor. POS_WEIGHT is used as a multiplier
model.load_weights(
'/home/AN35190/fer+/fer+_data/data_48x48/best/best.hdf5')
model.layers.pop()
model.outputs = [model.layers[-1].output]
model.layers[-1].outbound_nodes = []
model.add(Dense(1, activation=last))
# Compile model
optimizer = Adam(lr=ler)
model.compile(loss='mse', optimizer=optimizer, metrics=['mse'])
return model
model = larger_model()
model.summary()
call = my_callback(file)
lrate = LearningRateScheduler(step_decay)
csv_logger = CSVLogger('log_' + file + '.csv', append=True, separator=';')
model.fit_generator(
batch_generator_train.my_generator_train(),
epochs=500,
validation_data=batch_generator_dev.my_generator_dev(),
steps_per_epoch=int(52352 * 10 / batch),
validation_steps=int(59175 / batch),
callbacks=[call, csv_logger,
TensorBoard(log_dir='./logs/' + str(file))],
verbose=2)
#247500
#train 297000/batch dev 49800/batch
if debug: print model_noNorm.output_shape
current_lr = LEARNING_RATE_SCHEDULE[0]
def lr_function(e):
global current_lr
if e in LEARNING_RATE_SCHEDULE:
_current_lr = LEARNING_RATE_SCHEDULE[e]
current_lr = _current_lr
else:
_current_lr = current_lr
return _current_lr
lr_callback = LearningRateScheduler(lr_function)
if getWinSolWeights:
w_load_worked = False
for l in model_norm.layers:
if debug: print '---'
if debug: print len(l.get_weights())
l_weights = l.get_weights()
if len(l_weights) == len(w_kSorted):
if debug:
for i in range(len(l_weights)):
print type(l_weights[i])
print np.shape(l_weights[i])
if not np.shape(l_weights[i]) == np.shape(w_kSorted[i]):
"somethings wrong with the loaded weight shapes"
l.set_weights(w_kSorted)
