def create_model(img_channels, img_rows, img_cols, n_conv1=32, n_conv2=64,
n_conv3=64, n_out1=512, n_out2=-1, lr=.001,
n_actions=4, loss='mse'):
"""
Make a keras CNN model.
"""
model = Sequential()
model.add(Convolution2D(n_conv1, 5, 5, border_mode='same',subsample=(2,2),
input_shape=(img_channels, img_rows, img_cols)))
model.add(PReLU())
model.add(Convolution2D(n_conv2, 3, 3, border_mode='same',subsample=(2,2)))
model.add(PReLU())
model.add(Convolution2D(n_conv3, 3, 3, border_mode='same'))
model.add(PReLU())
model.add(Flatten())
model.add(Dense(n_out1))
model.add(PReLU())
model.add(Dense(n_actions))
model.add(Activation('linear'))
# try clipping or huber loss
model.compile(loss=loss, optimizer=adam(lr=lr))
return model
def build_model(self, p):
S = Input(p['input_shape'], name='input_state')
A = Input((1,), name='input_action', dtype='int32')
R = Input((1,), name='input_reward')
T = Input((1,), name='input_terminate', dtype='int32')
NS = Input(p['input_shape'], name='input_next_sate')
self.Q_model = self.build_cnn_model(p)
self.Q_old_model = self.build_cnn_model(p, False) # Q hat in paper
self.Q_old_model.set_weights(self.Q_model.get_weights()) # Q' = Q
Q_S = self.Q_model(S) # batch * actions
Q_NS = disconnected_grad(self.Q_old_model(NS)) # disconnected gradient is not necessary
y = R + p['discount'] * (1-T) * K.max(Q_NS, axis=1, keepdims=True) # batch * 1
action_mask = K.equal(Tht.arange(p['num_actions']).reshape((1, -1)), A.reshape((-1, 1)))
output = K.sum(Q_S * action_mask, axis=1).reshape((-1, 1))
loss = K.sum((output - y) ** 2) # sum could also be mean()
optimizer = adam(p['learning_rate'])
params = self.Q_model.trainable_weights
update = optimizer.get_updates(params, [], loss)
self.training_func = K.function([S, A, R, T, NS], loss, updates=update)
self.Q_func = K.function([S], Q_S)
def _getopt(self):
from keras.optimizers import adam, rmsprop, sgd, adadelta
learning_rate = 0.001
opt = adam(lr=learning_rate, epsilon = 0.001)
#opt = rmsprop(learning_rate)
#opt = adadelta()
#opt = sgd(learning_rate = 0.01, momentum= 0.8, nesterov=True)
return opt
def buildMixModel(img_channels=3, lr = 0.01,weight_decay = 1e-7, loss='mse',activ='relu', last_activ='sigmoid'):
# just build a tiny fcn model, you can use more layers and more filters as you want
main_input = Input(shape=(img_channels, None, None), name='input')
conv_1 = Convolution2D(4,3,3, border_mode = 'same', activation= activ, init='orthogonal',name='conv_1',
W_regularizer=l2(weight_decay), b_regularizer=l2(weight_decay))(main_input)
max_1 = MaxPooling2D(pool_size = (2,2))(conv_1)
conv_2 = Convolution2D(8,3,3, border_mode = 'same', activation=activ, init='orthogonal',name='conv_2',
W_regularizer=l2(weight_decay), b_regularizer=l2(weight_decay))(max_1)
max_2 = MaxPooling2D(pool_size = (2,2))(conv_2)
dp_0 = Dropout(0.25)(max_2)
conv_3 = Convolution2D(16,3,3, border_mode = 'same', activation= activ, init='orthogonal',name='conv_3',
W_regularizer=l2(weight_decay), b_regularizer=l2(weight_decay))(dp_0) # 25
max_3 = MaxPooling2D(pool_size = (2,2))(conv_3) # 12
conv_4 = Convolution2D(32,3,3, border_mode = 'same', activation=activ, init='orthogonal',name='conv_4',
W_regularizer=l2(weight_decay), b_regularizer=l2(weight_decay))(max_3) # 12
max_4 = MaxPooling2D(pool_size = (2,2))(conv_4) # 12
dp_1 = Dropout(0.25)(max_4)
conv_5 = Convolution2D(64,3,3, border_mode = 'same', activation=activ, init='orthogonal',name='conv_5',
W_regularizer=l2(weight_decay), b_regularizer=l2(weight_decay))(dp_1) # 6
upsamp_0 = UpSampling2D((2,2))(conv_5)
resize_0 = Resize2D(K.shape(conv_4))(upsamp_0)
deconv_0 = Convolution2D(32,3,3, border_mode = 'same', activation=activ, init='orthogonal',name='deconv_0',
W_regularizer=l2(weight_decay), b_regularizer=l2(weight_decay))(resize_0)
dp_2 = Dropout(0.25)(deconv_0)
upsamp_1 = UpSampling2D((2,2))(dp_2)
resize_1 = Resize2D(K.shape(conv_3))(upsamp_1)
deconv_1 = Convolution2D(16,3,3, border_mode = 'same', activation=activ, init='orthogonal',name='deconv_1',
W_regularizer=l2(weight_decay), b_regularizer=l2(weight_decay))(resize_1)
upsamp_2 = UpSampling2D((2,2))(deconv_1)
resize_2 = Resize2D(K.shape(conv_2))(upsamp_2)
deconv_2 = Convolution2D(8,3,3, border_mode = 'same', activation=activ,init='orthogonal',name='deconv_2',
W_regularizer=l2(weight_decay), b_regularizer=l2(weight_decay))(resize_2)
dp_3 = Dropout(0.25)(deconv_2)
upsamp_3 = UpSampling2D((2,2))(dp_3)
resize_3 = Resize2D(K.shape(conv_1))(upsamp_3)
deconv_3 = Convolution2D(4,3,3, border_mode = 'same', activation=activ,init='orthogonal',name='deconv_3',
W_regularizer=l2(weight_decay), b_regularizer=l2(weight_decay))(resize_3)
last_conv = Convolution2D(1,3,3, border_mode = 'same', activation=last_activ,init='orthogonal', name= 'output_mask',
W_regularizer=l2(weight_decay), b_regularizer=l2(weight_decay))(deconv_3)
model = Model(input=[main_input], output=[last_conv])
#opt = SGD(lr=lr, decay= 1e-6, momentum=0.9,nesterov=True)
#opt = Adadelta(lr=lr, rho=0.95, epsilon=1e-06,clipvalue=10)
opt = adam(lr=lr)
model.compile(loss={'output_mask': loss }, optimizer=opt)
return model
from keras.models import Sequential
from keras.layers import Dense, CuDNNGRU
from keras.optimizers import adam
from keras.callbacks import ModelCheckpoint
cb = [ModelCheckpoint("model.hdf5", save_best_only=True, period=3)]
model = Sequential()
model.add(CuDNNGRU(48, input_shape=(None, n_features)))
model.add(Dense(10, activation='relu'))
model.add(Dense(1))
model.summary()
# Compile and fit model
model.compile(optimizer=adam(lr=0.0005), loss="mae")
history = model.fit_generator(train_gen,
steps_per_epoch=1000,
epochs=30,
verbose=0,
callbacks=cb,
validation_data=valid_gen,
validation_steps=200)
# Visualize accuracies
import matplotlib.pyplot as plt
def perf_plot(history, what = 'loss'):
x = history.history[what]
val_x = history.history['val_' + what]
# design network
model = Sequential()
model.add(
LSTM(NUM_LSTM[n],
input_shape=(train_X.shape[1], train_X.shape[2])))
model.add(Dense(NUM_Dense[d]))
model.add(Dense(PRIDICT * NUM_FEATURE))
# model.compile(loss='mae', optimizer='adam')
# history = model.fit(train_X, train_y, epochs=NUM_EPOCH[e], batch_size=BATCH_SIZE,
# validation_data=(test_X, test_y), verbose=2,
# shuffle=False)
# time based learning rate decay: lr *= (1. / (1. + self.decay * self.iterations))
Lr_decay = Lr_initial / NUM_EPOCH[e]
adam = optimizers.adam(lr=Lr_initial,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=Lr_decay,
amsgrad=False)
model.compile(loss='mae', optimizer=adam)
#fit network
history = model.fit(train_X,
train_y,
epochs=NUM_EPOCH[e],
batch_size=BATCH_SIZE,
validation_data=(test_X, test_y),
verbose=2,
shuffle=False)
#calculate the learning rate
lr_inter = Lr_initial
lr_logger = list()
for iteration in range(1, NUM_EPOCH[e] + 1, 1):
model.add(GlobalAveragePooling2D())
# model.add(Flatten())
model.add(Dense(2048))
model.add(LeakyReLU(alpha=0.1))
model.add(Dropout(0.4))
model.add(Dense(1024, kernel_regularizer=regularizers.l2(0.01)))
model.add(LeakyReLU(alpha=0.1))
model.add(Dropout(0.4))
model.add(Dense(500, activation='softmax'))
# model.summary()
##########################
model.compile(optimizer=optimizers.adam(lr=1e-4),
loss='categorical_crossentropy',
metrics=['accuracy'])
model.summary()
###########################
# Train = np.load("label.npy")
# # print Train.shape
# Test = np.load("test_labels.npy")
# print Test.shape
# data = ImageDataGenerator()
# train_data = data.flow_from_directory(
# directory=path,
# color_mode='grayscale',
# target_size=(100,80),
print(X_test.shape)
print("Test Y shape:")
print(Y_test.shape)
print("Start Training:")
# create model
model= Sequential()
model.add(Dense(first_hidden_layer, input_dim=input_nodes, kernel_initializer=initialization, kernel_regularizer=regularizers.l2(regularization), activation='relu'))
model.add(Dense(second_hidden_layer, kernel_initializer=initialization, kernel_regularizer=regularizers.l2(regularization), activation='relu'))
model.add(Dropout(0.5))
model.add(
Dense(third_hidden_layer, kernel_initializer=initialization, kernel_regularizer=regularizers.l2(regularization),
activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(output_nodes, kernel_initializer=initialization, activation='relu'))
# Compile model
Adam = optimizers.adam(learning_rate=lr, decay=1e-7)
model.compile(loss=loss,optimizer=Adam)
# Fit the model
import time
train_start=time.clock()
#model.fit(trainX, trainY, epochs=epoch, batch_size=10, verbose=0) #Fit the model based on current training set, excluding the test sample.
history = model.fit(augment_trainX, augment_trainY, validation_split = 0.2, epochs=epoch, batch_size=128, verbose=0) #Fit the model based on expanded training set, after excluding the test sample.
train_finished=time.clock()
#plt.title("learning curve epoch: {}, lr: {}".format(str(epoch),str(lr)))
#loss, = plt.plot(history.history['loss'])
#val_loss, = plt.plot(history.history['val_loss'])
#plt.legend([loss, val_loss], ['loss', 'Val_loss'])
#plt.show()
train_time=train_finished-train_start
print("Training time: %.3f", train_time)
l[i] = to_categorical(labels[i], num_classes = 5)
elif task_flag == 2:
l = np.zeros([len(labels),6])
for i in range(labels.shape[0]):
l[i] = to_categorical(labels[i], num_classes = 6)
labels = to_categorical(labels)
labels = l.astype(np.uint8)
#images, labels = data_augmentation(images, labels, 9, in_shape)
images_train, images_test, labels_train, labels_test = train_test_split(images, labels, test_size=0.3)
my_loss = losses.categorical_crossentropy
my_metrics = [metrics.mae, metrics.categorical_accuracy]
my_opt = optimizers.adam(lr = 0.00001, decay=0.00000001)
kernel_init = initializers.VarianceScaling(scale=1, mode="fan_avg", distribution="uniform", seed=None)
bias_init = initializers.VarianceScaling(scale=1, mode="fan_avg", distribution="uniform", seed=None)
model = Sequential()
model.add(Conv3D(filters=64, kernel_size=(3,3,2), strides=(2,2,2), \
data_format="channels_last", input_shape=(in_shape[0], in_shape[1], in_shape[2], in_shape[3]), \
kernel_initializer=kernel_init, bias_initializer=bias_init, \
activation="relu"))
model.add(BatchNormalization())
model.add(MaxPooling3D(pool_size=(2,2,1)))
model.add(SpatialDropout3D(rate=0.6))
model.add(Conv3D(filters=64, kernel_size=(3,3,2), strides=(2,2,2), activation="relu", \
test_y = test_df.is_attributed
train_x = train_df.drop(['is_attributed'], axis=1)
test_x = test_df.drop(['is_attributed'], axis=1)
train_x = np.array(train_x)
test_x = np.array(test_x)
train_y = np.array(train_y)
test_y = np.array(test_y)
# In[7]:
#-------------------Build the Neural Network model-------------------
print('Building Neural Network model...')
adam = optimizers.adam(lr=0.005, decay=0.0000001)
model = Sequential()
model.add(
Dense(
48,
input_dim=train_x.shape[1],
kernel_initializer='normal',
#kernel_regularizer=regularizers.l2(0.02),
activation="relu"))
model.add(Dropout(0.2))
model.add(
Dense(
24,
#kernel_regularizer=regularizers.l2(0.02),
activation="tanh"))
model.add(BatchNormalization())
model.add(Activation('tanh'))
model.add(Dropout(0.2))
model.add(
MaxPooling2D(pool_size=(2, 2),
strides=None,
padding='valid',
data_format=None))
model.add(Flatten())
model.add(Dropout(drop_out_rate))
model.add(Dense(2, activation='sigmoid'))
model.summary()
#%%
model.compile(loss='binary_crossentropy',
metrics=['acc'],
optimizer=adam(learning_rate))
history = model.fit(mesh_train,
train_y,
batch_size=batch_size,
epochs=epoch,
validation_split=0.2,
class_weight='auto')
#%%
import matplotlib.pyplot as plt
save_Path_fig = r'C:\Users\leeyo\OneDrive\Documents\Research\Log\Nov 27th 2018'
plot_Name1 = 'Accuracy and Losses#4.png'
full_plot_Path1 = os.path.join(save_Path_fig, plot_Name1)
fig1 = plt.figure()
plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.plot(history.history['loss'])
earlyStopping = EarlyStopping(monitor='val_loss',
patience=10,
verbose=0,
mode='min')
mcp_save = ModelCheckpoint('.mdl_wts.hdf5',
save_best_only=True,
monitor='val_loss',
mode='min')
reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=5,
verbose=1,
epsilon=1e-4,
mode='min')
model.compile(loss='mean_absolute_error',
optimizer=adam(lr=1e-4),
metrics=['mean_absolute_error'])
# In[ ]:
################################################################################
model.fit(X_train,
y_train,
batch_size=16,
epochs=100,
validation_data=(X_val, y_val),
callbacks=[earlyStopping, mcp_save, reduce_lr_loss],
verbose=0)
###############################################################################
################################################################################
blocks_model.add(Dense(input_dim=8192, output_dim=300, activation='relu'))
blocks_model.add(Dropout(0.5))
blocks_model.add(Dense(300, activation='relu'))
feature_model.add(Dense(input_dim=9, output_dim=300, activation='relu'))
feature_model.add(Dropout(0.5))
feature_model.add(Dense(300, activation='relu'))
final_model.add(
Merge([blocks_model, feature_model], mode='concat', concat_axis=-1))
final_model.add(Dense(output_dim=2))
final_model.add(Activation("relu"))
final_model.add(Activation("softmax"))
print("compiling")
optim = opt.adam(0.00004, 0.9, 0.999)
final_model.compile(loss='categorical_crossentropy',
optimizer=optim,
metrics=['accuracy'])
print("done")
turns = []
global_k_res = []
for j in range(6):
global_k_res.append(0)
turns.append(0)
for i in range(1, 500):
a = 0
versus = 0
changed = 0
def buildMIAResidule(img_channels=3,
lr=0.01,
weight_decay=1e-7,
loss='mse',
activ='relu',
bn=False,
scaling=0.3,
nf=32,
pooling='max',
last_activ='sigmoid',
opt=None,
make_predict=False):
if opt is None:
opt = adam(lr=lr)
droprate = 0.33
if pooling == 'max':
Pooling = MaxPooling2D #AveragePooling2D # ,
elif pooling == 'avg':
Pooling = AveragePooling2D
else:
assert 0, 'unkonwn pooling setting.'
main_input = Input(shape=(img_channels, None, None), name='input')
double_0 = bn_conv(main_input,
filters=nf,
kernel=(1, 1),
base_name='bn_double_0',
activ=activ,
bn=bn,
weight_decay=weight_decay)
block1 = fcn_residual(double_0,
filters=nf,
base_name='block_1',
activ=activ,
bn=bn,
scaling=scaling,
weight_decay=weight_decay)
double_1 = bn_conv(block1,
filters=nf * 2,
kernel=(1, 1),
base_name='bn_double_1',
activ=activ,
bn=bn,
weight_decay=weight_decay)
maxpool = Pooling(pool_size=(2, 2))(double_1)
block2 = fcn_residual(maxpool,
filters=nf * 2,
base_name='block_2',
activ=activ,
bn=bn,
scaling=scaling,
weight_decay=weight_decay)
double_2 = bn_conv(block2,
filters=nf * 4,
kernel=(1, 1),
base_name='bn_double_2',
activ=activ,
bn=bn,
weight_decay=weight_decay)
maxpool = Pooling(pool_size=(2, 2))(double_2)
block3 = fcn_residual(maxpool,
filters=nf * 4,
base_name='block_3',
activ=activ,
bn=bn,
scaling=scaling,
weight_decay=weight_decay)
double_3 = bn_conv(block3,
filters=nf * 8,
kernel=(1, 1),
base_name='bn_double_3',
activ=activ,
bn=bn,
weight_decay=weight_decay)
maxpool = Pooling(pool_size=(2, 2))(double_3)
block4 = fcn_residual(maxpool,
filters=nf * 8,
base_name='block_4',
activ=activ,
bn=bn,
scaling=scaling,
weight_decay=weight_decay)
#double_4 = bn_conv(block4, filters=512, base_name='bn_double_4', activ=activ,weight_decay = weight_decay)
maxpool = Pooling(pool_size=(2, 2))(block4)
block5 = fcn_residual(maxpool,
filters=nf * 8,
base_name='block_5',
activ=activ,
bn=bn,
scaling=scaling,
weight_decay=weight_decay)
up_2 = up_conv(block5,
reference_tensor=block4,
filters=nf * 8,
kernel=(1, 1),
base_name='up_bn_conv_2')
deconv_2 = fcn_residual(up_2,
filters=nf * 8,
base_name='deconv_2',
activ=activ,
bn=bn,
scaling=scaling,
weight_decay=weight_decay)
up_3 = up_conv(deconv_2,
reference_tensor=block3,
filters=nf * 4,
kernel=(1, 1),
base_name='up_bn_conv_3')
deconv_3 = fcn_residual(up_3,
filters=nf * 4,
base_name='deconv_3',
activ=activ,
bn=bn,
scaling=scaling,
weight_decay=weight_decay)
up_4 = up_conv(deconv_3,
reference_tensor=block2,
filters=nf * 2,
kernel=(1, 1),
base_name='up_bn_conv_4')
deconv_4 = fcn_residual(up_4,
filters=nf * 2,
base_name='deconv_4',
activ=activ,
bn=bn,
scaling=scaling,
weight_decay=weight_decay)
up_5 = up_conv(deconv_4,
reference_tensor=block1,
filters=nf,
kernel=(1, 1),
merge_tensor=False,
base_name='up_bn_conv_5')
deconv_5 = fcn_residual(up_5,
filters=nf,
base_name='deconv_5',
activ=activ,
bn=bn,
scaling=scaling,
weight_decay=weight_decay)
last_conv = bn_conv(deconv_5,
filters=1,
kernel=(1, 1),
base_name='output_mask',
activ=activ,
bn=bn,
conv_activ=last_activ,
weight_decay=weight_decay)
model = Model(input=[main_input], output=[last_conv])
#opt = SGD(lr=lr, decay= 1e-7, momentum=0.9,nesterov=True)
#opt = Adadelta(lr=lr, rho=0.95, epsilon=1e-06,clipvalue=10)
model.compile(loss={'output_mask': loss},
optimizer=opt,
make_predict=make_predict)
return model
print('Split test: ', len(X_test), len(X_feat_test))
history = model1.fit(X_build, y_build,
validation_data=[X_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[ model_checkpoint,reduce_lr],
verbose=1)
model1 = load_model(save_model_name,custom_objects={'my_iou_metric': my_iou_metric})
# remove layter activation layer and use losvasz loss
input_x = model1.layers[0].input
output_layer = model1.layers[-1].input
model = Model(input_x, output_layer)
c = optimizers.adam(lr = 0.01)
# lovasz_loss need input range (-∞，+∞), so cancel the last "sigmoid" activation
# Then the default threshod for pixel prediction is 0 instead of 0.5, as in my_iou_metric_2.
model.compile(loss=lovasz_loss, optimizer=c, metrics=[my_iou_metric_2])
model.summary()
early_stopping = EarlyStopping(monitor='val_my_iou_metric_2', mode = 'max',patience=20, verbose=1)
model_checkpoint = ModelCheckpoint(save_model_name,monitor='val_my_iou_metric_2',
mode = 'max', save_best_only=True, verbose=1)
reduce_lr = ReduceLROnPlateau(monitor='val_my_iou_metric_2', mode = 'max',factor=0.5, patience=5, min_lr=0.0001, verbose=1)
#epochs = 50
#batch_size = 32
history = model.fit(X_build, y_build,
validation_data=[X_valid, y_valid],
def train_and_predict(x_train, y_train, x_valid, y_valid, fold):
# data augmentation
x_train = np.append(x_train, [np.fliplr(x) for x in x_train], axis=0)
y_train = np.append(y_train, [np.fliplr(x) for x in y_train], axis=0)
print("x_train after hflip", x_train.shape)
print("y_train after hflip", y_valid.shape)
# model
input_layer = Input((img_size_target, img_size_target, 3))
output_layer = build_model(input_layer, 16, 0.5)
model1 = Model(input_layer, output_layer)
c = optimizers.adam(lr=0.005)
model1.compile(loss="binary_crossentropy",
optimizer=c,
metrics=[my_iou_metric])
save_model_name = f"{basic_name}_stage1_fold{fold}.hdf5"
early_stopping = EarlyStopping(monitor='my_iou_metric',
mode='max',
patience=15,
verbose=1)
model_checkpoint = ModelCheckpoint(save_model_name,
monitor='my_iou_metric',
mode='max',
save_best_only=True,
verbose=1)
reduce_lr = ReduceLROnPlateau(monitor='my_iou_metric',
mode='max',
factor=0.5,
patience=5,
min_lr=0.0001,
verbose=1)
epochs = 80
batch_size = 128
if not PREDICT_ONLY:
history = model1.fit(
x_train,
y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[early_stopping, model_checkpoint, reduce_lr],
verbose=2)
model1 = load_model(save_model_name,
custom_objects={'my_iou_metric': my_iou_metric})
# remove activation layer and use lovasz loss
input_x = model1.layers[0].input
output_layer = model1.layers[-1].input
model = Model(input_x, output_layer)
c = optimizers.adam(lr=0.01)
model.compile(loss=lovasz_loss, optimizer=c, metrics=[my_iou_metric_2])
save_model_name = f"{basic_name}_stage2_fold{fold}.hdf5"
early_stopping = EarlyStopping(monitor='val_my_iou_metric_2',
mode='max',
patience=30,
verbose=1)
model_checkpoint = ModelCheckpoint(save_model_name,
monitor='val_my_iou_metric_2',
mode='max',
save_best_only=True,
verbose=1)
reduce_lr = ReduceLROnPlateau(monitor='val_my_iou_metric_2',
mode='max',
factor=0.5,
patience=5,
min_lr=0.00005,
verbose=1)
epochs = 120
batch_size = 128
if not PREDICT_ONLY:
history = model.fit(
x_train,
y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[model_checkpoint, reduce_lr, early_stopping],
verbose=2)
model = load_model(save_model_name,
custom_objects={
'my_iou_metric_2': my_iou_metric_2,
'lovasz_loss': lovasz_loss
})
def predict_result(model, x_test,
img_size_target): # predict both orginal and reflect x
x_test_reflect = np.array([np.fliplr(x) for x in x_test])
preds_test = model.predict(x_test).reshape(-1, img_size_target,
img_size_target)
preds_test2_refect = model.predict(x_test_reflect).reshape(
-1, img_size_target, img_size_target)
preds_test += np.array([np.fliplr(x) for x in preds_test2_refect])
return preds_test / 2
preds_valid = predict_result(model, x_valid, img_size_target)
preds_test = predict_result(model, x_test, img_size_target)
return preds_valid, preds_test
def build_model(french_index,eng_index,index_french,index_eng,English,French):
input_sequence = Input(shape=(maxlen + 1,));
input_tensor = Embedding(input_length=maxlen + 1, input_dim=len(french_index) + 1, output_dim=500)(input_sequence);
encoder1 = Conv1D(filters=500, kernel_size=1, strides=1, padding="same")(input_tensor);
encoder1 = Activation("relu")(encoder1);
encoder1 = Conv1D(filters=250, kernel_size=5, strides=1, padding="same", dilation_rate=1)(encoder1);
encoder1 = BatchNormalization(axis=-1)(encoder1);
encoder1 = Activation("relu")(encoder1);
encoder1 = Conv1D(filters=500, kernel_size=1, strides=1, padding="same")(encoder1);
input_tensor = merge([input_tensor, encoder1], mode="sum");
encoder2 = BatchNormalization(axis=-1)(input_tensor);
encoder2 = Activation("relu")(encoder2);
encoder2 = Conv1D(filters=500, kernel_size=1, strides=1, padding="same")(input_tensor);
encoder2 = BatchNormalization(axis=-1)(encoder2);
encoder2 = Activation("relu")(encoder2);
encoder2 = Conv1D(filters=250, kernel_size=5, strides=1, padding="same", dilation_rate=2)(encoder2);
encoder2 = BatchNormalization(axis=-1)(encoder2);
encoder2 = Activation("relu")(encoder2);
encoder2 = Conv1D(filters=500, kernel_size=1, strides=1, padding="same")(encoder2);
input_tensor = merge([input_tensor, encoder2], mode="sum");
encoder3 = BatchNormalization(axis=-1)(input_tensor);
encoder3 = Activation("relu")(encoder3);
encoder3 = Conv1D(filters=500, kernel_size=1, strides=1, padding="same")(encoder3);
encoder3 = BatchNormalization(axis=-1)(encoder3);
encoder3 = Activation("relu")(encoder3);
encoder3 = Conv1D(filters=250, kernel_size=5, strides=1, padding="same", dilation_rate=4)(encoder3);
encoder3 = BatchNormalization(axis=-1)(encoder3);
encoder3 = Activation("relu")(encoder3);
encoder3 = Conv1D(filters=500, kernel_size=1, strides=1, padding="same")(encoder3);
input_tensor = merge([input_tensor, encoder3], mode="sum");
encoder4 = BatchNormalization(axis=-1)(input_tensor);
encoder4 = Activation("relu")(encoder4);
encoder4 = Conv1D(filters=500, kernel_size=1, strides=1, padding="same")(encoder4);
encoder4 = BatchNormalization(axis=-1)(encoder4);
encoder4 = Activation("relu")(encoder4);
encoder4 = Conv1D(filters=250, kernel_size=5, strides=1, padding="same", dilation_rate=8)(encoder4);
encoder4 = BatchNormalization(axis=-1)(encoder4);
encoder4 = Activation("relu")(encoder4);
encoder4 = Conv1D(filters=500, kernel_size=1, strides=1, padding="same")(encoder4);
input_tensor = merge([input_tensor, encoder4], mode="sum");
encoder5 = BatchNormalization(axis=-1)(input_tensor);
encoder5 = Activation("relu")(encoder5);
encoder5 = Conv1D(filters=500, kernel_size=1, strides=1, padding="same")(encoder5);
encoder5 = BatchNormalization(axis=-1)(encoder5);
encoder5 = Activation("relu")(encoder5);
encoder5 = Conv1D(filters=250, kernel_size=5, strides=1, padding="same", dilation_rate=16)(encoder5);
encoder5 = BatchNormalization(axis=-1)(encoder5);
encoder5 = Activation("relu")(encoder5);
encoder5 = Conv1D(filters=500, kernel_size=1, strides=1, padding="same")(encoder5);
input_tensor = merge([input_tensor, encoder5], mode="sum");
input_tensor = Activation("relu")(input_tensor);
input_tensor = Conv1D(filters=500, kernel_size=1, padding="same", activation="relu")(input_tensor);
target_sequence = Input(shape=(maxlen,));
t0 = Input(shape=(1, 500));
target_input = Embedding(input_length=maxlen, input_dim=len(eng_index) + 1, output_dim=500)(target_sequence);
target_input = merge([t0, target_input], concat_axis=1, mode="concat");
input_to_decoder_sequence = merge([input_tensor, target_input], concat_axis=-1, mode="concat");
decoder1 = Conv1D(filters=1000, kernel_size=1, padding="same")(input_to_decoder_sequence);
decoder1 = BatchNormalization(axis=-1)(decoder1);
decoder1 = Activation("relu")(decoder1);
decoder1 = Conv1D(filters=500, kernel_size=3, padding="causal", dilation_rate=1)(decoder1);
decoder1 = BatchNormalization(axis=-1)(decoder1);
decoder1 = Activation("relu")(decoder1);
decoder1 = Conv1D(filters=1000, kernel_size=1, padding="same")(decoder1);
output_tensor = merge([input_to_decoder_sequence, decoder1], mode="sum");
decoder2 = BatchNormalization(axis=-1)(output_tensor);
decoder2 = Activation("relu")(decoder2);
decoder2 = Conv1D(filters=1000, kernel_size=1, strides=1, padding="same")(decoder2);
decoder2 = BatchNormalization(axis=-1)(decoder2);
decoder2 = Activation("relu")(decoder2);
decoder2 = Conv1D(filters=500, kernel_size=3, padding="causal", dilation_rate=2)(decoder2);
decoder2 = BatchNormalization(axis=-1)(decoder2);
decoder2 = Activation("relu")(decoder2);
decoder2 = Conv1D(filters=1000, kernel_size=1, padding="same")(decoder2);
output_tensor = merge([output_tensor, decoder2], mode="sum");
decoder3 = BatchNormalization(axis=-1)(output_tensor);
decoder3 = Activation("relu")(decoder3);
decoder3 = Conv1D(filters=1000, kernel_size=1, strides=1, padding="same")(decoder3);
decoder3 = BatchNormalization(axis=-1)(decoder3);
decoder3 = Activation("relu")(decoder3);
decoder3 = Conv1D(filters=500, kernel_size=3, padding="causal", dilation_rate=4)(decoder3);
decoder3 = BatchNormalization(axis=-1)(decoder3);
decoder3 = Activation("relu")(decoder3);
decoder3 = Conv1D(filters=1000, kernel_size=1, padding="same")(decoder3);
output_tensor = merge([output_tensor, decoder3], mode="sum");
decoder4 = BatchNormalization(axis=-1)(output_tensor);
decoder4 = Activation("relu")(decoder4);
decoder4 = Conv1D(filters=1000, kernel_size=1, strides=1, padding="same")(decoder4);
decoder4 = BatchNormalization(axis=-1)(decoder4);
decoder4 = Activation("relu")(decoder4);
decoder4 = Conv1D(filters=500, kernel_size=3, padding="causal", dilation_rate=8)(decoder4);
decoder4 = BatchNormalization(axis=-1)(decoder4);
decoder4 = Activation("relu")(decoder4);
decoder4 = Conv1D(filters=1000, kernel_size=1, padding="same")(decoder4);
output_tensor = merge([output_tensor, decoder4], mode="sum");
decoder5 = BatchNormalization(axis=-1)(output_tensor);
decoder5 = Activation("relu")(decoder5);
decoder5 = Conv1D(filters=1000, kernel_size=1, strides=1, padding="same")(decoder5);
decoder5 = BatchNormalization(axis=-1)(decoder5);
decoder5 = Activation("relu")(decoder5);
decoder5 = Conv1D(filters=500, kernel_size=3, padding="causal", dilation_rate=16)(decoder5);
decoder5 = BatchNormalization(axis=-1)(decoder5);
decoder5 = Activation("relu")(decoder5);
decoder5 = Conv1D(filters=1000, kernel_size=1, padding="same")(decoder5);
output_tensor = merge([output_tensor, decoder5], mode="sum");
output_tensor = Activation("relu")(output_tensor);
# decoder=Dropout(0.1)(decoder);
result = Conv1D(filters=len(eng_index), kernel_size=1, padding="same", activation="softmax")(output_tensor);
model = Model(inputs=[input_sequence, target_sequence, t0], outputs=result);
opt = adam(lr=0.0003); # as in the paper, we choose adam optimizer with lr=0.0003
model.compile(loss='categorical_crossentropy', optimizer="adam", metrics=['categorical_accuracy'],
sample_weight_mode="temporal");
return model;
else:
input_shape = (img_width, img_height, 3)
model = Sequential()
model.add(Conv2D(32, (2, 2), input_shape=input_shape))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dropout(0.2))
model.add(Dense(1))
model.add(Activation('sigmoid'))
#es = EarlyStopping(monitor='val_loss', mode='min', patience=7, verbose=1)
optimiser = optimizers.adam(learning_rate=0.01,
beta_1=0.9,
beta_2=0.999,
amsgrad=False)
model.compile(loss='binary_crossentropy',
optimizer=optimiser,
metrics=['accuracy'])
train_datagen = ImageDataGenerator(rescale=1. / 255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
test_datagen = ImageDataGenerator(rescale=1. / 255)
train_generator = train_datagen.flow_from_directory(train_data_dir,
target_size=(img_width,
model.add(Flatten())
# 中間層
model.add(Dense(256, activation='relu'))
# 全結合 特徴が似ているものを判定
model.add(Dropout(0.5))
# num_classesに相当するノードを出力
# softmax : 合計が1になるようにする関数を指定
model.add(Dense(num_classes, activation='softmax'))
# 最適化手法設定
# チュートリアルの値
# opt = SGD(lr=0.01)
opt = adam()
# 最適化
# metrics=['accuracy'] : 精度について表示する
model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])
# トレーニング実行
# batch_size : 一度に投入するデータサイズ
# epochs : この指標に試行が達するまで訓練する(与えられた反復回数の訓練をするわけではない)
epochs = 30
model.fit(X_train, y_train, batch_size=32, epochs=epochs)
# 評価
# トレーニングに使用していないデータをつかって、評価を行う
score = model.evaluate(X_test, y_test, batch_size=32)
print(f"score --> {score}")
#!/usr/bin/env python
# -*- coding: utf-8 -*-
__author__ = 'Yunchuan Chen'
from utils import get_unigram_probtable
from models import NCELangModel
from keras.optimizers import adam
NB_RUN_WORDS = 100000000
NB_VOCAB = 10000
NB_RUN_VAL = 100000
NB_EVALUATE = 5000000
SAVE_PATH = '../data/models/lang/nce0-neg50-e128-c128-lr0.005.pkl'
DATA_PATH = '../data/corpus/wiki-sg-norm-lc-drop-bin.bz2'
BATCH_SIZE = 256
VAL_INTER = 1200
unigram_table = get_unigram_probtable(nb_words=NB_VOCAB)
opt = adam(lr=0.005)
model = NCELangModel(vocab_size=NB_VOCAB, nb_negative=50, embed_dims=128, context_dims=128,
negprob_table=unigram_table, optimizer=opt)
model.compile()
model.train(data_file=DATA_PATH,
save_path=SAVE_PATH,
batch_size=BATCH_SIZE, train_nb_words=NB_RUN_WORDS,
val_nb_words=NB_EVALUATE, train_val_nb=NB_RUN_VAL, validation_interval=VAL_INTER)
model.add(Dense(50, activation='relu'))
model.add(Dropout(drop_out))
model.add(Dense(10, activation='relu'))
model.add(Dropout(drop_out))
model.add(Dense(OUT_SHAPE, activation='softsign'))
return model
if __name__ == '__main__':
# Load Training Data
x_train = np.load("data/X.npy")
y_train = np.load("data/Y.npy")
print(x_train.shape[0], 'train samples')
# Training loop variables
epochs = 10
batch_size = 64
model = create_model()
model.compile(loss=customized_loss, optimizer=optimizers.adam())
model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=epochs,
shuffle=True,
validation_split=0.1)
model.save_weights('model_weights.h5')
def PG_LSTM_train_test(iteration, tr_frac, val_frac, patience_val, num_epochs,
batch_size, lstm_nodes, feedforward_nodes, mask_value,
drop_frac, lstm_bias, n_nodes, use_GLM,
use_temporal_feature, lamda_reg, shuffle, tol,
pad_steps, n_layer_temp, n_layer_density, lamda_phy):
#read Data
[
train_X, train_y_temperature, train_y_glm, test_X, test_y_temperature,
test_y_glm
] = read_Data()
#Add GLM feature
if use_GLM == 1:
train_X = np.dstack((train_X, train_y_glm))
test_X = np.dstack((test_X, test_y_glm))
depth_steps = train_X.shape[1]
#Normalise Data
train_X = transform_3d_to_2d(train_X)
test_X = transform_3d_to_2d(test_X)
m1_train_mean = train_X.mean(axis=0)
m1_train_std = train_X.std(axis=0)
train_X = (train_X - m1_train_mean) / m1_train_std
test_X = (test_X - m1_train_mean) / m1_train_std
train_X = transform_2d_to_3d(train_X, depth_steps)
test_X = transform_2d_to_3d(test_X, depth_steps)
#Create train subset
[ix, train_X, train_y_temperature,
train_y_glm] = train_subset(train_X, train_y_temperature, train_y_glm)
#creating path and filename for storage of results
exp_name = 'pg_lstm_model_' + str(num_epochs) + '_dropout_frac_' + str(drop_frac) + '_feedforward_nodes' + str(feedforward_nodes) + '_lstm_nodes' + str(
lstm_nodes) + '_val_frac' + str(val_frac) + '_trfrac' + str(tr_frac) + '_iter' + str(iteration)+'_n_nodes'+str(n_nodes)+'_use_GLM'+str(use_GLM)+'_use_temporal_features'+str(use_temporal_feature) + \
'_lamda_reg'+str(lamda_reg)+'_lamda_phy'+str(lamda_phy)+'_n_layer_density'+str(n_layer_density)+'_n_layer_temp'+str(n_layer_temp)+'_pad_steps'+str(pad_steps)
exp_name = exp_name.replace('.', 'pt')
results_dir = '/home/karpatne/Documents/Lake Temperature Modelling/Results/SDM_results/PG_LSTM_l2/'
#results_dir='C:\\Users\\arkad\\Documents\\Lake Temperature Modelling\\Results\\test\\'
model_name = results_dir + exp_name + '_model.h5' # storing the trained model
results_name = results_dir + exp_name + '_results.mat' # storing the results of the model
train_y_temperature = transform_3d_to_2d(train_y_temperature)
test_y_temperature = transform_3d_to_2d(test_y_temperature)
##Create Density labels
#train_y_density=np.zeros(np.shape(train_y_temperature));
#test_y_density=np.zeros(np.shape(test_y_temperature));
#train_y_density[:,1], test_y_density[:,1] = train_y_temperature[:,1], test_y_temperature[:,1]
#train_y_density[:,0], test_y_density[:,0] = density(train_y_temperature[:,0]), density(test_y_temperature[:,0])
## density output normalisation
#mean = np.nanmean(train_y_density[:, 0]);
#std = np.nanstd(train_y_density[:, 0]);
#train_y_density[:, 0] = (train_y_density[:, 0] - mean) / std;
#test_y_density[:, 0] = (test_y_density[:, 0] - mean) / std;
#masking output labels
train_y_temperature = masking_output_labels(train_y_temperature,
mask_value)
test_y_temperature = masking_output_labels(test_y_temperature, mask_value)
#reshaping train/test output density/temperature labels
train_y_temperature = transform_2d_to_3d(train_y_temperature, depth_steps)
test_y_temperature = transform_2d_to_3d(test_y_temperature, depth_steps)
#perform edge padding
[train_X_pad,
train_y_temperature_pad] = edge_padding(train_X, train_y_temperature,
pad_steps)
[test_X_pad,
test_y_temperature_pad] = edge_padding(test_X, test_y_temperature,
pad_steps)
if shuffle == 1:
ix = np.arange(train_X_pad.shape[0])
random.shuffle(ix)
train_X_pad = train_X_pad[ix, :, :]
train_y_temperature_pad = train_y_temperature_pad[ix, :, :]
#create auxiliary dataset (same as input data)
model = createModel(train_X_pad.shape[1], train_X_pad.shape[2], lstm_nodes,
lstm_bias, drop_frac, feedforward_nodes, lamda_reg,
n_nodes, n_layer_density, n_layer_temp)
model.summary()
X_total = np.concatenate((train_X_pad, test_X_pad), axis=0)
#X_total=train_X_pad
uin = K.constant(value=X_total)
uin_value = K.get_value(uin)
lam = K.constant(value=lamda_phy)
uout1 = model(uin)
uou1_pred = K.get_value(uout1)
uout2 = uout1[:, pad_steps + 1:, :]
uout1 = uout1[:, pad_steps:-1, :]
uout1_before = K.get_value(uout1)
uout2_before = K.get_value(uout2)
uout1_shape = K.shape(uout1)
uout2_shape = K.shape(uout2)
uout1 = K.reshape(uout1, (uout1_shape[0] * uout1_shape[1], uout1_shape[2]))
uout2 = K.reshape(uout2, (uout2_shape[0] * uout2_shape[1], uout2_shape[2]))
# uout1 = K.print_tensor(uout1, message='uout1 = ')
uout1_reshape = K.get_value(uout1)
uout2_reshape = K.get_value(uout2)
# uout2 = K.print_tensor(uout2, message='uout2 = ')
# uout1=uout1.reshape(uout1.shape[0]*uout1.shape[1],uout1.shape[2])
# uout2=uout2.reshape(uout2.shape[0]*uout2.shape[1],uout2.shape[2])
udendiff = -(density(uout2) - density(uout1))
udendiff_value = K.get_value(udendiff)
totloss = combined_loss([udendiff, lam])
phyloss = phy_loss_mean([udendiff, lam])
optimizer = optimizers.adam(clipnorm=1)
model.compile(loss=totloss,
optimizer=optimizer,
metrics=[masked_mean_squared_error, phyloss])
early_stopping = EarlyStopping(monitor='val_masked_mean_squared_error',
patience=patience_val,
verbose=1)
# available metrics : val_loss,val_masked_root_mean_squared_error,loss,masked_root_mean_squared_error
history = model.fit(train_X_pad,
train_y_temperature_pad,
epochs=num_epochs,
batch_size=batch_size,
verbose=2,
shuffle=False,
validation_split=val_frac,
callbacks=[early_stopping,
TerminateOnNaN()])
# pyplot.plot(history.history['loss'], label='train_loss')
# pyplot.plot(history.history['loss_1'], label='phy_loss')
# pyplot.plot(history.history['masked_mean_squared_error'], label='train_rmse')
# pyplot.plot(history.history['val_masked_mean_squared_error'], label='val_rmse')
# pyplot.legend()
# pyplot.show()
#Calculating model RMSE
[test_rmse1, test_pred1,
test_y1] = normal_prediction(model, test_X_pad, test_y_temperature_pad,
pad_steps)
[train_rmse1, train_pred1,
train_y1] = normal_prediction(model, train_X_pad, train_y_temperature_pad,
pad_steps)
print('Without dropout = TrainRMSE : ' + str(train_rmse1) +
' TestRMSE : ' + str(test_rmse1))
[test_rmse_dropout, test_pred_do,
test_y1] = dropout_prediction(model, 100, test_X_pad,
test_y_temperature_pad[:, pad_steps:, :],
pad_steps)
[train_rmse_dropout, train_pred_do,
train_y1] = dropout_prediction(model, 100, train_X_pad,
train_y_temperature_pad[:, pad_steps:, :],
pad_steps)
print('With dropout = TrainRMSE : ' + str(train_rmse_dropout) +
' TestRMSE : ' + str(test_rmse_dropout))
#Calculating Physical Inconsistencies
test_inconsistency_without_dropout = normal_physical_inconsistency(
tol, test_pred1, depth_steps)
train_inconsistency_without_dropout = normal_physical_inconsistency(
tol, train_pred1, depth_steps)
print("Without dropout : Test Incon = " +
str(test_inconsistency_without_dropout) + ' Train Incon = ' +
str(train_inconsistency_without_dropout))
test_pred_uq_mean = test_pred_do.mean(axis=-1).transpose()
train_pred_uq_mean = train_pred_do.mean(axis=-1).transpose()
test_inconsistency_dropout_mean = normal_physical_inconsistency(
tol, test_pred_uq_mean, depth_steps)
train_inconsistency_dropout_mean = normal_physical_inconsistency(
tol, train_pred_uq_mean, depth_steps)
print("With dropout Inconsistency of sample mean: Test Incon = " +
str(test_inconsistency_dropout_mean) + ' Train Incon = ' +
str(train_inconsistency_dropout_mean))
test_inconsistency_dropout_all = physical_inconsistency_all_sample(
tol, test_pred_do, depth_steps)
train_inconsistency_dropout_all = physical_inconsistency_all_sample(
tol, train_pred_do, depth_steps)
print("With dropout Inconsistency of all samples: Test Incon = " +
str(test_inconsistency_dropout_all) + ' Train Incon = ' +
str(train_inconsistency_dropout_all))
test_p_values = compute_p_values(test_pred_do, test_y1)
train_p_values = compute_p_values(train_pred_do, train_y1)
print('Train p-values : ' + str(np.nanmean(train_p_values)))
print('Test p-values : ' + str(np.nanmean(test_p_values)))
[mean_p_value,
p_values] = plot_p_value_vs_residuals(test_p_values, test_pred_do,
test_y1)
z_score = plot_percent_percentile_plot(test_pred_do, test_y1)
#model.save(model_name)
spio.savemat(
results_name, {
'test_pred_temperature': test_pred1,
'test_y_temperature': test_y1,
'train_pred_temperature': train_pred1,
'train_y_temperature': train_y1,
'train_rmse': train_rmse1,
'test_rmse': test_rmse1,
'train_rmse_dropout': train_rmse_dropout,
'test_rmse_dropout': test_rmse_dropout,
'test_incon_without_dropout': test_inconsistency_without_dropout,
'train_inconsistency_without_dropout':
train_inconsistency_without_dropout,
'test_inconsistency_dropout_mean': test_inconsistency_dropout_mean,
'train_inconsistency_dropout_mean':
train_inconsistency_dropout_mean,
'test_inconsistency_dropout_all': test_inconsistency_dropout_all,
'train_inconsistency_dropout_all': train_inconsistency_dropout_all,
'test_predictions_all': test_pred_do,
'train_predictions_all': train_pred_do,
'train_index': ix,
'train_loss': history.history['loss'],
'phy_loss': history.history['loss_1'],
'train_mse': history.history['masked_mean_squared_error'],
'val_mse': history.history['val_masked_mean_squared_error'],
'z_scores': z_score,
'mean_p_value': mean_p_value,
'p_values': p_values
})
model.add(Activation('relu'))
model.add(AveragePooling2D(pool_size=(2, 2), strides=2))
model.add(Flatten())
model.add(Dense(128, kernel_initializer='glorot_uniform'))
model.add(BatchNormalization(epsilon=1e-5))
model.add(Activation('relu'))
model.add(Dropout(0.3))
model.add(Dense(1, kernel_initializer='glorot_uniform'))
model.add(BatchNormalization(epsilon=1e-5))
model.add(Activation('linear'))
adam = optimizers.adam(lr=0.001)
model.compile(optimizer=adam,
loss=root_mean_squared_error,
metrics=[root_mean_squared_error, 'mean_squared_error'])
#early_stopping = EarlyStopping(monitor='val_loss', mode='min', baseline=0.01, patience=0)
#os.remove('best_model.h5')
model_checkpoint = ModelCheckpoint('best_model.h5',
monitor='val_loss',
mode='min',
save_best_only=True)
history = model.fit([X_train],
Y_Train,
validation_data=(X_validation, Y_Validation),
epochs=100,
batch_size=80,
model_2 = Sequential()
model_2.add(Conv2D(32, (3, 3), input_shape=(32, 32 ,3), activation='relu', padding='same'))
model_2.add(Conv2D(64, (3, 3), activation='relu', padding='same'))
model_2.add(Conv2D(64, (3, 3), activation='relu', padding='same'))
model_2.add(MaxPooling2D(pool_size=(2,2)))
model_2.add(Conv2D(64, (3, 3), activation='relu', padding='same'))
model_2.add(Conv2D(64, (3, 3), activation='relu', padding='same'))
model_2.add(MaxPooling2D(pool_size=(2,2)))
model_2.add(Conv2D(32, (3, 3), activation='relu', padding='same'))
model_2.add(Conv2D(32, (3, 3), activation='relu', padding='same'))
model_2.add(Conv2D(32, (3, 3), padding='same'))
model_2.add(Conv2D(32, (3, 3)))
model_2.add(Dense(256, activation="relu"))
model_2.add(Flatten())
model_2.add(Dense(2, activation='softmax'))
optimizer = optimizers.adam(lr=0.00001)
model_2.compile(loss='binary_crossentropy', optimizer=optimizer,metrics=["accuracy"])
model_2.summary()
mhis = model_2.fit(X_train, y_train, epochs=200, validation_split=0.2)
mhis2 = model_2.fit(X_train, y_train, epochs=200, validation_split=0.2)
model_2.save('FirstModel2.h5')
from google.colab import files
files.download("FirstModel2.h5")
y_pred = modeltest.predict(X_test)
target_names = ['cbsd','not_cbsd']
from sklearn.metrics import accuracy_score, classification_report
print(accuracy_score(y_test.argmax(axis=1), y_pred.argmax(axis=1)))
nb_layer = 3
lr = 0.001
xtr, ytr, xte, yte = load_BikeNYC_new(window_len=window_len,
nb_flow=2,
len_test=240)
epochs = 500
batch_size = 64
# model = build_model(window_len=window_len, lr=lr)
model = myCrossLayer(nb_flow=2,
map_height=16,
map_width=8,
nb_layers=nb_layer,
window_len=window_len,
nb_filter=nb_filter,
filter_size=filter_size)
my_optim = adam(lr=lr)
model.compile(loss='mse', optimizer=my_optim, metrics=[rmse])
model.summary()
plot_model(model, to_file='densenet.png', show_shapes=True)
print('hybrid model built ... ')
# earlystoping = EarlyStopping(monitor='val_root_mean_square_error', patience=30, mode='min')
earlystoping = EarlyStopping(monitor='val_loss', patience=30, mode='min')
checkpoint = ModelCheckpoint(
filepath=model_name,
monitor='val_root_mean_square_error',
verbose=1,
save_best_only=True,
mode='min',
period=1
) # checkpoint2 = ModelCheckpoint(filepath='./others/model_best_2.h5', monitor='val_root_mean_square_error', verbose=1, save_best_only=True,mode='min', period=1)
def create_nn(self, n_layers, n_units, n_epochs, batch_size, activ_funct, learning_rate):
from keras import optimizers
#K.clear_session()
self.model = Sequential()
"""
The function trains a simple Recurrent neural network.
Parameters:
time_steps: Sequence of time steps back, that have to be included in the network.
n_units: Number of neurons in the corresponding layer. Input as a list, eg [neurons_1_layer, neurons_2_layer]
n_layers: Number of layers.
self.nn_inputs: Number inputs. Since we are concentrated just on one variable, it is one for us.
n_classes: Number of outputs. Since we are concentrated just on one variable, it is one for us.
act: Activation function as a string.
"""
from keras.layers import SimpleRNN
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers import GRU
from keras.layers.advanced_activations import LeakyReLU, PReLU
import math
if self.NN_type == 'SimpleRNN':
if n_layers == 1:
self.model.add(SimpleRNN(n_units[0], input_shape=(self.time_steps, self.nn_inputs), activation = activ_funct))
else:
self.model.add(SimpleRNN(n_units[0], input_shape=(self.time_steps, self.nn_inputs), return_sequences = True, activation = activ_funct))
for i in range(2, n_layers+1):
if i < n_layers:
self.model.add(SimpleRNN(n_units[i-1], return_sequences = True, activation = activ_funct))
else:
self.model.add(SimpleRNN(n_units[len(n_units)-1], return_sequences = False, activation = activ_funct))
if self.NN_type == 'LSTM':
if n_layers == 1:
self.model.add(LSTM(n_units[0], input_shape=(self.time_steps, self.nn_inputs), activation = activ_funct))
else:
self.model.add(LSTM(n_units[0], input_shape=(self.time_steps, self.nn_inputs), return_sequences = True, activation = activ_funct))
for i in range(2, n_layers+1):
if i < n_layers:
self.model.add(LSTM(n_units[i-1], return_sequences = True, activation = activ_funct))
else:
self.model.add(LSTM(n_units[len(n_units)-1], return_sequences = False, activation = activ_funct))
if self.NN_type == 'GRU':
if n_layers == 1:
self.model.add(GRU(n_units[0], input_shape=(self.time_steps, self.nn_inputs), activation = activ_funct))
else:
self.model.add(GRU(n_units[0], input_shape=(self.time_steps, self.nn_inputs), return_sequences = True, activation = activ_funct))
for i in range(2, n_layers+1):
if i < n_layers:
self.model.add(GRU(n_units[i-1], return_sequences = True, activation = activ_funct))
else:
self.model.add(GRU(n_units[len(n_units)-1], return_sequences = False, activation = activ_funct))
self.model.add(Dense(self.output))
adam = optimizers.adam(lr = learning_rate)
self.model.compile(loss = 'mse', optimizer = 'adam', metrics=['mse', 'mae', 'mape'])
self.model.fit(self.train_X, self.train_Y, epochs = n_epochs, batch_size = batch_size,
verbose=0, shuffle = False, validation_data = (self.test_X, self.test_Y))
# define network architecture # adapted from # # https://machinelearningmastery.com/tutorial-first-neural-network-python-keras/ # with a few tweaks model = Sequential() model.add(Dense(24, input_dim=memory_length, activation='relu')) # input_dim = #variables # Dense = fully connected, 12 = number of neurons in layer #model.add(Dense(16, activation='relu')) # 8 neurons model.add(Dense(8, activation='relu')) # 8 neurons model.add(Dense(1, activation='relu')) # signoid for 0<output<1 bounding # Compile model ADAM = optimizers.adam(lr=0.00005) model.compile(loss='mean_squared_error', optimizer=ADAM, metrics=['mae']) # Fit the model (training) [Finding best weigths for prediction] model.fit(X, Y, epochs=300, batch_size=50) scores = model.evaluate(X, Y) # (loss, metric) print(scores) #%% test #print(model.predict(np.reshape((2626,3269,3983,5018),(1,memory_length)))) #print(model.predict(np.reshape((38,53,64,73),(1,memory_length)))) #print(np.shape(np.reshape((2626,3269,3983,5018),(1,memory_length)))) #print(np.shape(model.predict(np.reshape((2626,3269,3983,5018),(1,memory_length)))))
# Yield for data generators
def generate_data_generator_for_two_images(genX1):
while True:
X1i = genX1.next()
yield X1i[0], X1i[1]
# Yield for data generators
dataset_train_gen = generate_data_generator_for_two_images(train_generator)
dataset_val_gen = generate_data_generator_for_two_images(
validation_generator)
############################################
# Training
optimizer = adam(lr=0.0001)
model.compile(optimizer=optimizer,
loss='mean_squared_error',
metrics=['accuracy'])
'''
Callbacks
'''
filepath = 'weights'
checkpointer = ModelCheckpoint(filepath=os.path.join(
filepath, '_epoch-{epoch:02d}_val-accu-{val_acc:.2f}.hdf5'),
verbose=1) #, save_best_only=True)
csv_logger = CSVLogger('logs/' + __model__ + '.log',
separator=',',
append=False)
model.fit_generator(dataset_train_gen,
x = BatchNormalization()(x)
x = Dropout(0.25)(x)
x = Dense(30, activation='relu',kernel_regularizer=regularizers.l2(0.01))(x)
x = BatchNormalization()(x)
x = Dropout(0.25)(x)
# and a logistic layer -- let's say we have 200 classes
predictions = Dense(2, activation='softmax')(x)
# this is the model we will train
head_model = Model(inputs=base_model.input, outputs=predictions)
'''
#compile the model
Adam = adam(lr=0.0001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-07,
decay=0.0,
amsgrad=False,
clipnorm=1.)
head_model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
head_model.fit_generator(train_generator,
steps_per_epoch=steps_epoch,
epochs=epochs,
validation_data=validation_generator,
validation_steps=validation_steps,
callbacks=[checkpoint])
end_model = load_model('voVGG16_2c_2.h5')
#
# pred = tf.reshape(dense, (tf.shape(dense)[0], 1))
# optimizer = tf.train.AdamOptimizer(learning_rate=0.001)
# #cost = tf.losses.mean_squared_error(target, pred)
# cost = tf.reduce_mean(tf.square(target - pred))
# minimize = optimizer.minimize(cost)
ins = [1, 2, 3, 4, 2, 6]
outs = [0.7, 0.6, 0.5, 0.4, 0.1, 0.2]
rnn = Sequential()
lstm = GRU(nodes, input_shape=(1, 1), batch_size=1, stateful=True)
rnn.add(lstm)
#rnn.add(Dropout(0.5))
rnn.add(Dense(1))
opt = adam(lr=0.003, decay=0.0) # 1e-3)
rnn.compile(loss='mae', optimizer=opt)
print rnn.summary()
for epoch in tqdm(range(epochs)):
#for i in range(len(ins)):
rnn.fit(np.reshape(np.array(ins), (6, 1, 1)),
np.reshape(np.array(outs), (6, 1)),
epochs=1,
batch_size=1,
shuffle=False,
verbose=2 if epoch >= epochs - 2 else 0)
lstm.reset_states()
# old_rnn = rnn
# rnn = Sequential()
train = files[:threshold]
test = files[threshold:]
# ind = int(len(data) * split)
# x_train, x_test = data[:ind], data[ind:]
# x_train, x_test = x_train, x_test
# x_train = normalization(x_train, 0, 255, -1, 1)
# x_test = normalization(x_test, 0, 255, -1, 1)
# print(data.shape)
# print(x_train.shape)
# print(x_test.shape)
# autoencoder = model(x_train)
autoencoder = model(train[0])
opt = optimizers.adam(lr=1e-4)
autoencoder.compile(loss='mean_squared_error',
optimizer=opt,
metrics=['accuracy'])
autoencoder_train = autoencoder.fit_generator(
generate_arrays_from_file(train, BATCH_SIZE),
validation_data=generate_arrays_from_file(test, BATCH_SIZE),
steps_per_epoch=160000 // BATCH_SIZE,
validation_steps=40000 // BATCH_SIZE,
epochs=EPOCHS)
# autoencoder_train = autoencoder.fit(x_train, x_train,
# batch_size=BATCH_SIZE,
# epochs=EPOCHS,
# validation_data=(x_test, x_test))
strides=1))
model.add(BatchNormalization())
# Block 4
model.add(Conv1D(128,
3,
padding='valid',
activation='elu',
strides=1))
model.add(MaxPooling1D(pool_size=2))
model.add(BatchNormalization())
# Recurrent Block
model.add(LSTM(256))
model.add(BatchNormalization())
# Dense Block
model.add(Dense(1028))
model.add(Activation('elu'))
model.add(Dropout(0.2))
model.add(BatchNormalization())
model.add(Dense(len(cats)))
model.add(Activation('softmax'))
sgd = adam(lr=5e-5, epsilon=1e-7)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
model.fit(X_train_n, y_train, validation_data=[X_test_n, y_test],
batch_size=512, epochs=10)
def train(num_epochs=10,
batch_size=8,
R=16,
BN=False,
name="TEST",
folder_rasters="../Data/rasters_2019",
occur_file="../Data/full_occur.csv",
taxanames="../Data/test/taxaNameTest.csv",
onlytest=0,
w=2,
opt="sgd",
loss="ce",
gam=2,
LR=0.001,
sep=True,
decay=True,
patience=2,
scale=0.2,
tolerance=10,
metric="acc",
Kacc=30,
alt=1,
drop=1,
weighted=True,
actemb=None,
act="leakyrelu",
actfc="relu",
vocab_size=1055,
window=50,
EMBNP=100,
regional=20,
archi=0,
grinton_mode=0,
gpus=1,
init_weights=None,
runmode=0):
print("Radius ", R, sep=" ")
print("BatchNorm ", BN)
clim_vars = [
'alti', 'etp', 'chbio_1', 'chbio_2', 'chbio_3', 'chbio_4', 'chbio_5',
'chbio_6', 'chbio_7', 'chbio_8', 'chbio_9', 'chbio_10', 'chbio_11',
'chbio_12', 'chbio_13', 'chbio_14', 'chbio_15', 'chbio_16', 'chbio_17',
'chbio_18', 'chbio_19', 'proxi_eau_fast'
]
pedo_vars = [
'awc_top', 'bs_top', 'cec_top', 'crusting', 'dgh', 'dimp', 'erodi',
'oc_top', 'pd_top', 'text'
]
comps = {
'clim': (clim_vars, 0),
'landcover': (["clc"], 1),
'pedo': (pedo_vars, 1)
}
patch_extractors = []
if (archi != 2):
for k in comps.keys():
trt = comps.get(k)[1]
varlist = comps.get(k)[0]
if (trt == 0):
patch_extractor = PatchExtractor(folder_rasters,
size=R,
verbose=True)
for v in varlist:
patch_extractor.append(v, normalized=True)
patch_extractors.append(patch_extractor)
else: ### one patch extractor for each raster (case of categorical variables)
for v in varlist:
pe = PatchExtractor(folder_rasters, size=R, verbose=True)
pe.append(v, normalized=False)
patch_extractors.append(pe)
allspecies = pd.read_csv(taxanames, sep=";", decimal=".")
testspecies = allspecies[allspecies.test == True]["glc19SpId"]
del allspecies
th = 1
dataset = pd.read_csv("../Data/occurrence/full_occur.csv",
sep=";",
decimal=".")
if (onlytest == 1): ##Train only on test species
dataset = dataset[dataset["glc19SpId"].isin(testspecies.tolist())]
th = 0
classes = dataset["glc19SpId"].sort_values().unique()
prevs = np.array(
[len(dataset[dataset["glc19SpId"] == c]) for c in classes])
freq = classes[np.where(prevs > th)[0]]
dataset = dataset[dataset["glc19SpId"].isin(freq)]
class_codes = np.array(range(0, len(freq)))
for i in range(len(freq)):
dataset["glc19SpId"] = dataset["glc19SpId"].replace(
freq[i], class_codes[i])
teststatus = [w * int(x in testspecies.tolist()) for x in freq]
encoding = pd.DataFrame(
data=np.stack(
(freq, class_codes, prevs[np.where(prevs > th)[0]], teststatus),
axis=1),
columns=["glc19SpId", "codes", "prevalences", "teststatus"])
encoding.to_csv(name + "_trace_encoding.csv",
sep=";",
decimal=".",
index=False)
partitions = OccurrencePartition("stratified", dataset)
partitions.cross_val_part(0.9, encoding)
partitions.shuffle_idx()
## Example of dataset
x_train = [
tuple(x) for x in (dataset.iloc[partitions.train_idx, :]
)[["Latitude", "Longitude"]].values
]
x_val = [
tuple(x) for x in (dataset.iloc[partitions.val_idx, :]
)[["Latitude", "Longitude"]].values
]
y_train = dataset.iloc[partitions.train_idx, :]["glc19SpId"]
y_val = dataset.iloc[partitions.val_idx, :]["glc19SpId"]
data_train_generator = GlcGenerator(patch_extractors,
x_train,
y_train,
comps,
batch_size,
shuffle=True,
name="train",
folder_np="../Data/retained_np/",
window=window,
vocab_size=vocab_size,
archi=archi)
data_val_generator = GlcGenerator(patch_extractors,
x_val,
y_val,
comps,
batch_size,
shuffle=False,
name="valid",
folder_np="../Data/retained_np/",
window=window,
vocab_size=vocab_size,
archi=archi)
gp = GrintonParams(NBPLANTS=partitions.get_poolsize(),
R=R,
BN=BN,
NBNP=vocab_size,
window=window,
EMBNP=EMBNP)
gp.update_params(alt=alt, drop=drop, act=act, actfc=actfc, actemb=actemb)
if (archi != 2):
#### First architecture: Grinnell ####
Grinnell = GrinnellianNiche(sep, archi)
if (sep):
Grinnell.create_grinnell(gp.Anames, gp.Pnames,
gp.topoHydroClim_params, gp.pedo_params,
gp.anthropo_params, gp.ft_params_sep,
gp.spat_params_list, gp.trt,
gp.feat_names_list, gp.im_name_list)
else:
Grinnell.create_grinnell(gp.Anames, gp.Pnames,
gp.topoHydroClim_params, gp.pedo_params,
gp.anthropo_params, gp.ft_params_join,
gp.spat_params_list, gp.trt,
gp.feat_names_list, gp.im_name_list)
### Plot model params and architecture ###
Grinnell.plot_grinnell(name + "_grinnell.png")
#Grinnell.grinnell.summary()
if (archi != 1):
### Second architecture: Elton ####
if (archi == 2):
isfinal = True
else:
isfinal = False
Elton = EltonianNiche(final=isfinal)
Elton.create_elton(bio_params=gp.bio_params, emb=0)
#Elton.plot_elton(name+"_elton.png")
#Elton.elton.summary()
if (archi == 1):
parallel_model = Grinnell.grinnell
elif (archi == 2):
parallel_model = Elton.elton
else:
grinton = Grinton(Grinnell.grinnell, Elton.elton)
if grinton_mode:
grinton.create_grinton(gp.ensemble_grinton_params)
else:
grinton.create_grinton(gp.joint_grinton_params)
#grinton.plot_grinton(name+"_grinton.png")
parallel_model = grinton.grinton
parallel_model.summary()
### use GPU for data parallelism
if gpus > 1:
parallel_model = multi_gpu_model(parallel_model, gpus=4)
if (init_weights != None):
parallel_model.load_weights(init_weights)
if (opt == "adam"):
optimizer = adam(lr=LR)
else:
optimizer = sgd(lr=LR, momentum=0.9, nesterov=True)
if (loss == "fl"):
obj = focal_loss_softmax(gamma=gam)
else:
obj = "sparse_categorical_crossentropy"
if (metric == "acc"):
met = "sparse_categorical_accuracy"
else:
met = topk_accuracy(Kacc)
parallel_model.compile(optimizer, obj, [met])
#Grinnell.compile_grinnell(optimizer,obj,[topk_accuracy(3)])
if runmode:
#### Callbacks ####
callbcks = []
##TensorBoard
tbc = TensorBoard(log_dir='./logs_' + name)
callbcks.append(tbc)
##Training hyperparameters update => weight decay
if (decay):
wdc = ReduceLROnPlateau(monitor='val_loss',
factor=scale,
patience=patience,
min_lr=0.000001,
min_delta=0.001)
esc = EarlyStopping(monitor='val_loss',
min_delta=1E-4,
patience=tolerance)
callbcks.append(wdc)
callbcks.append(esc)
##Checkpointing the model periodically
filepath = name + "_weights.{epoch:02d}-{val_loss:.2f}.hdf5"
cpc = ModelCheckpoint(filepath,
monitor='val_loss',
verbose=0,
save_best_only=False,
save_weights_only=True,
mode='auto',
period=5)
callbcks.append(cpc)
#### Start training #####
#batch_size=2
if (onlytest == 2):
classweights = [
max(1, encoding.loc[i, "teststatus"])
for i in range(len(encoding))
]
elif (weighted):
classweights = (1 / encoding["prevalences"].values).tolist()
else:
classweights = np.ones(len(encoding))
#oi=parallel_model.get_weights()
#init_weights="elton_standalone_weights.15-6.57.hdf5"
#ol=parallel_model.get_weights()
print("Train Mode")
train_history = parallel_model.fit_generator(
generator=data_train_generator,
steps_per_epoch=(len(partitions.train_idx) // batch_size),
epochs=num_epochs,
verbose=1,
validation_data=data_val_generator,
validation_steps=(len(partitions.val_idx) // batch_size),
use_multiprocessing=False,
shuffle=True,
callbacks=callbcks,
class_weight=classweights,
workers=1)
print("End of training")
#perform_test=Grinnell.grinnell.evaluate_generator(generator=data_val_generator,verbose=1)
#trainex=data_train_generator.extracted
#validex=data_val_generator.extracted
### Save model ###
print("Saving model")
path = name + ".h5"
parallel_model.save_weights(path)
return train_history
else:
if onlytest == 1:
encod_file = "pretrain/test_encoding.csv"
else:
encod_file = "pretrain/full_encoding.csv"
predict(parallel_model,
patch_extractors,
comps,
testset="../Data/test/testSet.csv",
encoding_file=encod_file,
R=R,
archi=archi,
folder_rasters=folder_rasters,
window=window,
vocab_size=vocab_size,
run_name=name)
def create_dueling_net(img_channels, img_rows, img_cols, n_conv1=32, n_conv2=64,
n_conv3=64, n_out1=512, n_out2=-1, lr=.001,
n_actions=4, loss='mse', use_perm_drop=False, drop_o=.25):
def make_output(x):
x = -K.mean(x, axis=1, keepdims=True)
x = K.tile(x, 4)
return x
def make_output_shape(input_shape):
shape = list(input_shape)
return tuple(shape)
def perm_drop(x):
return K.dropout(x, .25)
# input for the netwrok
input = Input(shape=(img_channels, img_rows, img_cols))
# conv layers - shared by both netwroks
conv1 = Convolution2D(n_conv1, 5, 5, border_mode='same',subsample=(2,2))(input)
prelu1 = PReLU()(conv1)
conv2 = Convolution2D(n_conv2, 3, 3, border_mode='same',subsample=(2,2))(prelu1)
prelu2 = PReLU()(conv2)
conv3 = Convolution2D(n_conv2, 3, 3, border_mode='same')(prelu2)
prelu3 = PReLU()(conv3)
flatten = Flatten()(prelu3)
# A(s,a)
dense11 = Dense(n_out1)(flatten)
if not use_perm_drop:
prelu31 = PReLU()(dense11)
else:
prelu310 = PReLU()(dense11)
prelu31 = Lambda(perm_drop, output_shape=make_output_shape)(prelu310)
dense21 = Dense(n_actions)(prelu31)
out1 = Activation('linear')(dense21)
# V(s)
dense12 = Dense(n_out1)(flatten)
if not use_perm_drop:
prelu32 = PReLU()(dense12)
else:
prelu320 = PReLU()(dense12)
prelu32 = Lambda(perm_drop, output_shape=make_output_shape)(prelu320)
dense22 = Dense(1)(prelu32)
out2 = Activation('linear')(dense22)
out2 = RepeatVector(n_actions)(out2)
out2 = Reshape((n_actions,))(out2)
# - E[ A(s,a) ]
out3 = Lambda(make_output, output_shape=make_output_shape)(out1)
output = merge([out1, out2, out3], mode='sum', concat_axis=1)
model = Model(input=input,output=output)
model.compile(loss=loss, optimizer=adam(lr=lr))
return model
# Activation
model.add(Activation('relu'))
# Dropout
model.add(Dropout(0.5))
# Fully-Connected Layer -4
model.add(Dense(10, name='fc4'))
# Activation
model.add(Activation('relu'))
# Output Layer
model.add(Dense(1, name='fc5'))
return model
model = DNN()
adm = adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
model.compile(optimizer=adm, loss='mse')
model.fit(x_train_resize1,
y_train_all1,
batch_size=256,
nb_epoch=5,
validation_split=0.01,
shuffle=True)
json_string = model.to_json()
with open("model.json", "w") as json_file:
json_file.write(json_string)
print("DNN architecture Saved,")
out = Conv1D(64,
3,
activation='elu',
padding='valid',
strides=1)(out)
out = BatchNormalization()(out)
out = Bidirectional(LSTM(512, return_sequences=True))(out)
out = attention_3d_block(out)
out = Flatten()(out)
out = Dense(1, activation='sigmoid')(out)
model = Model(inputs, out)
sgd = adam(0.0005)
model.compile(loss='binary_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
model.set_weights(model_weights)
###############################################################################
def predict(tweet):
tweet_p = process_tweet(tweet)
ttp = tokenizer.texts_to_sequences([tweet_p])
ttpt = pad_sequences(ttp, maxlen=30, padding='post', truncating='post')
t_idf = vectorizer.transform([tweet_p])
return (( 0.5 * model.predict(ttpt)[0,0]) +
( 0.5 * clf.predict_proba(t_idf)[0][1]))
########################
x_train = np.append(x_train, [np.fliplr(x) for x in x_train], axis=0)
y_train = np.append(y_train, [np.fliplr(x) for x in y_train], axis=0)
###################
learning_rate = 0.01
input_layer = Input((img_size_target, img_size_target, 1))
output_layer = build_model(input_layer, 16, 0.5)
model1 = Model(input_layer, output_layer)
c = optimizers.adam(lr=learning_rate / 2)
model1.compile(loss="binary_crossentropy",
optimizer=c,
metrics=[my_iou_metric])
model1.summary()
#######################
early_stopping = EarlyStopping(monitor='my_iou_metric',
mode='max',
patience=15,
verbose=1)
model_checkpoint = ModelCheckpoint(save_model_name,
monitor='my_iou_metric',
mode='max',
save_best_only=True,
verbose=1)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
__author__ = 'Yunchuan Chen'
from utils import get_unigram_probtable
from models import NCELangModel
from keras.optimizers import adam
NB_RUN_WORDS = 100000000
NB_VOCAB = 10000
NB_RUN_VAL = 100000
NB_EVALUATE = 5000000
SAVE_PATH = '../data/models/lang/nce0-neg50-e128-c128-lr0.01.pkl'
DATA_PATH = '../data/corpus/wiki-sg-norm-lc-drop-bin.bz2'
BATCH_SIZE = 256
VAL_INTER = 1200
unigram_table = get_unigram_probtable(nb_words=NB_VOCAB)
opt = adam(lr=0.01)
model = NCELangModel(vocab_size=NB_VOCAB, nb_negative=50, embed_dims=128, context_dims=128,
negprob_table=unigram_table, optimizer=opt)
model.compile()
model.train(data_file=DATA_PATH,
save_path=SAVE_PATH,
batch_size=BATCH_SIZE, train_nb_words=NB_RUN_WORDS,
val_nb_words=NB_EVALUATE, train_val_nb=NB_RUN_VAL, validation_interval=VAL_INTER)
def custom_error(y_true, y_pred, Qsa):
cce = 0.001 * (y_true - y_pred) * Qsa
return cce
#nitialize the Reward predictor model
Qmodel = Sequential()
#model.add(Dense(num_env_variables+num_env_actions, activation='tanh', input_dim=dataX.shape[1]))
Qmodel.add(Dense(2048, activation='tanh', input_dim=dataX.shape[1]))
#Qmodel.add(Dropout(0.2))
Qmodel.add(Dense(64 * 2, activation='relu'))
#Qmodel.add(Dropout(0.2))
#Qmodel.add(Dense(256, activation='relu'))
#Qmodel.add(Dropout(0.2))
Qmodel.add(Dense(dataY.shape[1]))
opt = optimizers.adam(lr=learning_rate)
Qmodel.compile(loss='mse', optimizer=opt, metrics=['accuracy'])
#initialize the action predictor model
action_predictor_model = Sequential()
#model.add(Dense(num_env_variables+num_env_actions, activation='tanh', input_dim=dataX.shape[1]))
action_predictor_model.add(
Dense(2048, activation='tanh', input_dim=apdataX.shape[1]))
#action_predictor_model.add(Dropout(0.2))
action_predictor_model.add(Dense(64 * 2, activation='relu'))
#action_predictor_model.add(Dropout(0.2))
#action_predictor_model.add(Dense(256, activation='relu'))
#action_predictor_model.add(Dropout(0.2))
action_predictor_model.add(Dense(apdataY.shape[1]))
opt2 = optimizers.adam(lr=apLearning_rate)
parser.add_option("-s", "--save", type="str", dest="save", default='',
help="amount of training data (number of words)")
options, args = parser.parse_args()
nb_run_words = options.running_words
nb_vocab = options.vocab_size
nb_run_val = options.val_run
nb_evaluate = options.nb_evaluation
unigram_table = get_unigram_probtable(nb_words=nb_vocab,
save_path='../data/wiki-unigram-prob-size%d.pkl' %
nb_vocab)
if options.decay:
opt = AdamAnneal(lr=options.lr, lr_min=options.lr_min, gamma=options.gamma)
else:
opt = adam(lr=options.lr)
if options.log_file == '':
log_file = None
else:
log_file = options.log_file
if options.save == '':
save_path = None
else:
save_path = options.save
model = NCELangModelV2(vocab_size=nb_vocab, nb_negative=options.negative,
embed_dims=options.embed_size, context_dims=options.context_size,
negprob_table=unigram_table, optimizer=opt)
model.compile()
for key in keys:
score = np.log(mu*total/float(labels_dict[key]))
class_weight[key] = score if score > 1.0 else 1.0
return class_weight
class_count_fish = {0:1719,1:200, 2:117, 3:66, 4:176, 5:734, 6:299}
class_count_FoN = {0:2500, 1:500}
class_wt_fish = create_class_weight(class_count_fish)
class_wt_FoN = create_class_weight(class_count_FoN)
from keras.optimizers import adam
sgd = adam(lr=0.00001)
convnet_fish_or_no.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy'])
sgd2 = adam(lr=0.000007)
vgg16_fish.compile(optimizer=sgd2, loss='categorical_crossentropy', metrics=['accuracy'])
# Training Fish-or-no-fish
train_path = '/fp/train'
epoch = 1
while epoch <= 9:
