def Keras_B01(Xtr,Ytr,Xte,num_list, cat_list, Din, Dout,cv_i):
X_list = []
for col in cat_list:
X_list.append(Xte[col].values)
X_list.append(Xte[num_list].values)
X_test = X_list
l2_emb = 0.00001
#emb_layers=[]
cat_out = []
cat_in = []
#cat var
for idx, var_name in enumerate(cat_list):
x_in = Input(shape=(1,), dtype='int64', name=var_name+'_in')
input_dim = Din[var_name]
output_dim = Dout[var_name]
x_out = Embedding(input_dim,
output_dim,
input_length=1,
name = var_name,
embeddings_regularizer=l1(l2_emb))(x_in)
flatten_c = Flatten()(x_out)
#emb_layers.append(x_out)
cat_in.append(x_in)
cat_out.append(flatten_c)
x_emb = layers.concatenate(cat_out,name = 'emb')
#continuous variables
cont_in = Input(shape=(len(num_list),), name='continuous_input')
cont_out = Lambda(expand_dims, expand_dims_output_shape)(cont_in)
x_num = Flatten(name = 'num')(cont_out)
cat_in.append(cont_in)
#merge
x = layers.concatenate([x_emb,x_num],name = 'emb_num')
x = Dense(512 ,activation='relu')(x)
x = PReLU()(x)
x = Dropout(0.6)(x)
x = Dense(64)(x)
x = PReLU()(x)
x = Dropout(0.3)(x)
x = Dense(32)(x)
x = PReLU()(x)
x = Dropout(0.2)(x)
x = Dense(1, activation='sigmoid')(x)
model = Model(inputs = cat_in, outputs = x)
model.compile(optimizers.Adam(), loss='binary_crossentropy', metrics=['accuracy'])
batch_size = 256
model.fit_generator(generator=batch_generator(Xtr, Ytr,cat_list,num_list, batch_size),
epochs=10, verbose=0, steps_per_epoch= np.floor(Xtr.shape[0]/batch_size))
Yt = model.predict(X_test).flatten()
K.clear_session()
return Yt
y, encoder = preprocess_labels(labels)
X_test, ids = load_data('test.csv', train=False)
X_test, _ = preprocess_data(X_test, scaler)
nb_classes = y.shape[1]
print(nb_classes, 'classes')
dims = X.shape[1]
print(dims, 'dims')
print('Building model...')
model = Sequential()
model.add(Dense(512, input_shape=(dims, )))
model.add(PReLU())
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(512))
model.add(PReLU())
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(512))
model.add(PReLU())
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
xg_test = xgb.DMatrix(T)
p_w = np.zeros((T.shape[0], 9), np.float32)
#generate an ensemble of 10 DNN models and 10 GBM models
for r in range(10):
nb_classes = yt.shape[1]
print(nb_classes, 'classes')
dims = X.shape[1]
print(dims, 'dims')
np.random.seed((r + 1) * 10)
model = Sequential()
model.add(Dense(dims, 1000, init='glorot_uniform'))
model.add(PReLU((1000, )))
model.add(BatchNormalization((1000, )))
model.add(Dropout(0.5))
model.add(Dense(1000, 500, init='glorot_uniform'))
model.add(PReLU((500, )))
model.add(BatchNormalization((500, )))
model.add(Dropout(0.5))
model.add(Dense(500, nb_classes, init='glorot_uniform'))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer="adagrad")
print("Training DNN model...")
padding='post',
truncating='post',
value=0.)
array_train_x = np.reshape(
array_train_x, [array_train_x.shape[0], array_train_x.shape[1], 1])
rnn_model = Sequential()
init_one = keras.initializers.Ones()
##rnn_model.add(GRU(rnn_hidden_node, input_shape=(None, 1), kernel_initializer=init_one))
rnn_model.add(
LSTM(rnn_hidden_node,
input_shape=(None, 1),
kernel_initializer=init_one))
#rnn_model.add(SimpleRNN(rnn_hidden_node, input_shape=(None, 1), kernel_initializer=init_one))
rnn_model.add(BatchNormalization())
rnn_model.add(PReLU())
rnn_model.add(Dropout(rnn_dropout))
rnn_model.add(Dense(1))
rnn_model.add(Activation('sigmoid'))
rnn_model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['binary_accuracy'])
array_train_x = sequence.pad_sequences(rnn_train_x,
maxlen=None,
dtype='float64',
padding='post',
truncating='post',
value=0.)
array_train_x = np.reshape(
array_train_x, [array_train_x.shape[0], array_train_x.shape[1], 1])
def bottleneck(inp,
output,
internal_scale=4,
asymmetric=0,
dilated=0,
downsample=False,
dropout_rate=0.1):
# main branch
internal = output // internal_scale
encoder = inp
# 1x1
input_stride = 2 if downsample else 1 # the 1st 1x1 projection is replaced with a 2x2 convolution when downsampling
encoder = Conv2D(
internal,
(input_stride, input_stride),
# padding='same',
strides=(input_stride, input_stride),
use_bias=False)(encoder)
# Batch normalization + PReLU
encoder = BatchNormalization(momentum=0.1)(
encoder) # enet uses momentum of 0.1, keras default is 0.99
encoder = PReLU(shared_axes=[1, 2])(encoder)
# conv
if not asymmetric and not dilated:
encoder = Conv2D(internal, (3, 3), padding='same')(encoder)
elif asymmetric:
encoder = Conv2D(internal, (1, asymmetric),
padding='same',
use_bias=False)(encoder)
encoder = Conv2D(internal, (asymmetric, 1), padding='same')(encoder)
elif dilated:
encoder = Conv2D(internal, (3, 3),
dilation_rate=(dilated, dilated),
padding='same')(encoder)
else:
raise (Exception('You shouldn\'t be here'))
encoder = BatchNormalization(momentum=0.1)(
encoder) # enet uses momentum of 0.1, keras default is 0.99
encoder = PReLU(shared_axes=[1, 2])(encoder)
# 1x1
encoder = Conv2D(output, (1, 1), use_bias=False)(encoder)
encoder = BatchNormalization(momentum=0.1)(
encoder) # enet uses momentum of 0.1, keras default is 0.99
encoder = SpatialDropout2D(dropout_rate)(encoder)
other = inp
# other branch
if downsample:
other = MaxPooling2D()(other)
other = Permute((1, 3, 2))(other)
pad_feature_maps = output - inp.get_shape().as_list()[3]
tb_pad = (0, 0)
lr_pad = (0, pad_feature_maps)
other = ZeroPadding2D(padding=(tb_pad, lr_pad))(other)
other = Permute((1, 3, 2))(other)
encoder = add([encoder, other])
encoder = PReLU(shared_axes=[1, 2])(encoder)
return encoder
json.dump(word2idx, j)
autoencoder = Sequential()
autoencoder.add(
Embedding(input_dim=vocab_size,
output_dim=emb_dim,
input_length=maxlen,
weights=[embedding_weights]))
autoencoder.add(Dropout(0.5))
autoencoder.add(Lambda(lambda x: K.sum(x, axis=1), output_shape=(300, )))
autoencoder.add(Dropout(0.5))
autoencoder.add(Dense(512))
autoencoder.add(BatchNormalization())
autoencoder.add(PReLU())
autoencoder.add(Dropout(0.5))
autoencoder.add(RepeatVector(maxlen))
autoencoder.add(TimeDistributed(Dense(300)))
autoencoder.add(BatchNormalization())
autoencoder.add(PReLU())
autoencoder.add(Dropout(0.5))
autoencoder.add(TimeDistributed(Dense(vocab_size, activation='softmax')))
autoencoder.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print_summary(autoencoder.layers)
def build_cnn(dim, n_classes):
model = Sequential()
model.add(
Convolution2D(96,
11,
11,
subsample=(4, 4),
input_shape=(dim, dim, 3),
init='glorot_uniform',
border_mode='same'))
model.add(PReLU())
model.add(MaxPooling2D((3, 3), strides=(2, 2)))
model.add(
Convolution2D(256,
5,
5,
subsample=(1, 1),
init='glorot_uniform',
border_mode='same'))
model.add(PReLU())
model.add(MaxPooling2D((3, 3), strides=(2, 2)))
model.add(
Convolution2D(384,
3,
3,
subsample=(1, 1),
init='glorot_uniform',
border_mode='same'))
model.add(PReLU())
model.add(
Convolution2D(384,
3,
3,
subsample=(1, 1),
init='glorot_uniform',
border_mode='same'))
model.add(PReLU())
model.add(
Convolution2D(256,
3,
3,
subsample=(1, 1),
init='glorot_uniform',
border_mode='same'))
model.add(PReLU())
model.add(MaxPooling2D((3, 3), strides=(2, 2)))
model.add(Flatten())
model.add(Dropout(0.5))
model.add(Dense(2048, init='glorot_uniform'))
model.add(PReLU())
model.add(Dropout(0.5))
model.add(Dense(256, init='glorot_uniform'))
model.add(PReLU())
model.add(Dense(n_classes, init='glorot_uniform', activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def Regularize(layer,
params,
shared_layers=False,
name='',
apply_noise=True,
apply_batch_normalization=True,
apply_prelu=True,
apply_dropout=True,
apply_l2=True,
trainable=True):
"""
Apply the regularization specified in parameters to the layer
:param layer: Layer to regularize
:param params: Params specifying the regularizations to apply
:param shared_layers: Boolean indicating if we want to get the used layers for applying to a shared-layers model.
:param name: Name prepended to regularizer layer
:param apply_noise: If False, noise won't be applied, independently of params
:param apply_dropout: If False, dropout won't be applied, independently of params
:param apply_prelu: If False, prelu won't be applied, independently of params
:param apply_batch_normalization: If False, batch normalization won't be applied, independently of params
:param apply_l2: If False, l2 normalization won't be applied, independently of params
:return: Regularized layer
"""
shared_layers_list = []
if apply_noise and params.get('USE_NOISE', False):
shared_layers_list.append(
GaussianNoise(params.get('NOISE_AMOUNT', 0.01),
name=name + '_gaussian_noise',
trainable=trainable))
if apply_batch_normalization and params.get('USE_BATCH_NORMALIZATION',
False):
if params.get('WEIGHT_DECAY'):
l2_gamma_reg = l2(params['WEIGHT_DECAY'])
l2_beta_reg = l2(params['WEIGHT_DECAY'])
else:
l2_gamma_reg = None
l2_beta_reg = None
bn_mode = params.get('BATCH_NORMALIZATION_MODE', 0)
shared_layers_list.append(
BatchNormalization(mode=bn_mode,
gamma_regularizer=l2_gamma_reg,
beta_regularizer=l2_beta_reg,
name=name + '_batch_normalization',
trainable=trainable))
if apply_prelu and params.get('USE_PRELU', False):
shared_layers_list.append(
PReLU(name=name + '_PReLU', trainable=trainable))
if apply_dropout and params.get('DROPOUT_P', 0) > 0:
shared_layers_list.append(
Dropout(params.get('DROPOUT_P', 0.5),
name=name + '_dropout',
trainable=trainable))
if apply_l2 and params.get('USE_L2', False):
shared_layers_list.append(
Lambda(L2_norm, name=name + '_L2_norm', trainable=trainable))
# Apply all the previously built shared layers
for l in shared_layers_list:
layer = l(layer)
result = layer
# Return result or shared layers too
if shared_layers:
return result, shared_layers_list
return result
def run_network_keras(hidden_connections, hidden_layers, loss,
single_output=None, single_input=None,
normalise_input=None, normalise_output=None,
input_filter_types=None,
use_graph=False,
unknown_input_handler=None):
config['input_filter_types'] = input_filter_types
train_data = get_training_data(config,
tmp_file,
single_output,
single_input,
normalise_input,
normalise_output,
unknown_input_handler,
percentile_bin=config['percentile_bin'],
erase_above=config['erase_above'])
test_data = config['test_data'] # number of test sets
train_in = train_data['train_in']
train_out = train_data['train_out']
test_in = train_data['test_in']
test_out = train_data['test_out']
redshifts_train = train_data['redshifts_train']
galaxy_ids_test = train_data['galaxy_ids_test']
out_normaliser = train_data['out_normaliser']
mean_in = train_data['mean_in']
mean_out = train_data['mean_out']
stddev_in = train_data['stddev_in']
stddev_out = train_data['stddev_out']
print np.shape(train_in)
print np.shape(train_out)
print np.shape(test_in)
print np.shape(test_out)
LOG.info('Compiling neural network model')
optimiser = SGD(lr=learning_params['learning_rate'], momentum=learning_params['momentum'], decay=learning_params['decay'], nesterov=True)
input_dim = len(train_in[0])
try:
output_dim = len(train_out[0])
except:
output_dim = 1
model = Sequential()
model.add(Dense(output_dim=hidden_connections, input_dim=input_dim))
model.add(PReLU())
model.add(Dense(output_dim=input_dim, input_dim=hidden_connections))
model.compile(loss='mse', optimizer=RMSprop(lr=0.001), class_mode='binary')
model.fit(train_in, train_in, 5000, 10000, validation_split=0.3, verbose=True, show_accuracy=True)
for i in range(0, 30):
ans = model.predict(np.array([test_in[i]]))
print 'Test', test_in[i]
print 'Ans', ans[0]
print
print
exit()
def _cnn(imgs_dim, compile_=True):
model = Sequential()
model.add(_convolutional_layer(nb_filter=16, input_shape=imgs_dim))
model.add(BatchNormalization(axis=1))
model.add(PReLU(alpha_initializer=W_INIT))
model.add(_convolutional_layer(nb_filter=16))
model.add(BatchNormalization(axis=1))
model.add(PReLU(alpha_initializer=W_INIT))
model.add(MaxPooling2D(pool_size=(pool_size, pool_size)))
model.add(_convolutional_layer(nb_filter=32))
model.add(BatchNormalization(axis=1))
model.add(PReLU(alpha_initializer=W_INIT))
model.add(_convolutional_layer(nb_filter=32))
model.add(BatchNormalization(axis=1))
model.add(PReLU(alpha_initializer=W_INIT))
model.add(_convolutional_layer(nb_filter=32))
model.add(BatchNormalization(axis=1))
model.add(PReLU(alpha_initializer=W_INIT))
model.add(MaxPooling2D(pool_size=(pool_size, pool_size)))
model.add(_convolutional_layer(nb_filter=64))
model.add(BatchNormalization(axis=1))
model.add(PReLU(alpha_initializer=W_INIT))
model.add(_convolutional_layer(nb_filter=64))
model.add(BatchNormalization(axis=1))
model.add(PReLU(alpha_initializer=W_INIT))
model.add(_convolutional_layer(nb_filter=64))
model.add(BatchNormalization(axis=1))
model.add(PReLU(alpha_initializer=W_INIT))
model.add(MaxPooling2D(pool_size=(pool_size, pool_size)))
model.add(_convolutional_layer(nb_filter=128))
model.add(BatchNormalization(axis=1))
model.add(PReLU(alpha_initializer=W_INIT))
model.add(_convolutional_layer(nb_filter=128))
model.add(BatchNormalization(axis=1))
model.add(PReLU(alpha_initializer=W_INIT))
model.add(_convolutional_layer(nb_filter=128))
model.add(BatchNormalization(axis=1))
model.add(PReLU(alpha_initializer=W_INIT))
model.add(MaxPooling2D(pool_size=(pool_size, pool_size)))
model.add(_convolutional_layer(nb_filter=256))
model.add(BatchNormalization(axis=1))
model.add(PReLU(alpha_initializer=W_INIT))
model.add(_convolutional_layer(nb_filter=256))
model.add(BatchNormalization(axis=1))
model.add(PReLU(alpha_initializer=W_INIT))
model.add(_convolutional_layer(nb_filter=256))
model.add(BatchNormalization(axis=1))
model.add(PReLU(alpha_initializer=W_INIT))
model.add(MaxPooling2D(pool_size=(pool_size, pool_size)))
model.add(Dropout(rate=dropout))
model.add(Flatten())
model.add(_dense_layer(output_dim=PENULTIMATE_SIZE))
model.add(BatchNormalization())
model.add(PReLU(alpha_initializer=W_INIT))
if compile_:
model.add(Dropout(rate=dropout))
# Output: a vector of size (# of classes).
model.add(_dense_layer(output_dim=SOFTMAX_SIZE))
model.add(BatchNormalization())
model.add(Activation(activation='softmax'))
return compile_model(model)
# return model
return model
model.add(BatchNormalization((64,)))
model.add(Dropout(0.5))
model.add(Dense(64, nb_classes, init='glorot_uniform'))
model.add(Activation('softmax'))
"""
NN = 1024
dropout = 0.7
model = Sequential()
#model.add(Dense(dims, 30, W_regularizer = l1(.01)))
model.add(Dense(dims, 1024, init='glorot_uniform'))
model.add(Activation('tanh'))
model.add(PReLU((1024, )))
model.add(BatchNormalization((1024, )))
model.add(Dropout(dropout))
model.add(Dense(1024, NN, init='glorot_uniform'))
model.add(Activation('tanh'))
model.add(PReLU((NN, )))
model.add(BatchNormalization((NN, )))
model.add(Dropout(dropout))
model.add(Dense(NN, 20, init='glorot_uniform'))
model.add(Activation('tanh'))
model.add(PReLU((20, )))
model.add(BatchNormalization((20, )))
model.add(Dropout(dropout))
def Blender_CNN(num_of_res, num_of_inp, params):
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Flatten, Activation
from keras.layers.normalization import BatchNormalization
from keras.layers.advanced_activations import PReLU
level1_size = 300
if 'level1_size' in params:
level1_size = params['level1_size']
level2_size = 300
if 'level2_size' in params:
level2_size = params['level2_size']
level3_size = 250
if 'level3_size' in params:
level3_size = params['level3_size']
dropout_val_1 = 0.1
if 'dropout_val_1' in params:
dropout_val_1 = params['dropout_val_1']
dropout_val_2 = 0.1
if 'dropout_val_2' in params:
dropout_val_2 = params['dropout_val_2']
dropout_val_3 = 0.1
if 'dropout_val_3' in params:
dropout_val_3 = params['dropout_val_3']
activation_1 = 'prelu'
if 'activation_1' in params:
activation_1 = params['activation_1']
activation_2 = 'prelu'
if 'activation_2' in params:
activation_2 = params['activation_2']
activation_3 = 'prelu'
if 'activation_3' in params:
activation_3 = params['activation_3']
use_3rd_level = 1
if 'use_3rd_level' in params:
use_3rd_level = params['use_3rd_level']
model = Sequential()
model.add(Dense(level1_size, input_shape=(num_of_inp,)))
if activation_1 == 'prelu':
model.add(PReLU())
elif activation_1 == 'relu':
model.add(Activation('relu'))
else:
model.add(Activation('elu'))
model.add(Dropout(dropout_val_1))
model.add(Dense(level2_size))
if activation_2 == 'prelu':
model.add(PReLU())
elif activation_2 == 'relu':
model.add(Activation('relu'))
else:
model.add(Activation('elu'))
model.add(Dropout(dropout_val_2))
if use_3rd_level == 1:
model.add(Dense(level3_size))
if activation_3 == 'prelu':
model.add(PReLU())
elif activation_3 == 'relu':
model.add(Activation('relu'))
else:
model.add(Activation('elu'))
model.add(Dropout(dropout_val_3))
model.add(Dense(num_of_res, activation='sigmoid'))
return model
@author: Emrick """ from keras.models import Sequential from keras.models import Model from keras.layers import Convolution2D from keras.layers import MaxPooling2D from keras.layers import Flatten from keras.layers import Dense from keras.layers import Input from keras.layers.core import Dropout from keras.layers.advanced_activations import PReLU identifier = Sequential() x = Input((64, 64, 1)) y = (Convolution2D(32, 3, 3, border_mode="valid"))(x) y = PReLU(init='zero', weights=None, shared_axes=None)(y) y = (MaxPooling2D(pool_size=(2, 2)))(y) y = (Dropout(0.5))(y) y = (Convolution2D(32, 3, 3, border_mode="valid"))(y) y = PReLU(init='zero', weights=None, shared_axes=None)(y) y = (MaxPooling2D(pool_size=(2, 2)))(y) y = (Dropout(0.5))(y) y = (Convolution2D(32, 3, 3, border_mode="valid"))(y) y = PReLU(init='zero', weights=None, shared_axes=None)(y) y = (MaxPooling2D(pool_size=(2, 2)))(y) y = (Dropout(0.5))(y) y = (Flatten())(y) y = (Dense(output_dim=512, activation="relu"))(y) #y = (Dense(output_dim=256,activation = "relu"))(y) y = (Dense(output_dim=256, activation="relu"))(y)
def keras_mlp(train_feat, valid_feat, train_target, valid_target, arch, slr,
keepprob, problem, pred_file, batch_size, epochs):
# Params
batch_size = batch_size
epochs = epochs
# Standardize initial inputs to 0-1 range
train_feat, valid_feat = scale_0_1_multiple(train_feat, valid_feat)
print("SHAPES!!!", train_feat.shape, valid_feat.shape)
if problem == "classification":
print("Classification problem")
# Get class weights
ones = np.count_nonzero(train_target)
zeros = train_target.shape[0] - ones
zero_weight = float(zeros) / ones
# class_weight = {0:1.0, 1:zero_weight}
# class_weight = {0:zero_weight, 1:1.0}
class_weight = {0: 1.0, 1: 2.5}
print(class_weight)
# One-hot encode labels (2-Classes)
y_train = to_categorical(train_target, num_classes=2)
y_valid = to_categorical(valid_target, num_classes=2)
elif problem == "regression":
y_train = train_target
y_valid = valid_target
class_weight = None
# Architecture
init_features = train_feat.shape[1]
print("init arch: ", init_features, arch)
layers = arch_layers(init_features, arch)
print(layers)
# Build KERAS Fully connected network
model = Sequential()
model.add(Dense(layers[1], activation=PReLU(), input_dim=layers[0]))
model.add(BatchNormalization())
model.add(Dropout(keepprob))
for l in layers[2:]:
model.add(Dense(l, activation=PReLU()))
model.add(BatchNormalization())
model.add(Dropout(keepprob))
# Add the output layer and compile
if problem == "classification":
model.add(Dense(2, activation='softmax'))
# Optimizer
optimizer = Nadam(lr=slr, beta_1=0.9, beta_2=0.999)
# Compile
model.compile(optimizer=optimizer,
loss="binary_crossentropy",
metrics=[metrics.binary_crossentropy])
elif problem == "regression":
model.add(Dense(1))
# Optimizer
optimizer = Nadam(lr=slr, beta_1=0.9, beta_2=0.999)
# Compile
model.compile(optimizer=optimizer, loss="mse", metrics=["mse"])
print(model.summary())
# Run model
# reduce_lr = LearningRateScheduler(lambda x: 1e-3 * 0.9 ** x)
early_stopping = EarlyStopping(monitor='val_loss', patience=20)
# results = model.fit(train_feat, y_train, batch_size=batch_size, epochs=epochs,
# validation_data=(valid_feat, y_valid), verbose=2,
# callbacks=[reduce_lr], class_weight=class_weight)
# model.fit(x=np.concatenate((train_feat, valid_feat), axis=0),
# y=np.concatenate((y_train, y_valid), axis=0),
# batch_size=batch_size, epochs=5, validation_split=0.1, verbose=2,
# callbacks=[reduce_lr], class_weight=class_weight)
results = model.fit(x=train_feat,
y=y_train,
batch_size=batch_size,
epochs=200,
validation_split=0.1,
verbose=2,
callbacks=[early_stopping],
class_weight=class_weight)
# Clean up
# K.clear_session()
return results, model, early_stopping
X_test, _ = preprocess_data(X_test_list, scaler)
nb_classes = Y.shape[1]
print(nb_classes, 'classes')
dims = X.shape[1]
print(dims, 'dims')
print("Building model...")
neuro_num = 16
# setup deep NN
model = Sequential()
model.add(Dense(dims, neuro_num, init='glorot_uniform'))
model.add(PReLU((neuro_num,)))
model.add(BatchNormalization((neuro_num,)))
model.add(Dropout(0.5))
model.add(Dense(neuro_num, neuro_num, init='glorot_uniform'))
model.add(PReLU((neuro_num,)))
model.add(BatchNormalization((neuro_num,)))
model.add(Dropout(0.5))
model.add(Dense(neuro_num, neuro_num, init='glorot_uniform'))
model.add(PReLU((neuro_num,)))
model.add(BatchNormalization((neuro_num,)))
model.add(Dropout(0.5))
model.add(Dense(neuro_num, nb_classes, init='glorot_uniform'))
model.add(Activation('sigmoid'))
def create_dueling_net(img_channels,
img_rows,
img_cols,
n_conv1=32,
n_conv2=64,
n_conv3=64,
n_out1=512,
n_out2=-1,
lr=.001,
n_actions=4,
loss='mse',
use_perm_drop=False,
drop_o=.25):
def make_output(x):
x = -K.mean(x, axis=1, keepdims=True)
x = K.tile(x, 4)
return x
def make_output_shape(input_shape):
shape = list(input_shape)
return tuple(shape)
def perm_drop(x):
return K.dropout(x, .25)
# input for the netwrok
input = Input(shape=(img_channels, img_rows, img_cols))
# conv layers - shared by both netwroks
conv1 = Convolution2D(n_conv1, 5, 5, border_mode='same',
subsample=(2, 2))(input)
prelu1 = PReLU()(conv1)
conv2 = Convolution2D(n_conv2, 3, 3, border_mode='same',
subsample=(2, 2))(prelu1)
prelu2 = PReLU()(conv2)
conv3 = Convolution2D(n_conv2, 3, 3, border_mode='same')(prelu2)
prelu3 = PReLU()(conv3)
flatten = Flatten()(prelu3)
# A(s,a)
dense11 = Dense(n_out1)(flatten)
if not use_perm_drop:
prelu31 = PReLU()(dense11)
else:
prelu310 = PReLU()(dense11)
prelu31 = Lambda(perm_drop, output_shape=make_output_shape)(prelu310)
dense21 = Dense(n_actions)(prelu31)
out1 = Activation('linear')(dense21)
# V(s)
dense12 = Dense(n_out1)(flatten)
if not use_perm_drop:
prelu32 = PReLU()(dense12)
else:
prelu320 = PReLU()(dense12)
prelu32 = Lambda(perm_drop, output_shape=make_output_shape)(prelu320)
dense22 = Dense(1)(prelu32)
out2 = Activation('linear')(dense22)
out2 = RepeatVector(n_actions)(out2)
out2 = Reshape((n_actions, ))(out2)
# - E[ A(s,a) ]
out3 = Lambda(make_output, output_shape=make_output_shape)(out1)
output = merge([out1, out2, out3], mode='sum', concat_axis=1)
model = Model(input=input, output=output)
model.compile(loss=loss, optimizer=adam(lr=lr))
return model
def buildNN(architecture, training_params, input_dim):
"""Lay out a Neural Network as described in the YAML file."""
current_units = input_dim
model = Sequential()
is_first_layer = True
for layer in architecture:
layer_name = layer.keys()[0]
layer_params = layer[layer_name]
if layer_name == 'Activation':
model.add(Activation(layer_params['type']))
if layer_name == 'PReLU':
model.add(PReLU())
if layer_name == 'Dropout':
if is_first_layer:
model.add(
Dropout(layer_params['p'],
input_shape=(None, current_units)))
else:
model.add(Dropout(layer_params['p']))
if layer_name == 'Dense':
model.add(Dense(layer_params['num_units']))
current_units = layer_params['num_units']
if layer_name == 'Conv':
# a filter covers all predictions for a single time sample
# strides are made between time samples
model.add(Conv2D(layer_params['nb_filters'], 1))
current_units = layer_params['nb_filters'] * training_params[
'num_strides']
if layer_name == 'GRU':
model.add(
GRU(layer_params['num_units'],
input_shape=((None,
input_dim) if is_first_layer else None),
dropout=(layer_params['dropout']
if 'dropout' in layer_params else 0.0),
return_sequences=layer_params['next_GRU']))
current_units = layer_params['num_units']
if layer_name == 'LSTM':
model.add(
LSTM(layer_params['num_units'],
return_sequences=layer_params['next_GRU']))
current_units = layer_params['num_units']
if layer_name == 'BatchNormalization':
model.add(BatchNormalization())
if layer_name == 'Flatten':
model.add(Flatten())
if layer_name == 'Output':
model.add(Dense(N_EVENTS))
model.add(Activation('sigmoid'))
is_first_layer = False
if not training_params.has_key(
'optim') or training_params['optim'] == 'sgd':
optim = SGD(lr=training_params['lr'],
decay=float(training_params['decay']),
momentum=training_params['momentum'],
nesterov=True)
elif training_params['optim'] == 'adam':
optim = Adam(lr=training_params['lr'])
model.compile(loss='binary_crossentropy', optimizer=optim)
return model
def getModelGivenModelOptions(self, options):
np.random.seed(1234)
import keras
from keras.models import Sequential
from keras.layers.core import Dropout, Reshape, Dense, Activation, Flatten
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras.optimizers import Adadelta, SGD, RMSprop
from keras.constraints import maxnorm
from keras.layers.advanced_activations import PReLU
from keras.layers.normalization import BatchNormalization
from keras.regularizers import l1, l2, activity_l1, activity_l2
model = Sequential()
model.add(
Convolution2D(300,
4,
19,
input_shape=(1, 4, int(options.seq_length)),
W_learning_rate_multiplier=10.0))
model.add(BatchNormalization(mode=0, axis=1))
model.add(PReLU())
model.add(MaxPooling2D(pool_size=(1, 3)))
model.add(
Convolution2D(200,
1,
11,
W_learning_rate_multiplier=5.0,
W_constraint=maxnorm(m=7)))
model.add(BatchNormalization(mode=0, axis=1))
model.add(PReLU())
model.add(MaxPooling2D(pool_size=(1, 4)))
model.add(Convolution2D(200, 1, 7, W_constraint=maxnorm(m=7)))
model.add(BatchNormalization(mode=0, axis=1))
model.add(PReLU())
model.add(MaxPooling2D(pool_size=(1, 4)))
model.add(Flatten())
model.add(
Dense(1000,
activity_regularizer=activity_l1(0.00001),
W_constraint=maxnorm(m=7)))
model.add(PReLU())
model.add(Dropout(0.3))
model.add(
Dense(1000,
activity_regularizer=activity_l1(0.00001),
W_constraint=maxnorm(m=7)))
model.add(PReLU())
model.add(Dropout(0.3))
model.add(Dense(options.num_labels))
model.add(Activation("sigmoid"))
adam = keras.optimizers.Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08)
model.compile(loss="binary_crossentropy", optimizer=adam)
# model.save_weights('/mnt/lab_data/kundaje/users/pgreens/projects/hematopoiesis/models/saved_weights/dec_18_yaml_150K_neg_no_hema_diff_peaks_Leuk_75M_BassetModel_10xlearnrate_noweightnorml1.h5')
if options.init_weights_file != None:
model.save_weights(options.init_weights_file)
return model
def build_model(vectors, shape, settings):
'''Compile the model.'''
max_length, nr_hidden, nr_class, ncols = shape
# Declare inputs.
ids1 = Input(shape=(max_length, ), dtype='int32', name='words1')
ids2 = Input(shape=(max_length, ), dtype='int32', name='words2')
# Construct operations, which we'll chain together.
embed = _StaticEmbedding(vectors,
max_length,
nr_hidden,
dropout=0.2,
nr_tune=5000)
if settings['gru_encode']:
encode = _BiRNNEncoding(max_length,
nr_hidden,
dropout=settings['dropout'])
attend = _Attention(max_length, nr_hidden, dropout=settings['dropout'])
align = _SoftAlignment(max_length, nr_hidden)
compare = _Comparison(max_length, nr_hidden, dropout=settings['dropout'])
entail = _Entailment(nr_hidden, nr_class, dropout=settings['dropout'])
# Declare the model as a computational graph.
sent1 = embed(ids1) # Shape: (i, n)
sent2 = embed(ids2) # Shape: (j, n)
if settings['gru_encode']:
sent1 = encode(sent1)
sent2 = encode(sent2)
attention = attend(sent1, sent2) # Shape: (i, j)
align1 = align(sent2, attention)
align2 = align(sent1, attention, transpose=True)
feats1 = compare(sent1, align1)
feats2 = compare(sent2, align2)
scores = entail(feats1, feats2)
dense_input = Input(shape=(ncols, ))
d = Dense(256, kernel_initializer='he_normal')(dense_input)
d = PReLU()(d)
d = BatchNormalization()(d)
d = Dropout(0.4)(d)
d2 = Dense(512, kernel_initializer='he_normal')(d)
d2 = PReLU()(d2)
d2 = BatchNormalization()(d2)
d2 = Dropout(0.2)(d2)
d3 = Dense(512, kernel_initializer='he_normal')(d2)
d3 = PReLU()(d3)
d3 = BatchNormalization()(d3)
d3 = Dropout(0.2)(d3)
merged = concatenate([feats1, feats2, d3])
merged = Dropout(0.2)(merged)
merged = BatchNormalization()(merged)
merged = Dense(512)(merged)
merged = PReLU()(merged)
merged = Dropout(0.2)(merged)
merged = BatchNormalization()(merged)
preds = Dense(1, activation='sigmoid')(merged)
model = Model(input=[ids1, ids2, dense_input], output=[preds])
model.compile(optimizer=Adam(lr=settings['lr']),
loss='binary_crossentropy',
metrics=['accuracy'])
return model
def get_model(args):
# Dataset config
config = data_constants[args.dataset.lower()]
inputCNNshape = config['lstm_inputCNNshape']
inputMLPshape = config['lstm_inputMLPshape']
nb_classes = config['nb_classes']
# Build the CNN - pre-cross-connections
inputCNN = Input(shape=inputCNNshape)
inputNorm = TimeDistributed(Flatten())(inputCNN)
inputNorm = Masking(mask_value=0.)(inputNorm)
inputNorm = TimeDistributed(Reshape((90, 120, 1)))(inputNorm)
inputNorm = BatchNormalization(axis=1)(inputNorm)
conv = TimeDistributed(
Convolution2D(8, 3, 3, border_mode='same', activation='relu'), name='conv11')(inputNorm)
pool = TimeDistributed(
MaxPooling2D((2,2), strides=(2, 2)), name='maxpool1')(conv)
# Build the MLP - pre-cross-connections
inputMLP = Input(shape=inputMLPshape)
inputMasked = Masking(mask_value=0., input_shape=inputMLPshape)(inputMLP)
fcMLP = TimeDistributed(
Dense(32, activation='relu'), name='fc1')(inputMasked)
# Add the 1st round of cross-connections - CNN to MLP
x21 = TimeDistributed(Convolution2D(8, 1, 1, border_mode='same'))(pool)
x21 = TimeDistributed(PReLU())(x21)
x21 = TimeDistributed(Flatten())(x21)
x21 = TimeDistributed(Dense(32))(x21)
x21 = TimeDistributed(PReLU())(x21)
# Add 1st shortcut (residual connection) from CNN input to MLP
short1_2dto1d = TimeDistributed(MaxPooling2D((4,4), strides=(4,4)))(inputNorm)
short1_2dto1d = TimeDistributed(Flatten())(short1_2dto1d)
short1_2dto1d = TimeDistributed(Dense(32))(short1_2dto1d)
short1_2dto1d = TimeDistributed(PReLU())(short1_2dto1d)
# Cross-connections - MLP to CNN
x12 = TimeDistributed(Dense(14*29))(fcMLP)
x12 = TimeDistributed(PReLU())(x12)
x12 = TimeDistributed(Reshape((14,29,1)))(x12)
x12 = TimeDistributed(Conv2DTranspose(8, (32, 32), padding='valid'))(x12)
x12 = TimeDistributed(PReLU(), name='x1_to_cnn')(x12)
# 1st shortcut (residual connection) from MLP input to CNN
short1_1dto2d = TimeDistributed(Dense(14*29))(inputMasked)
short1_1dto2d = TimeDistributed(PReLU())(short1_1dto2d)
short1_1dto2d = TimeDistributed(Reshape((14,29,1)))(short1_1dto2d)
short1_1dto2d = TimeDistributed(Conv2DTranspose(8, (32, 32), padding='valid'))(short1_1dto2d)
short1_1dto2d = TimeDistributed(PReLU(), name='res1_to_cnn')(short1_1dto2d)
# CNN - post-cross-connections 1
pool = add([pool, short1_1dto2d])
merged = concatenate([pool, x12])
conv = TimeDistributed(
Convolution2D(16, 3, 3, border_mode='same', activation='relu'), name='conv21')(merged)
pool = TimeDistributed(
MaxPooling2D((2,2), strides=(2, 2)), name='maxpool2')(conv)
# MLP - post-cross-connections 1
fcMLP = add([fcMLP, short1_2dto1d])
fcMLP = concatenate([fcMLP, x21])
fcMLP = TimeDistributed(
Dense(32, activation='relu'), name='fc2')(fcMLP)
# Add the 2nd round of cross-connections - CNN to MLP
x21 = TimeDistributed(Convolution2D(16, 1, 1, border_mode='same'))(pool)
x21 = TimeDistributed(PReLU())(x21)
x21 = TimeDistributed(Flatten())(x21)
x21 = TimeDistributed(Dense(64))(x21)
x21 = TimeDistributed(PReLU())(x21)
# Add 2nd shortcut (residual connection) from CNN input to MLP
short2_2dto1d = TimeDistributed(MaxPooling2D((8,8), strides=(8,4)))(inputNorm)
short2_2dto1d = TimeDistributed(Flatten())(short2_2dto1d)
short2_2dto1d = TimeDistributed(Dense(32))(short2_2dto1d)
short2_2dto1d = TimeDistributed(PReLU())(short2_2dto1d)
# Cross-connections - MLP to CNN
x12 = TimeDistributed(Dense(7*15))(fcMLP)
x12 = TimeDistributed(PReLU())(x12)
x12 = TimeDistributed(Reshape((7,15,1)))(x12)
x12 = TimeDistributed(Conv2DTranspose(16, (16,16), padding='valid'))(x12)
x12 = TimeDistributed(PReLU(), name='x2_to_cnn')(x12)
# 2nd shortcut (residual connection) from MLP input to CNN
short2_1dto2d = TimeDistributed(Dense(7*15))(inputMasked)
short2_1dto2d = TimeDistributed(PReLU())(short2_1dto2d)
short2_1dto2d = TimeDistributed(Reshape((7,15,1)))(short2_1dto2d)
short2_1dto2d = TimeDistributed(Conv2DTranspose(16, (16, 16), padding='valid'))(short2_1dto2d)
short2_1dto2d = TimeDistributed(PReLU(), name='res2_to_cnn')(short2_1dto2d)
# CNN - post-cross-connections 2
pool = add([pool, short2_1dto2d])
merged = concatenate([pool, x12])
reshape = TimeDistributed(
Flatten(), name='flatten1')(merged)
fcCNN = TimeDistributed(
Dense(64, activation='relu'), name='fcCNN')(reshape)
# Merge the models
fcMLP = add([fcMLP, short2_2dto1d])
merged = concatenate([fcCNN, fcMLP, x21])
merged = BatchNormalization(axis=1, name='mergebn')(merged)
merged = Dropout(0.5, name='mergedrop')(merged)
lstm = LSTM(64)(merged)
out = Dense(nb_classes, activation='softmax')(lstm)
# Return the model object
model = Model(input=[inputCNN, inputMLP], output=out)
return model
model4.add(BatchNormalization())
model5 = Sequential()
model5.add(Embedding(len(word_index) + 1, 300, input_length=40, dropout=0.2))
model5.add(LSTM(300, dropout_W=0.2, dropout_U=0.2))
model6 = Sequential()
model6.add(Embedding(len(word_index) + 1, 300, input_length=40, dropout=0.2))
model6.add(LSTM(300, dropout_W=0.2, dropout_U=0.2))
merged_model = Sequential()
merged_model.add(
Merge([model1, model2, model3, model4, model5, model6], mode='concat'))
merged_model.add(BatchNormalization())
merged_model.add(Dense(300))
merged_model.add(PReLU())
merged_model.add(Dropout(0.2))
merged_model.add(BatchNormalization())
merged_model.add(Dense(300))
merged_model.add(PReLU())
merged_model.add(Dropout(0.2))
merged_model.add(BatchNormalization())
merged_model.add(Dense(300))
merged_model.add(PReLU())
merged_model.add(Dropout(0.2))
merged_model.add(BatchNormalization())
merged_model.add(Dense(300))
merged_model.add(PReLU())
def GetNetArchitecture(input_shape):
model = Sequential()
model.add(ZeroPadding2D((1, 1), input_shape=input_shape))
model.add(Convolution2D(64, (3, 3), activation='linear'))
model.add(BatchNormalization())
model.add(
PReLU(alpha_initializer='zeros',
alpha_regularizer=None,
alpha_constraint=None,
shared_axes=None))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(64, (3, 3), activation='linear'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(128, (3, 3), activation='linear'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(128, (3, 3), activation='linear'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, (3, 3), activation='linear'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, (3, 3), activation='linear'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, (3, 3), activation='linear'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, (3, 3), activation='linear'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, (3, 3), activation='linear'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, (3, 3), activation='linear'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, (3, 3), activation='linear'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, (3, 3), activation='linear'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, (3, 3), activation='linear'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, (3, 3), activation='linear'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, (3, 3), activation='linear'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, (3, 3), activation='linear'))
model.add(BatchNormalization())
model.add(Activation('relu'))
# model.add(MaxPooling2D((2,2), strides=(2,2)))
model.add(Flatten())
model.add(Dense(4096, activation='relu')) # Vgg 16 uses 232
model.add(Dropout(0.5))
model.add(Dense(4096, activation='relu')) #vgg 16 uses 232
model.add(Dropout(0.5))
model.add(Dense(2, activation='softmax'))
return model
def _bn_prelu(input_layer):
"""Helper to build a BN -> prelu block
the activation function could either be relu or prelu
"""
norm = BatchNormalization()(input_layer)
return PReLU()(norm)
def model():
input_1 = Input(shape=(7, 7, 200))
input_2 = Input(shape=(27, 27, 30))
# BAM
CAB_conv1 = Conv2D(
16,
(3, 3),
padding='same',
strides=(1, 1),
kernel_initializer='glorot_uniform',
kernel_regularizer=l2(1e-4),
# use_bias=False
)(input_1)
CAB_bn1 = BatchNormalization()(CAB_conv1)
CAB_relu1 = PReLU()(CAB_bn1)
CAB_avg_pool1 = AveragePooling2D()(CAB_relu1)
CAB_conv4 = Conv2D(
32,
(3, 3),
padding='same',
strides=(1, 1),
kernel_initializer='glorot_uniform',
kernel_regularizer=l2(1e-4),
# use_bias=False
)(CAB_avg_pool1)
CAB_bn4 = BatchNormalization()(CAB_conv4)
CAB_relu4 = PReLU()(CAB_bn4)
CAB_conv5 = Conv2D(
32,
(3, 3),
padding='same',
strides=(1, 1),
kernel_initializer='glorot_uniform',
kernel_regularizer=l2(1e-4),
# use_bias=False
)(CAB_relu4)
CAB_bn5 = BatchNormalization()(CAB_conv5)
CAB_relu5 = PReLU()(CAB_bn5)
CAB_global_pool = GlobalAveragePooling2D()(CAB_relu5)
# ===================================================================================================================
CAB_reshape = Reshape(
(1, CAB_global_pool._keras_shape[1]))(CAB_global_pool)
CAB_conv6 = Conv1D(50, (32),
padding='same',
strides=(1),
kernel_initializer='glorot_uniform',
use_bias=False)(CAB_reshape)
CAB_bn6 = BatchNormalization()(CAB_conv6)
CAB_relu6 = PReLU()(CAB_bn6)
CAB_conv7 = Conv1D(200, (50),
padding='same',
strides=(1),
kernel_initializer='glorot_uniform',
use_bias=False)(CAB_relu6)
CAB_bn7 = BatchNormalization()(CAB_conv7)
CAB_sigmoid = Activation('sigmoid')(CAB_bn7)
# =================================================================================================================
CAB_mul = multiply([input_1, CAB_sigmoid])
# Spectral feature extraction
input_spe = Reshape((CAB_mul._keras_shape[1], CAB_mul._keras_shape[2],
CAB_mul._keras_shape[3], 1))(CAB_mul)
# input_spe = Reshape((input_1._keras_shape[1], input_1._keras_shape[2], input_1._keras_shape[3], 1))(input_1)
conv_spe1 = Conv3D(32, (1, 1, 7), padding='valid',
strides=(1, 1, 2))(input_spe)
bn_spe1 = BatchNormalization()(conv_spe1)
relu_spe1 = PReLU()(bn_spe1)
conv_spe11 = Conv3D(num_filters_spe, (1, 1, 7),
padding='same',
strides=(1, 1, 1))(relu_spe1)
bn_spe11 = BatchNormalization()(conv_spe11)
relu_spe11 = PReLU()(bn_spe11)
# ==============================resnet block===========================================
blockconv_spe1 = Conv3D(num_filters_spe, (1, 1, 7),
padding='same',
strides=(1, 1, 1))(relu_spe11)
blockbn_spe1 = BatchNormalization()(blockconv_spe1)
blockrelu_spe1 = PReLU()(blockbn_spe1)
conv_spe2 = Conv3D(num_filters_spe, (1, 1, 7),
padding='same',
strides=(1, 1, 1))(blockrelu_spe1)
add_spe1 = add([relu_spe11, conv_spe2])
bn_spe2 = BatchNormalization()(add_spe1)
relu_spe2 = PReLU()(bn_spe2)
blockconv_spe2 = Conv3D(num_filters_spe, (1, 1, 7),
padding='same',
strides=(1, 1, 1))(relu_spe2)
blockbn_spe2 = BatchNormalization()(blockconv_spe2)
blockrelu_spe2 = PReLU()(blockbn_spe2)
conv_spe4 = Conv3D(num_filters_spe, (1, 1, 7),
padding='same',
strides=(1, 1, 1))(blockrelu_spe2)
add_spe2 = add([relu_spe2, conv_spe4])
bn_spe4 = BatchNormalization()(add_spe2)
relu_spe4 = PReLU()(bn_spe4)
blockconv_spe3 = Conv3D(num_filters_spe, (1, 1, 7),
padding='same',
strides=(1, 1, 1))(relu_spe4)
blockbn_spe3 = BatchNormalization()(blockconv_spe3)
blockrelu_spe3 = PReLU()(blockbn_spe3)
conv_spe41 = Conv3D(num_filters_spe, (1, 1, 7),
padding='same',
strides=(1, 1, 1))(blockrelu_spe3)
add_spe3 = add([relu_spe4, conv_spe41])
# ===================================================================================================
bn_spe41 = BatchNormalization()(add_spe3)
relu_spe41 = PReLU()(bn_spe41)
add_all_spe = add([relu_spe2, relu_spe4, relu_spe41])
conv_spe6 = Conv3D(8, (1, 1, 97), padding='valid',
strides=(1, 1, 1))(add_all_spe)
bn_spe6 = BatchNormalization()(conv_spe6)
relu_spe6 = PReLU()(bn_spe6)
# =========================== Spatial feature extraction=====================================================
input_spa = Reshape((input_2._keras_shape[1], input_2._keras_shape[2],
input_2._keras_shape[3], 1))(input_2)
conv_spa1 = Conv3D(16, (5, 5, 30), padding='valid',
strides=(1, 1, 1))(input_spa)
print('conv_spa1 shape:', conv_spa1.shape)
bn_spa1 = BatchNormalization()(conv_spa1)
relu_spa1 = PReLU()(bn_spa1)
reshape_spa1 = Reshape(
(relu_spa1._keras_shape[1], relu_spa1._keras_shape[2],
relu_spa1._keras_shape[4], relu_spa1._keras_shape[3]))(relu_spa1)
conv_spa11 = Conv3D(num_filters_spa, (3, 3, 1),
padding='same',
strides=(1, 1, 1))(reshape_spa1)
bn_spa11 = BatchNormalization()(conv_spa11)
relu_spa11 = PReLU()(bn_spa11)
# SAM
VIS_conv1 = Conv3D(16, (1, 1, 16),
padding='valid',
strides=(1, 1, 1),
kernel_initializer='glorot_uniform',
kernel_regularizer=l2(1e-4))(relu_spa11)
VIS_BN1 = BatchNormalization()(VIS_conv1)
VIS_relu1 = Activation('relu')(VIS_BN1)
VIS_SHAPE1 = Reshape(
(VIS_relu1._keras_shape[1] * VIS_relu1._keras_shape[2],
VIS_relu1._keras_shape[4]))(VIS_relu1)
VIS_conv2 = Conv3D(16, (1, 1, 16),
padding='valid',
strides=(1, 1, 1),
kernel_initializer='glorot_uniform',
kernel_regularizer=l2(1e-4))(relu_spa11)
VIS_BN2 = BatchNormalization()(VIS_conv2)
VIS_relu2 = Activation('relu')(VIS_BN2)
VIS_SHAPE2 = Reshape(
(VIS_relu2._keras_shape[1] * VIS_relu2._keras_shape[2],
VIS_relu2._keras_shape[4]))(VIS_relu2)
trans_VIS_SHAPE2 = Permute((2, 1))(VIS_SHAPE2)
VIS_conv3 = Conv3D(16, (1, 1, 16),
padding='valid',
strides=(1, 1, 1),
kernel_initializer='glorot_uniform',
kernel_regularizer=l2(1e-4))(relu_spa11)
VIS_BN3 = BatchNormalization()(VIS_conv3)
VIS_relu3 = Activation('relu')(VIS_BN3)
VIS_SHAPE3 = Reshape(
(VIS_relu3._keras_shape[1] * VIS_relu3._keras_shape[2],
VIS_relu3._keras_shape[4]))(VIS_relu3)
VIS_mul1 = dot([VIS_SHAPE1, trans_VIS_SHAPE2], axes=(2, 1))
VIS_sigmoid = Activation('sigmoid')(VIS_mul1)
VIS_mul2 = dot([VIS_sigmoid, VIS_SHAPE3], axes=(2, 1))
VIS_SHAPEall = Reshape((23, 23, 16, 1))(VIS_mul2)
VIS_conv4 = Conv3D(16, (16, 1, 1),
padding='same',
strides=(1),
kernel_initializer='glorot_uniform',
kernel_regularizer=l2(1e-4))(VIS_SHAPEall)
VIS_BN4 = BatchNormalization()(VIS_conv4)
VIS_ADD = add([relu_spa11, VIS_BN4])
# ===================================resnet block================================================
blockconv_spa1 = Conv3D(num_filters_spa, (3, 3, 1),
padding='same',
strides=(1, 1, 1))(VIS_ADD)
blockbn_spa1 = BatchNormalization()(blockconv_spa1)
blockrelu_spa1 = PReLU()(blockbn_spa1)
conv_spa2 = Conv3D(num_filters_spa, (3, 3, 1), padding='same',
strides=(1))(blockrelu_spa1)
add_spa1 = add([VIS_ADD, conv_spa2])
bn_spa2 = BatchNormalization()(add_spa1)
relu_spa2 = PReLU()(bn_spa2)
blockconv_spa2 = Conv3D(num_filters_spa, (3, 3, 1),
padding='same',
strides=(1))(relu_spa2)
blockbn_spa2 = BatchNormalization()(blockconv_spa2)
blockrelu_spa2 = PReLU()(blockbn_spa2)
conv_spa4 = Conv3D(num_filters_spa, (3, 3, 1), padding='same',
strides=(1))(blockrelu_spa2)
add_spa2 = add([relu_spa2, conv_spa4])
bn_spa4 = BatchNormalization()(add_spa2)
relu_spa4 = PReLU()(bn_spa4)
blockconv_spa3 = Conv3D(num_filters_spa, (3, 3, 1),
padding='same',
strides=(1))(relu_spa4)
blockbn_spa3 = BatchNormalization()(blockconv_spa3)
blockrelu_spa3 = PReLU()(blockbn_spa3)
conv_spa41 = Conv3D(num_filters_spa, (3, 3, 1),
padding='same',
strides=(1))(blockrelu_spa3)
add_spa3 = add([relu_spa4, conv_spa41])
bn_spa41 = BatchNormalization()(add_spa3)
relu_spa41 = PReLU()(bn_spa41)
# spa-multi-feature-fusion
add_all_spa = add([relu_spa2, relu_spa4, relu_spa41])
# feature alignment
conv_spa6 = Conv3D(num_filters_spa, (5, 5, 1),
padding='valid',
strides=(1, 1, 1))(add_all_spa)
bn_spa6 = BatchNormalization()(conv_spa6)
relu_spa6 = PReLU()(bn_spa6)
conv_spa7 = Conv3D(num_filters_spa, (5, 5, 1),
padding='valid',
strides=(1, 1, 1))(relu_spa6)
bn_spa7 = BatchNormalization()(conv_spa7)
relu_spa7 = PReLU()(bn_spa7)
conv_spa8 = Conv3D(num_filters_spa, (5, 5, 1),
padding='valid',
strides=(1, 1, 1))(relu_spa7)
bn_spa8 = BatchNormalization()(conv_spa8)
relu_spa8 = PReLU()(bn_spa8)
conv_spa81 = Conv3D(num_filters_spa, (5, 5, 1),
padding='valid',
strides=(1, 1, 1))(relu_spa8)
bn_spa81 = BatchNormalization()(conv_spa81)
relu_spa81 = PReLU()(bn_spa81)
conv_spa9 = Conv3D(8, (1, 1, 16), padding='valid',
strides=(1, 1, 1))(relu_spa81)
bn_spa9 = BatchNormalization()(conv_spa9)
relu_spa9 = PReLU()(bn_spa9)
# Spectral features and spatial features are connected
feature_fusion = concatenate([relu_spe6, relu_spa9])
reshape_all = Reshape(
(feature_fusion._keras_shape[1], feature_fusion._keras_shape[2],
feature_fusion._keras_shape[4],
feature_fusion._keras_shape[3]))(feature_fusion)
# joint spectral-spatial feature extraction
conv_all1 = Conv3D(16, (3, 3, 3), padding='same',
strides=(1, 1, 1))(reshape_all)
print('convall1 shape:', conv_all1.shape)
bn_all1 = BatchNormalization()(conv_all1)
relu_all1 = PReLU()(bn_all1)
# SAM2
VIS_conv11 = Conv3D(16, (1, 1, 16),
padding='valid',
strides=(1, 1, 1),
kernel_initializer='glorot_uniform',
kernel_regularizer=l2(1e-4))(relu_all1)
VIS_BN11 = BatchNormalization()(VIS_conv11)
VIS_relu11 = Activation('relu')(VIS_BN11)
VIS_SHAPE11 = Reshape(
(VIS_relu11._keras_shape[1] * VIS_relu11._keras_shape[2],
VIS_relu11._keras_shape[4]))(VIS_relu11)
VIS_conv21 = Conv3D(16, (1, 1, 16),
padding='valid',
strides=(1, 1, 1),
kernel_initializer='glorot_uniform',
kernel_regularizer=l2(1e-4))(relu_all1)
VIS_BN21 = BatchNormalization()(VIS_conv21)
VIS_relu21 = Activation('relu')(VIS_BN21)
VIS_SHAPE21 = Reshape(
(VIS_relu21._keras_shape[1] * VIS_relu21._keras_shape[2],
VIS_relu21._keras_shape[4]))(VIS_relu21)
trans_VIS_SHAPE21 = Permute((2, 1))(VIS_SHAPE21)
VIS_conv31 = Conv3D(16, (1, 1, 16),
padding='valid',
strides=(1, 1, 1),
kernel_initializer='glorot_uniform',
kernel_regularizer=l2(1e-4))(relu_all1)
VIS_BN31 = BatchNormalization()(VIS_conv31)
VIS_relu31 = Activation('relu')(VIS_BN31)
VIS_SHAPE31 = Reshape(
(VIS_relu31._keras_shape[1] * VIS_relu31._keras_shape[2],
VIS_relu31._keras_shape[4]))(VIS_relu31)
VIS_mul11 = dot([VIS_SHAPE11, trans_VIS_SHAPE21], axes=(2, 1))
VIS_sigmoid1 = Activation('sigmoid')(VIS_mul11)
VIS_mul21 = dot([VIS_sigmoid1, VIS_SHAPE31], axes=(2, 1))
VIS_SHAPEall1 = Reshape((7, 7, 16, 1))(VIS_mul21)
VIS_conv41 = Conv3D(16, (16, 1, 1),
padding='same',
strides=(1),
kernel_initializer='glorot_uniform',
kernel_regularizer=l2(1e-4))(VIS_SHAPEall1)
VIS_BN41 = BatchNormalization()(VIS_conv41)
VIS_ADD1 = add([relu_all1, VIS_BN41])
conv_all2 = Conv3D(16, (3), padding='valid', strides=(1, 1, 1))(VIS_ADD1)
bn_all2 = BatchNormalization()(conv_all2)
relu_all2 = PReLU()(bn_all2)
flatten = Flatten()(relu_all2)
dense = Dense(units=512, activation="relu",
kernel_initializer="he_normal")(flatten)
drop = Dropout(0.5)(dense)
dense_2 = Dense(units=256,
activation="relu",
kernel_initializer="he_normal")(drop)
drop1 = Dropout(0.5)(dense_2)
dense_3 = Dense(units=nb_classes,
activation="softmax",
kernel_initializer="he_normal")(drop1)
model = Model(inputs=[input_1, input_2], outputs=dense_3)
sgd = keras.optimizers.sgd(lr=0.001, momentum=0.9)
# adam = Adam(lr=0.001)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
model.summary()
return model
#============================================================================================================================================================
from keras.layers.core import Dense, Activation, Dropout
from keras.layers.recurrent import LSTM
from keras.models import Sequential, load_model
from keras.optimizers import Adam
from keras.layers.advanced_activations import PReLU
from keras.callbacks import EarlyStopping
from keras.utils import plot_model
model = Sequential()
model.add(LSTM(input_dim=1, output_dim=50, return_sequences=True))
model.add(Dropout(0.2))
model.add(LSTM(input_dim=50, output_dim=100, return_sequences=False))
model.add(Dropout(0.2))
model.add(Dense(output_dim=1))
model.add(PReLU(
)) ############sigmoid,tanh,relu,Leakyrelu,LeakyRelu(alpha = 0.001),PReLU()
optimizer = Adam(lr=0.0001, decay=1e-6)
model.compile(loss='mse', optimizer=optimizer, metrics=['accuracy'])
if os.path.exists('log/stock_price_pred_model.h5'):
model = load_model('log/stock_price_pred_model.h5')
#============================================================================================================================================================
#fit에 사용되는 valid값 저장
#============================================================================================================================================================
def valid_increasing(model, X_train, Y_train, epochs, val_acc):
epochs = epochs
for i in range(epochs):
val_split = i / 100 + 0.001
history = model.fit(X_train,
Y_train,
print('Shape of x_test:', x_test.shape)
print("Preparing x_test...")
for c in x_test.dtypes[x_test.dtypes == object].index.values:
x_test[c] = (x_test[c] == True)
sc = StandardScaler()
x_train = sc.fit_transform(x_train)
len_x = int(x_train.shape[1])
print("len_x is:", len_x)
# Neural Network
print("\nSetting up neural network model...")
nn = Sequential()
nn.add(Dense(units=400, kernel_initializer='normal', input_dim=len_x))
nn.add(PReLU())
nn.add(Dropout(.4))
nn.add(Dense(units=160, kernel_initializer='normal'))
nn.add(PReLU())
nn.add(BatchNormalization())
nn.add(Dropout(.63))
nn.add(Dense(units=64, kernel_initializer='normal'))
nn.add(PReLU())
nn.add(BatchNormalization())
nn.add(Dropout(.45))
nn.add(Dense(units=28, kernel_initializer='normal'))
nn.add(PReLU())
nn.add(BatchNormalization())
nn.add(Dropout(.5))
nn.add(Dense(1, kernel_initializer='normal'))
nn.compile(loss='mae', optimizer=Adam(lr=4e-3, decay=1e-4))
def create_model(nb_classes, input_shape, config=None):
"""Create a VGG-16 like model."""
if len(input_shape) != 3:
raise Exception("Input shape should be a tuple (nb_channels, nb_rows, "
"nb_cols) or (nb_rows, nb_cols, nb_channels), "
"depending on your backend.")
if config is None:
config = {'model': {}}
min_feature_map_dimension = min(input_shape[:2])
if min_feature_map_dimension < 32:
print("ERROR: Please upsample the feature maps to have at least "
"a size of 32 x 32. Currently, it has {}".format(input_shape))
nb_filter = 32
if 'l2' not in config['model']:
config['model']['l2'] = 0.0001
# Network definition
# input_shape = (None, None, 3) # for fcn
input_ = Input(shape=input_shape)
x = input_
# Scale feature maps down to [63, 32] x [63, 32]
tmp = min_feature_map_dimension / 32.
if tmp >= 2:
while tmp >= 2.:
for _ in range(2):
x = Convolution2D(nb_filter, (3, 3),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(
config['model']['l2']))(x)
x = BatchNormalization()(x)
x = PReLU()(x)
x = MaxPooling2D(pool_size=(3, 3), padding='same', strides=2)(x)
nb_filter *= 2
tmp /= 2
# 32x32
x = Convolution2D(nb_filter + 37, (3, 3),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(config['model']['l2']))(x)
x = BatchNormalization()(x)
x = PReLU()(x)
x = Convolution2D(nb_filter + 37, (3, 3),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(config['model']['l2']))(x)
x = BatchNormalization()(x)
x = PReLU()(x)
# 16x16
x = MaxPooling2D(pool_size=(3, 3), padding='same', strides=2)(x)
x = Convolution2D(2 * nb_filter, (3, 3),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(config['model']['l2']))(x)
x = BatchNormalization()(x)
x = PReLU()(x)
x = Convolution2D(2 * nb_filter, (3, 3),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(config['model']['l2']))(x)
x = BatchNormalization()(x)
x = PReLU()(x)
# 8x8
x = MaxPooling2D(pool_size=(3, 3), padding='same', strides=2)(x)
x = Convolution2D(2 * nb_filter, (3, 3),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(config['model']['l2']))(x)
x = BatchNormalization()(x)
x = PReLU()(x)
# 4x4
x = MaxPooling2D(pool_size=(3, 3), padding='same', strides=2)(x)
x = Convolution2D(512, (4, 4),
padding='valid',
kernel_initializer='he_uniform',
kernel_regularizer=l2(config['model']['l2']),
use_bias=False)(x)
x = BatchNormalization()(x)
x = PReLU()(x)
x = Dropout(0.5)(x)
# 1x1
x = Convolution2D(512, (1, 1),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(config['model']['l2']),
use_bias=False)(x)
x = BatchNormalization()(x)
x = PReLU()(x)
x = Dropout(0.5)(x)
x = Convolution2D(nb_classes, (1, 1),
padding='same',
kernel_initializer='he_uniform',
kernel_regularizer=l2(config['model']['l2']),
use_bias=False)(x)
x = GlobalAveragePooling2D()(x) # Adjust for FCN
x = BatchNormalization()(x)
x = Activation('softmax')(x)
model = Model(inputs=input_, outputs=x)
return model
y, encoder = preprocess_labels(labels)
X_test, ids = load_data('test.csv', train=False)
X_test, _ = preprocess_data(X_test, scaler)
nb_classes = y.shape[1]
print(nb_classes, 'classes')
dims = X.shape[1]
print(dims, 'dims')
print("Building model...")
model = Sequential()
model.add(Dense(dims, 512, init='glorot_uniform'))
model.add(PReLU((512, )))
model.add(BatchNormalization((512, )))
model.add(Dropout(0.5))
model.add(Dense(512, 512, init='glorot_uniform'))
model.add(PReLU((512, )))
model.add(BatchNormalization((512, )))
model.add(Dropout(0.5))
model.add(Dense(512, 512, init='glorot_uniform'))
model.add(PReLU((512, )))
model.add(BatchNormalization((512, )))
model.add(Dropout(0.5))
model.add(Dense(512, nb_classes, init='glorot_uniform'))
model.add(Activation('softmax'))
def Conv1DLSTMbranchedV2(params):
Embedding_layer = Embedding(params['nb_words'],
params['embedding_dim'],
weights=[params['embedding_matrix']],
input_length=params['sequence_length'],
trainable=False)
input_ = Input(shape=(params['sequence_length'], ))
embed_input_ = Embedding_layer(input_)
conv1 = Conv1D(filters=128, kernel_size=1,
padding='same', activation='relu')
conv2 = Conv1D(filters=128, kernel_size=2,
padding='same', activation='relu')
conv3 = Conv1D(filters=128, kernel_size=3,
padding='same', activation='relu')
conv4 = Conv1D(filters=128, kernel_size=4,
padding='same', activation='relu')
conv5 = Conv1D(filters=32, kernel_size=5,
padding='same', activation='relu')
conv6 = Conv1D(filters=32, kernel_size=6,
padding='same', activation='relu')
conv1a = conv1(embed_input_)
glob1a = GlobalAveragePooling1D()(conv1a)
conv2a = conv2(embed_input_)
glob2a = GlobalAveragePooling1D()(conv2a)
conv3a = conv3(embed_input_)
glob3a = GlobalAveragePooling1D()(conv3a)
conv4a = conv4(embed_input_)
glob4a = GlobalAveragePooling1D()(conv4a)
conv5a = conv5(embed_input_)
glob5a = GlobalAveragePooling1D()(conv5a)
conv6a = conv6(embed_input_)
glob6a = GlobalAveragePooling1D()(conv6a)
if params['bidirectional']:
rnn_branch = Bidirectional(
GRU(params['lstm_units'], return_sequences=True))(embed_input_)
else:
rnn_branch = GRU(params['lstm_units'],
return_sequences=True)(embed_input_)
# rnn_branch = AttentionWithContext()(rnn_branch)
rnn_branch = GlobalAveragePooling1D()(rnn_branch)
mlp_input = Input(shape=(params['num_columns'],))
mlp = BatchNormalization()(mlp_input)
mlp = Dense(256)(mlp)
mlp = PReLU()(mlp)
mlp = BatchNormalization()(mlp)
mlp = Dense(256)(mlp)
mlp = PReLU()(mlp)
merge_layer = concatenate([glob1a, glob2a, glob3a, glob4a, glob5a, glob6a,
rnn_branch, mlp])
x = Dropout(0.2)(merge_layer)
x = BatchNormalization()(x)
x = Dense(256)(x)
x = PReLU()(x)
x = Dropout(0.2)(x)
x = BatchNormalization()(x)
x = Dense(6, activation='sigmoid')(x)
model = Model(inputs=[input_, mlp_input], outputs=x)
model.compile(loss='binary_crossentropy', optimizer=params['optimizer'],
metrics=['accuracy'])
return model
def test_prelu():
from keras.layers.advanced_activations import PReLU
np.random.seed(1337)
inp = get_standard_values()
for train in [True, False]:
# test with custom weights
alphas = np.random.random(inp.shape)
layer = PReLU(weights=alphas, input_shape=inp.flatten().shape)
# calling build here causes an error, unclear if this is a bug
# layer.build()
layer.input = K.variable(inp)
outp = K.eval(layer.get_output(train))
assert_allclose(inp, outp)
layer.input = K.variable(-inp)
outp = K.eval(layer.get_output(train))
assert_allclose(-alphas*inp, outp)
# test with default weights
layer = PReLU(input_shape=inp.flatten().shape)
# layer.build()
layer.input = K.variable(inp)
outp = K.eval(layer.get_output(train))
assert_allclose(inp, outp)
layer.input = K.variable(-inp)
outp = K.eval(layer.get_output(train))
assert_allclose(0., alphas*outp)
layer.get_config()
def model_generator(self,
units=512,
dropout=0.5,
reg=lambda: regularizers.l1_l2(l1=1e-7, l2=1e-7)):
decoder = Sequential(name="decoder")
h = 5
decoder.add(
Dense(units * 4 * 4,
input_dim=self.latent_dim,
kernel_regularizer=reg()))
# check channel order on below
decoder.add(Reshape((4, 4, units)))
# decoder.add(SpatialDropout2D(dropout))
#decoder.add(LeakyReLU(0.2))
decoder.add(PReLU())
decoder.add(
Conv2D(units // 2, (h, h),
padding='same',
kernel_regularizer=reg()))
# decoder.add(SpatialDropout2D(dropout))
#decoder.add(LeakyReLU(0.2))
decoder.add(PReLU())
decoder.add(UpSampling2D(size=(2, 2)))
decoder.add(
Conv2D(units // 2, (h, h),
padding='same',
kernel_regularizer=reg()))
# decoder.add(SpatialDropout2D(dropout))
#decoder.add(LeakyReLU(0.2))
decoder.add(PReLU())
decoder.add(UpSampling2D(size=(2, 2)))
decoder.add(
Conv2D(units // 4, (h, h),
padding='same',
kernel_regularizer=reg()))
# decoder.add(SpatialDropout2D(dropout))
#decoder.add(LeakyReLU(0.2))
decoder.add(PReLU())
decoder.add(UpSampling2D(size=(2, 2)))
decoder.add(Conv2D(3, (h, h), padding='same',
kernel_regularizer=reg()))
# add one more PReLU for fine scale detail?
# added another upsampling step to get to 64 x 64
#decoder.add(LeakyReLU(0.2))
decoder.add(PReLU())
decoder.add(UpSampling2D(size=(2, 2)))
decoder.add(Conv2D(3, (h, h), padding='same',
kernel_regularizer=reg()))
decoder.add(Activation('sigmoid'))
#decoder.summary()
# above assumes a particular output dimension, instead try below
#decoder.add(Dense(np.prod(self.img_shape), activation='sigmoid'))
#decoder.add(Reshape(self.img_shape))
decoder.summary()
z = Input(shape=(self.latent_dim, ))
img = decoder(z)
return Model(z, img)
