def _fcn(self, features: tf.Tensor,
mode: tf.estimator.ModeKeys) -> tf.Tensor:
"""
Sequence of FullyConnected Layers
:param features: input of the sub network
:param mode: standard names for Estimator model modes
:return: output of the sub network
"""
activation = 'relu'
kernel_initializer = initializers.TruncatedNormal(mean=0, stddev=0.1)
bias_initializer = 'zeros'
fc6 = Dense(units=496,
activation=activation,
use_bias=True,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer)(features)
drop7 = Dropout(rate=0.5)(fc6)
fc7 = Dense(units=496,
activation=activation,
use_bias=True,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer)(drop7)
drop8 = Dropout(rate=0.5)(fc7)
return drop8
def build_model(units,
inputs_dim,
output="regression",
sparse_dim=[],
with_ts=False,
ts_maxlen=0):
assert output == "regression" or output == "binary_clf", "This output type is not supported."
assert len(sparse_dim) == inputs_dim[1], "Dimension not match."
# Inputs for basic features.
inputs1 = Input(shape=(inputs_dim[0], ), name="basic_input")
x1 = Dense(units, kernel_regularizer='l2', activation="relu")(inputs1)
# Inputs for long one-hot features.
inputs2 = Input(shape=(inputs_dim[1], ), name="one_hot_input")
for i in range(len(sparse_dim)):
if i == 0:
x2 = Embedding(sparse_dim[i], units,
mask_zero=True)(slice(inputs2, i))
else:
tmp = Embedding(sparse_dim[i], units,
mask_zero=True)(slice(inputs2, i))
x2 = Concatenate()([x2, tmp])
x2 = tf.reshape(x2, [-1, units * inputs_dim[1]])
x = Concatenate()([x1, x2])
if with_ts:
inputs3 = Input(shape=(
None,
inputs_dim[2],
), name="ts_input")
x3 = LSTM(units,
input_shape=(ts_maxlen, inputs_dim[2]),
return_sequences=0)(inputs3)
x = Concatenate()([x, x3])
x = Dense(units, kernel_regularizer='l2', activation="relu")(x)
x = Dropout(0.5)(x)
x = Dense(units, kernel_regularizer='l2', activation="relu")(x)
x = Dropout(0.5)(x)
if output == "regression":
x = Dense(1, kernel_regularizer='l2')(x)
model = Model(inputs=[inputs1, inputs2], outputs=x)
if with_ts:
model = Model(inputs=[inputs1, inputs2, inputs3], outputs=x)
model.compile(optimizer='adam', loss='mean_squared_error')
elif output == "binary_clf":
x = Dense(1, kernel_regularizer='l2', activation="sigmoid")(x)
model = Model(inputs=[inputs1, inputs2], outputs=x)
if with_ts:
model = Model(inputs=[inputs1, inputs2, inputs3], outputs=x)
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['acc'])
#model.summary()
return model
def NN_huaweiv1(maxlen, embedding_matrix=None, class_num1=17, class_num2=12):
emb_layer = Embedding(
embedding_matrix.shape[0],
embedding_matrix.shape[1],
input_length=maxlen,
weights=[embedding_matrix],
trainable=False,
)
seq1 = Input(shape=(maxlen, ))
x1 = emb_layer(seq1)
sdrop = SpatialDropout1D(rate=0.2)
lstm_layer = Bidirectional(CuDNNGRU(128, return_sequences=True))
gru_layer = Bidirectional(CuDNNGRU(128, return_sequences=True))
cnn1d_layer = Conv1D(64,
kernel_size=3,
padding="same",
kernel_initializer="he_uniform")
x1 = sdrop(x1)
lstm1 = lstm_layer(x1)
gru1 = gru_layer(lstm1)
att_1 = Attention(maxlen)(lstm1)
att_2 = Attention(maxlen)(gru1)
cnn1 = cnn1d_layer(lstm1)
avg_pool = GlobalAveragePooling1D()
max_pool = GlobalMaxPooling1D()
x1 = concatenate([
att_1, att_2,
Attention(maxlen)(cnn1),
avg_pool(cnn1),
max_pool(cnn1)
])
x = Dropout(0.2)(Activation(activation="relu")(BatchNormalization()(
Dense(128)(x1))))
x = Activation(activation="relu")(BatchNormalization()(Dense(64)(x)))
pred1_d = Dense(class_num1)(x)
pred1 = Activation(activation='sigmoid', name='pred1')(pred1_d)
y = concatenate([x1, x])
y = Activation(activation="relu")(BatchNormalization()(Dense(64)(x)))
pred2_d = Dense(class_num2)(y)
pred2 = Activation(activation='sigmoid', name='pred2')(pred2_d)
z = Dropout(0.2)(Activation(activation="relu")(BatchNormalization()(
Dense(128)(x1))))
z = concatenate([pred1_d, pred2_d, z])
pred3 = Dense(class_num1 + class_num2, activation='sigmoid',
name='pred3')(z)
model = Model(inputs=seq1, outputs=[pred1, pred2, pred3])
return model
def build_lstm_model(input_data, output_size, neurons=20, activ_func='linear',
dropout=0.25, loss='mae', optimizer='adam'):
model = Sequential()
model.add(CuDNNLSTM(neurons, input_shape=(input_data.shape[1], input_data.shape[2]), return_sequences=True))
model.add(Dropout(dropout))
model.add(CuDNNLSTM(neurons, input_shape=(input_data.shape[1], input_data.shape[2])))
model.add(Dropout(dropout))
model.add(Dense(units=output_size))
model.add(Activation(activ_func))
model.compile(loss=loss, optimizer=optimizer)
return model
def DNNclassifier_crps(self, p, num_cut, optimizer, seeding):
tf.set_random_seed(seeding)
inputs = Input(shape=(p,))
if isinstance(optimizer, str):
opt = optimizer
else:
opt_name = optimizer.__class__.__name__
opt_config = optimizer.get_config()
opt_class = getattr(optimizers, opt_name)
opt = opt_class(**opt_config)
for i, n_neuron in enumerate(self.hidden_list):
if i == 0:
net = Dense(n_neuron, kernel_initializer = 'he_uniform')(inputs)
else:
net = Dense(n_neuron, kernel_initializer = 'he_uniform')(net)
net = Activation(activation = 'elu')(net)
net = BatchNormalization()(net)
net = Dropout(rate=self.dropout_list[i])(net)
softmaxlayer = Dense(num_cut + 1, activation='softmax',
kernel_initializer = 'he_uniform')(net)
output = Lambda(self.tf_cumsum)(softmaxlayer)
model = Model(inputs = [inputs], outputs=[output])
model.compile(optimizer=opt, loss=self.crps_loss)
return model
def get_seq_model():
"""Define three channel input shape depending on image data format."""
if K.image_data_format() == 'channels_first':
input_shape = (3, img_width, img_height)
else:
input_shape = (img_width, img_height, 3)
# Initialize CNN by creating a sequential model.
model = Sequential()
model.add(Conv2D(32, (3, 3), input_shape=input_shape))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(32, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(64))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(2))
model.add(Activation('sigmoid'))
model.compile(
loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy'])
return model
def inner(x):
x = LayerNormalization()(x)
x = Activation("relu")(x)
x = Convolution2D(channels, 3, strides=strides, **params)(x)
x = Dropout(drop_rate)(x) if drop_rate > 0 else x
x = LayerNormalization()(x)
x = Activation("relu")(x)
x = Convolution2D(channels, 3, **params)(x)
return x
def build_cnn(input_shape, output_size):
kwargs = {"kernel_size": 3, "activation": "relu", "padding": "same"}
model = Sequential([
# conv1_*
Convolution2D(64, input_shape=input_shape, **kwargs),
BatchRenormalization(),
Convolution2D(64, **kwargs),
BatchRenormalization(),
MaxPooling2D(pool_size=(2, 2)),
Dropout(0.25),
# conv2_*
Convolution2D(128, **kwargs),
BatchRenormalization(),
Convolution2D(128, **kwargs),
BatchRenormalization(),
MaxPooling2D(pool_size=(2, 2)),
Dropout(0.25),
# conv3_*
Convolution2D(256, **kwargs),
BatchRenormalization(),
Convolution2D(256, **kwargs),
BatchRenormalization(),
MaxPooling2D(pool_size=(2, 2)),
Dropout(0.25),
# Fully connected
Flatten(),
Dense(1024),
Activation("relu"),
Dropout(0.5),
Dense(512),
Activation("relu"),
Dropout(0.5),
Dense(output_size),
Activation("softmax")
])
model.compile(loss="categorical_crossentropy",
optimizer="adam",
metrics=["accuracy"])
return model
def build_all_conv_nn(input_shape, output_size):
"""Build a small variation of the best performing network from
'Springenberg, Jost Tobias, et al. "Striving for simplicity: The all
convolutional net." arXiv preprint arXiv:1412.6806 (2014)' which should
achieve approximately 91% in CIFAR-10.
"""
kwargs = {"activation": "relu", "border_mode": "same"}
model = Sequential([
# conv1
Convolution2D(96, 3, 3, input_shape=input_shape, **kwargs),
BatchRenormalization(),
Convolution2D(96, 3, 3, **kwargs),
BatchRenormalization(),
Convolution2D(96, 3, 3, subsample=(2, 2), **kwargs),
BatchRenormalization(),
Dropout(0.25),
# conv2
Convolution2D(192, 3, 3, **kwargs),
BatchRenormalization(),
Convolution2D(192, 3, 3, **kwargs),
BatchRenormalization(),
Convolution2D(192, 3, 3, subsample=(2, 2), **kwargs),
BatchRenormalization(),
Dropout(0.25),
# conv3
Convolution2D(192, 1, 1, **kwargs),
BatchRenormalization(),
Dropout(0.25),
Convolution2D(output_size, 1, 1, **kwargs),
GlobalAveragePooling2D(),
Activation("softmax")
])
model.compile(loss="categorical_crossentropy",
optimizer=SGD(momentum=0.9),
metrics=["accuracy"])
return model
def build_lstm_lm3(input_shape, output_size):
# LM datasets will report the vocab_size as output_size
vocab_size = output_size
model = Sequential([
Embedding(vocab_size + 1,
128,
mask_zero=True,
input_length=input_shape[0]),
LSTM(650, unroll=True, return_sequences=True),
Dropout(0.5),
LSTM(650, unroll=True),
Dropout(0.5),
Dense(output_size),
Activation("softmax")
])
model.compile(optimizer="adam",
loss="sparse_categorical_crossentropy",
metrics=["accuracy"])
return model
def ablate_activations(model, layer_regex, pct_ablation, seed):
# Auxiliary dictionary to describe the network graph
network_dict = {'input_layers_of': {}, 'new_output_tensor_of': {}}
# Set the input layers of each layer
for layer in model.layers:
for node in layer._outbound_nodes:
layer_name = node.outbound_layer.name
if layer_name not in network_dict['input_layers_of']:
network_dict['input_layers_of'].update(
{layer_name: [layer.name]})
else:
network_dict['input_layers_of'][layer_name].append(layer.name)
# Set the output tensor of the input layer
network_dict['new_output_tensor_of'].update(
{model.layers[0].name: model.input})
# Iterate over all layers after the input
for layer in model.layers[1:]:
# Determine input tensors
layer_input = [
network_dict['new_output_tensor_of'][layer_aux]
for layer_aux in network_dict['input_layers_of'][layer.name]
]
if len(layer_input) == 1:
layer_input = layer_input[0]
x = layer(layer_input)
# Insert layer if name matches the regular expression
if re.match(layer_regex, layer.name):
ablation_layer = Dropout(
rate=pct_ablation,
noise_shape=(None, ) +
tuple(np.repeat(1,
len(layer.output_shape) - 2)) +
(layer.output_shape[-1], ),
seed=seed,
name='{}_dropout'.format(layer.name))
x = ablation_layer(x, training=True)
if seed:
seed += 1
# Set new output tensor (the original one, or the one of the inserted
# layer)
network_dict['new_output_tensor_of'].update({layer.name: x})
return Model(inputs=model.inputs, outputs=x)
def build_model(hidden_size):
inputs = Input(shape=(28, 28))
x1 = Flatten()(inputs)
x2 = Dense(hidden_size, activation=tf.nn.relu)(x1)
x3 = Dropout(0.2)(x2)
x4 = Dense(10, activation=tf.nn.softmax)(x3)
model = Model(inputs=inputs, outputs=x4)
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
# Train and fit model
model.fit(x_train, y_train, epochs=5)
[loss, acc] = model.evaluate(x_test, y_test)
return [model, acc]
def neural_network(input_shape):
inputs = keras.Input(shape=input_shape)
#Layer 1
x = MaxPooling2D(pool_size=(2, 2), name='MaxPooling2D_1')(inputs)
x = Conv2D(32, kernel_size=(5, 5), padding='same')(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = MaxPooling2D(pool_size=(4, 4))(x)
#Layer 2
x = Conv2D(64, kernel_size=(5, 5), padding='same', name='Conv2D_2')(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = MaxPooling2D(pool_size=(2, 2), name='MaxPooling2D_3')(x)
x = Flatten(name='Flatten')(x)
#Layer 3
#model.add(Dense(256,name = 'Dense_1'))
#model.add(BatchNormalization(name = 'BatchNormalization_2'))
#model.add(LeakyReLU(alpha=0.1))
#model.add(Dropout(0.5,name = 'Dropout_1'))
#Layer 4
x = Dense(128, name='Dense_2')(x)
x = BatchNormalization(name='BatchNormalization_3')(x)
x = LeakyReLU(alpha=0.1)(x)
x = Dropout(0.5, name='Dropout_2')(x)
#Layer 5
x = Dense(128, name='Dense_3')(x)
x = BatchNormalization(name='BatchNormalization_4')(x)
x = LeakyReLU(alpha=0.1)(x)
#model.add(Dropout(0.5,name = 'Dropout_3'))
outputs = Dense(1, activation='sigmoid', name='Dense_4')(x)
model = Model(inputs, outputs)
return model
def define_discriminator(in_shape=(32, 32, 3)):
model = Sequential()
# normal
model.add(Conv2D(64, (3, 3), padding='same', input_shape=in_shape))
model.add(LeakyReLU(alpha=0.2))
# downsample
model.add(Conv2D(128, (3, 3), strides=(2, 2), padding='same'))
model.add(LeakyReLU(alpha=0.2))
# downsample
model.add(Conv2D(128, (3, 3), strides=(2, 2), padding='same'))
model.add(LeakyReLU(alpha=0.2))
# downsample
model.add(Conv2D(256, (3, 3), strides=(2, 2), padding='same'))
model.add(LeakyReLU(alpha=0.2))
# classifier
model.add(Flatten())
model.add(Dropout(0.4))
model.add(Dense(1, activation='sigmoid'))
# compile model
opt = Adam(lr=0.0002, beta_1=0.5)
model.compile(loss='binary_crossentropy',
optimizer=opt,
metrics=['accuracy'])
return model
def model_fn_ALEXNET(features,
activation='relu',
kernel_initializer=tf.keras.initializers.TruncatedNormal(
mean=0, stddev=0.1),
bias_initializer='zeros'):
# input: [None, 227, 227, 3]
# conv1: f 96, k (11,11), s (4,4), VALID, relu --> [None, 54, 54, 96]
with tf.control_dependencies(
tf.debugging.assert_equal(features.get_shape()[1:],
[227, 227, 3])):
conv1 = Conv2D(filters=96,
kernel_size=(11, 11),
strides=(4, 4),
padding='valid',
activation=activation,
use_bias=True,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer)(features)
# pool1: k (3,3), s (2,2), VALID --> [None, 26, 26, 96]
with tf.control_dependencies(
tf.debugging.assert_equal(conv1.get_shape()[1:], [54, 54, 96])):
pool1 = MaxPool2D(pool_size=(3, 3), strides=(2, 2),
padding='valid')(conv1)
# conv2: f 256, k (5,5), s (1,1), SAME, relu --> [None, 26, 26, 256]
with tf.control_dependencies(
tf.debugging.assert_equal(features.get_shape()[1:], [26, 26, 96])):
conv2 = Conv2D(filters=256,
kernel_size=(5, 5),
strides=(1, 1),
padding='same',
activation=activation,
use_bias=True,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer)(pool1)
# pool2: k (3,3), s (2,2), VALID --> [None, 12, 12, 256]
with tf.control_dependencies(
tf.debugging.assert_equal(conv1.get_shape()[1:], [26, 26, 256])):
pool2 = MaxPool2D(pool_size=(3, 3), strides=(2, 2),
padding='valid')(conv2)
# conv3: f 384, k (3,3), s(1,1), SAME, relu --> [None, 12, 12, 384]
with tf.control_dependencies(
tf.debugging.assert_equal(features.get_shape()[1:],
[12, 12, 256])):
conv3 = Conv2D(filters=384,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
activation=activation,
use_bias=True,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer)(pool2)
# conv4: f 384, k (3,3), s(1,1), SAME, relu --> [None, 12, 12, 384]
with tf.control_dependencies(
tf.debugging.assert_equal(features.get_shape()[1:],
[12, 12, 384])):
conv4 = Conv2D(filters=384,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
activation=activation,
use_bias=True,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer)(conv3)
# conv5: f 256, k (3,3), s(1,1), SAME, relu --> [None, 12, 12, 256]
with tf.control_dependencies(
tf.debugging.assert_equal(features.get_shape()[1:],
[12, 12, 384])):
conv5 = Conv2D(filters=256,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
activation=activation,
use_bias=True,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer)(conv4)
# pool5: k (3,3), s (2,2) --> [None, 5, 5, 256]
with tf.control_dependencies(
tf.debugging.assert_equal(conv1.get_shape()[1:], [12, 12, 256])):
pool5 = MaxPool2D(pool_size=(3, 3), strides=(2, 2),
padding='valid')(conv5)
# flatten --> [None, 6400]
flatten = Flatten()(pool5)
# fc6: f 4096, relu --> [None, 4096]
with tf.control_dependencies(
tf.debugging.assert_equal(flatten.get_shape()[1:], [6400])):
fc6 = Dense(units=496,
activation=activation,
use_bias=True,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer)(flatten)
# drop7: p 0.5 --> [None, 4096]
drop7 = Dropout(rate=0.5)(fc6)
# fc7: f 4096, relu --> [None, 4096]
with tf.control_dependencies(
tf.debugging.assert_equal(fc6.get_shape()[1:], [6400])):
fc7 = Dense(units=496,
activation=activation,
use_bias=True,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer)(drop7)
# drop8: p 0.5 --> [None, 4096]
drop8 = Dropout(rate=0.5)(fc7)
return drop8
def stack_layers(inputs, layers, kernel_initializer='glorot_uniform'):
'''
Builds the architecture of the network by applying each layer specified in layers to inputs.
inputs: a dict containing input_types and input_placeholders for each key and value pair, respecively.
for spectralnet, this means the input_types 'Unlabeled' and 'Orthonorm'*
layers: a list of dicts containing all layers to be used in the network, where each dict describes
one such layer. each dict requires the key 'type'. all other keys are dependent on the layer
type
kernel_initializer: initialization configuration passed to keras (see keras initializers)
returns: outputs, a dict formatted in much the same way as inputs. it contains input_types and
output_tensors for each key and value pair, respectively, where output_tensors are
the outputs of the input_placeholders in inputs after each layer in layers is applied
* this is necessary since spectralnet takes multiple inputs and performs special computations on the
orthonorm layer
'''
outputs = dict()
for key in inputs:
outputs[key] = inputs[key]
for layer in layers:
# check for l2_reg argument
l2_reg = layer.get('l2_reg')
if l2_reg:
l2_reg = l2(layer['l2_reg'])
# create the layer
if layer['type'] == 'softplus_reg':
l = Dense(layer['size'],
activation='softplus',
kernel_initializer=kernel_initializer,
kernel_regularizer=l2(0.001),
name=layer.get('name'))
elif layer['type'] == 'softplus':
l = Dense(layer['size'],
activation='softplus',
kernel_initializer=kernel_initializer,
kernel_regularizer=l2_reg,
name=layer.get('name'))
elif layer['type'] == 'softmax':
l = Dense(layer['size'],
activation='softmax',
kernel_initializer=kernel_initializer,
kernel_regularizer=l2_reg,
name=layer.get('name'))
elif layer['type'] == 'tanh':
l = Dense(layer['size'],
activation='tanh',
kernel_initializer=kernel_initializer,
kernel_regularizer=l2_reg,
name=layer.get('name'))
elif layer['type'] == 'relu':
l = Dense(layer['size'],
activation='relu',
kernel_initializer=kernel_initializer,
kernel_regularizer=l2_reg,
name=layer.get('name'))
elif layer['type'] == 'selu':
l = Dense(layer['size'],
activation='selu',
kernel_initializer=kernel_initializer,
kernel_regularizer=l2_reg,
name=layer.get('name'))
elif layer['type'] == 'Conv2D':
l = Conv2D(layer['channels'],
kernel_size=layer['kernel'],
activation='relu',
data_format='channels_last',
kernel_regularizer=l2_reg,
name=layer.get('name'))
elif layer['type'] == 'BatchNormalization':
l = BatchNormalization(name=layer.get('name'))
elif layer['type'] == 'MaxPooling2D':
l = MaxPooling2D(pool_size=layer['pool_size'],
data_format='channels_first',
name=layer.get('name'))
elif layer['type'] == 'Dropout':
l = Dropout(layer['rate'], name=layer.get('name'))
elif layer['type'] == 'Flatten':
l = Flatten(name=layer.get('name'))
elif layer['type'] == 'Orthonorm':
l = Orthonorm(outputs['Orthonorm'], name=layer.get('name'))
else:
raise ValueError("Invalid layer type '{}'".format(layer['type']))
# apply the layer to each input in inputs
for k in outputs:
outputs[k] = l(outputs[k])
return outputs
kernel_initializer=keras.initializers.TruncatedNormal(stddev=init_stddev),
)(headModel)
headModel = BatchNormalization(momentum=MOM)(headModel)
headModel = LeakyReLU(alpha=0.2)(headModel)
headModel = AveragePooling2D(pool_size=(2, 2))(headModel)
headModel = Flatten(name="flatten")(headModel)
headModel = Dense(64, activation="relu")(headModel)
headModel = BatchNormalization(momentum=MOM)(headModel)
headModel = LeakyReLU(alpha=0.2)(headModel)
headModel = Dense(32, activation="relu")(headModel)
headModel = BatchNormalization(momentum=MOM)(headModel)
headModel = LeakyReLU(alpha=0.2)(headModel)
headModel = Dense(32, activation="relu")(headModel)
headModel = BatchNormalization(momentum=MOM)(headModel)
headModel = LeakyReLU(alpha=0.2)(headModel)
headModel = Dropout(DROP)(headModel)
headModel = Dense(2, activation="softmax")(headModel)
# place the head FC model on top of the base model (this will become the actual model we will train)
model = Model(inputs=baseModel.input, outputs=headModel)
model.summary()
# loop over all layers in the base model and freeze them so they will *not* be updated during the first training process
for layer in baseModel.layers:
layer.trainable = False
# compile our model
print("[INFO] compiling model...")
opt = Adam(lr=INIT_LR, decay=DEC)
model.compile(loss="binary_crossentropy", optimizer=opt, metrics=["accuracy"])
def keras_build_fn(num_feature,
num_output,
is_sparse,
embedding_dim=-1,
num_hidden_layer=2,
hidden_layer_dim=512,
activation='elu',
learning_rate=1e-3,
dropout=0.5,
l1=0.0,
l2=0.0,
loss='categorical_crossentropy'):
"""Initializes and compiles a Keras DNN model using the Adam optimizer.
Args:
num_feature: number of features
num_output: number of outputs (targets, e.g., classes))
is_sparse: boolean whether input data is in sparse format
embedding_dim: int number of nodes in embedding layer; if value is <= 0 then
no embedding layer will be present in the model
num_hidden_layer: number of hidden layers
hidden_layer_dim: int number of nodes in the hidden layer(s)
activation: string
activation function for hidden layers; see https://keras.io/activations/
learning_rate: float learning rate for Adam
dropout: float proportion of nodes to dropout; values in [0, 1]
l1: float strength of L1 regularization on weights
l2: float strength of L2 regularization on weights
loss: string
loss function; see https://keras.io/losses/
Returns:
model: Keras.models.Model
compiled Keras model
"""
assert num_hidden_layer >= 1
inputs = Input(shape=(num_feature, ), sparse=is_sparse)
activation_func_args = ()
if activation.lower() == 'prelu':
activation_func = PReLU
elif activation.lower() == 'leakyrelu':
activation_func = LeakyReLU
elif activation.lower() == 'elu':
activation_func = ELU
elif activation.lower() == 'thresholdedrelu':
activation_func = ThresholdedReLU
else:
activation_func = Activation
activation_func_args = (activation)
if l1 > 0 and l2 > 0:
reg_init = lambda: regularizers.l1_l2(l1, l2)
elif l1 > 0:
reg_init = lambda: regularizers.l1(l1)
elif l2 > 0:
reg_init = lambda: regularizers.l2(l2)
else:
reg_init = lambda: None
if embedding_dim > 0:
# embedding layer
e = Dense(embedding_dim)(inputs)
x = Dense(hidden_layer_dim, kernel_regularizer=reg_init())(e)
x = activation_func(*activation_func_args)(x)
x = Dropout(dropout)(x)
else:
x = Dense(hidden_layer_dim, kernel_regularizer=reg_init())(inputs)
x = activation_func(*activation_func_args)(x)
x = Dropout(dropout)(x)
# add additional hidden layers
for _ in range(num_hidden_layer - 1):
x = Dense(hidden_layer_dim, kernel_regularizer=reg_init())(x)
x = activation_func(*activation_func_args)(x)
x = Dropout(dropout)(x)
x = Dense(num_output)(x)
preds = Activation('softmax')(x)
model = Model(inputs=inputs, outputs=preds)
model.compile(optimizer=Adam(lr=learning_rate), loss=loss)
return model
random_state=777,
shuffle=True)
K.clear_session()
inputs = Input(shape=(8000, 1))
x = BatchNormalization(axis=-1,
momentum=0.99,
epsilon=1e-3,
center=True,
scale=True)(inputs)
# First Conv1D layer
x = Conv1D(8, 13, padding='valid', activation='relu', strides=1)(x)
x = MaxPooling1D(3)(x)
x = Dropout(0.3)(x)
# Second Conv1D layer
x = Conv1D(16, 11, padding='valid', activation='relu', strides=1)(x)
x = MaxPooling1D(3)(x)
x = Dropout(0.3)(x)
# Third Conv1D layer
x = Conv1D(32, 9, padding='valid', activation='relu', strides=1)(x)
x = MaxPooling1D(3)(x)
x = Dropout(0.3)(x)
x = BatchNormalization(axis=-1,
momentum=0.99,
epsilon=1e-3,
center=True,