def mk_model_with_bn():
model = Sequential()
model.add(
Convolution2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
input_shape=(IMAGE_SIZE, IMAGE_SIZE, 1),
kernel_initializer=kernel_initializer()))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(
Convolution2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
kernel_initializer=kernel_initializer()))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.5))
model.add(
Convolution2D(filters=128,
kernel_size=(3, 3),
kernel_initializer=kernel_initializer()))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(
Convolution2D(filters=128,
kernel_size=(3, 3),
kernel_initializer=kernel_initializer()))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.5))
model.add(
Convolution2D(filters=256,
kernel_size=(3, 3),
kernel_initializer=kernel_initializer()))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(
Convolution2D(filters=256,
kernel_size=(3, 3),
kernel_initializer=kernel_initializer()))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(256, kernel_initializer='he_normal'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(LABEL_NUM, kernel_initializer='he_normal'))
model.add(Activation('softmax'))
return model
def fit(self, eventlog_name):
import tensorflow as tf
from tensorflow.contrib.keras.python.keras.engine import Input, Model
from tensorflow.contrib.keras.python.keras.layers import Dense, GaussianNoise, Dropout
# load data
features = self.dataset.load(eventlog_name)
# parameters
input_size = features.shape[1]
hidden_size = np.round(input_size * 4)
# input layer
input_layer = Input(shape=(input_size,), name='input')
# hidden layer
hid = Dense(hidden_size, activation=tf.nn.relu)(GaussianNoise(0.1)(input_layer))
hid = Dense(hidden_size, activation=tf.nn.relu)(Dropout(0.5)(hid))
hid = Dense(hidden_size, activation=tf.nn.relu)(Dropout(0.5)(hid))
hid = Dense(hidden_size, activation=tf.nn.relu)(Dropout(0.5)(hid))
hid = Dense(hidden_size, activation=tf.nn.relu)(Dropout(0.5)(hid))
# output layer
output_layer = Dense(input_size, activation='linear')(Dropout(0.5)(hid))
# build model
self.model = Model(inputs=input_layer, outputs=output_layer)
# compile model
self.model.compile(
optimizer=tf.train.AdamOptimizer(learning_rate=0.0001),
loss=tf.losses.mean_squared_error
)
# train model
self.model.fit(
features,
features,
batch_size=100,
epochs=100,
validation_split=0.2,
)
def bidirectional_model():
inputs = Input(shape=(maxlen, ), dtype='int32')
x = Embedding(max_features, 128, input_length=maxlen)(inputs)
x = Bidirectional(LSTM(64))(x)
x = Dropout(0.5)(x)
x = Dense(1, activation='sigmoid')(x)
model = Model(inputs=inputs, outputs=x)
# try using different optimizers and different optimizer configs
model.compile('adam', 'binary_crossentropy', metrics=['accuracy'])
return model
def build_model(num_output_classes):
conv1size = 32
conv2size = 64
convfiltsize = 4
densesize = 128
poolsize = (2, 2)
imgdepth = 3
dropout = 0.3
if IMG_COLORMODE == 'grayscale':
imgdepth = 1
inpshape = IMG_TGT_SIZE + (imgdepth, )
inputs = Input(shape=inpshape)
conv1 = Convolution2D(conv1size,
convfiltsize,
strides=(1, 1),
padding='valid',
activation='relu',
name='conv1',
data_format='channels_last')(inputs)
pool1 = MaxPooling2D(pool_size=poolsize, name='pool1')(conv1)
drop1 = Dropout(dropout)(pool1)
conv2 = Convolution2D(conv2size,
convfiltsize,
strides=(1, 1),
padding='valid',
activation='relu',
name='conv2',
data_format='channels_last')(drop1)
pool2 = MaxPooling2D(pool_size=poolsize, name='pool2')(conv2)
drop2 = Dropout(dropout)(pool2)
flat2 = Flatten()(drop2)
dense = Dense(densesize, name='dense')(flat2)
denseact = Activation('relu')(dense)
output = Dense(num_output_classes, name='output')(denseact)
outputact = Activation('softmax')(output)
model = Model(inputs=inputs, outputs=outputact)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def cnn_model_fn():
'''
define the model in function way
'''
# input shape is (img_rows, img_cols, fea_channel)
inputs = Input(shape=(img_rows, img_cols, 1))
x = Conv2D(32, kernel_size=(3, 3), activation='relu')(inputs)
x = Conv2D(64, (3, 3), activation='relu')(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
x = Dropout(0.25)(x)
x = Flatten()(x)
x = Dense(128, activation='relu')(x)
x = Dropout(0.5)(x)
pred = Dense(num_classes, activation='softmax')(x)
# small change of Model parameters names, now is inputs, outputs
model = Model(inputs=inputs, outputs=pred)
model.compile(loss='categorical_crossentropy',
optimizer='adadelta',
metrics=['accuracy'])
return model
def mk_model():
model = Sequential()
model.add(
Convolution2D(filters=32,
kernel_size=(3, 3),
strides=(1, 1),
input_shape=(IMAGE_SIZE, IMAGE_SIZE, 1),
kernel_initializer=kernel_initializer()))
model.add(LeakyReLU(alpha=.1))
model.add(
Convolution2D(filters=32,
kernel_size=(3, 3),
strides=(1, 1),
kernel_initializer=kernel_initializer()))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.5))
model.add(
Convolution2D(filters=64,
kernel_size=(3, 3),
kernel_initializer=kernel_initializer()))
model.add(LeakyReLU(alpha=0.1))
model.add(
Convolution2D(filters=64,
kernel_size=(3, 3),
kernel_initializer=kernel_initializer()))
model.add(LeakyReLU(alpha=0.1))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(MaxoutDense(output_dim=256, nb_feature=4))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(LABEL_NUM, kernel_initializer='he_uniform'))
model.add(Activation('softmax'))
return model
def create_two_stream_classifier(
num_fc_neurons,
dropout_rate,
num_classes=24): # classifier_weights_path=None
classifier = Sequential()
classifier.add(
Dense(num_fc_neurons, name='fc7', input_shape=(num_fc_neurons * 2, )))
#classifier.add(BatchNormalization(axis=1, name='fc7_bn'))
classifier.add(Activation('relu', name='fc7_ac'))
classifier.add(Dropout(dropout_rate))
classifier.add(Dense(num_classes, activation='softmax',
name='predictions'))
return classifier
def cnn_model_fn():
''' '''
print('Build model...')
inputs = Input(shape=(maxlen, ),
dtype='int32') # a index sequence with lenght = maxlen
x = Embedding(max_features, embedding_dims, input_length=maxlen)(inputs)
x = Dropout(0.2)(x)
x = Conv1D(filters,
kernel_size,
padding='valid',
activation='relu',
strides=1)(x)
x = GlobalMaxPooling1D()(x)
x = Dense(hidden_dims)(x)
x = Dropout(0.2)(x)
x = Activation('relu')(x)
x = Dense(1)(x)
x = Activation('sigmoid')(x)
model = Model(inputs=inputs, outputs=x)
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def build(self,alpha, img_input, temp_softmax):
shape = (1, 1, int(1024 * alpha))
"""
This looks dangerous. Not sure how the model would get affected with the laarning_phase variable set to True.
"""
K.set_learning_phase(True)
with tf.name_scope('teacher') as scope:
self.conv1 = Conv2D(
int(32*alpha),
(3,3),
padding='same',
use_bias=False,
strides=(1,1),
name='teacher_conv1', trainable=self.trainable)(img_input)
self.conv2 = BatchNormalization(axis=-1, name='teacher_conv1_bn', trainable=self.trainable)(self.conv1)
self.conv3 = Activation(self.relu6, name='teacher_conv1_relu', trainable=self.trainable)(self.conv2)
self.conv4 = self._depthwise_conv_block(self.conv3, 64, alpha, depth_multiplier, block_id = 15)
self.conv5 = self._depthwise_conv_block(self.conv4, 128, alpha, depth_multiplier,strides=(2, 2), block_id =16)
self.conv6 =self. _depthwise_conv_block(self.conv5, 128, alpha, depth_multiplier,block_id =17)
self.conv7 = self._depthwise_conv_block(self.conv6, 256, alpha, depth_multiplier, strides=(2,2),block_id =18)
self.conv8 = self._depthwise_conv_block(self.conv7, 256, alpha, depth_multiplier, block_id =19)
self.conv9 = self._depthwise_conv_block(self.conv8, 512, alpha, depth_multiplier, strides = (2,2), block_id =20)
self.conv10 = self._depthwise_conv_block(self.conv9, 512, alpha, depth_multiplier, block_id =21)
self.conv11 = self._depthwise_conv_block(self.conv10, 512, alpha, depth_multiplier, block_id =22)
self.conv12 = self._depthwise_conv_block(self.conv11, 512, alpha, depth_multiplier, block_id =23)
self.conv13 = self._depthwise_conv_block(self.conv12, 512, alpha, depth_multiplier, block_id =24)
self.conv14 = self._depthwise_conv_block(self.conv13, 512, alpha, depth_multiplier, block_id =25)
self.conv15 = self._depthwise_conv_block(self.conv14, 1024, alpha, depth_multiplier,strides=(2,2), block_id =26)
self.conv16 = self._depthwise_conv_block(self.conv15, 1024, alpha, depth_multiplier, block_id =27)
self.conv17 = GlobalAveragePooling2D()(self.conv16)
self.conv18 = Reshape(shape, name='teacher_reshape_1', trainable=self.trainable)(self.conv17)
self.conv19 = Dropout(0.5, name='teacher_dropout', trainable=self.trainable)(self.conv18)
self.conv20 = Conv2D(self.num_classes, (1, 1), padding='same', name='teacher_conv_preds', trainable=self.trainable)(self.conv18)
self.conv21 = Activation('softmax', name='teacher_act_softmax', trainable=self.trainable)(tf.divide(self.conv20, temp_softmax))
self.conv22 = Reshape((self.num_classes,), name='teacher_reshape_2', trainable=self.trainable)(self.conv21)
return self
def cnn_lstm_model():
''' '''
print('Build model...')
inputs = Input(shape=(maxlen, ), dtype='int32')
x = Embedding(max_features, embedding_size, input_length=maxlen)(inputs)
x = Dropout(0.25)(x)
x = Conv1D(filters,
kernel_size,
padding='valid',
activation='relu',
strides=1)(x)
x = MaxPooling1D(pool_size=pool_size)(x)
x = LSTM(lstm_output_size)(x)
x = Dense(1, activation='sigmoid')(x)
model = Model(inputs=inputs, outputs=x)
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def mk_model(arch, data_type, classes):
if data_type == 'mnist':
channels = 1
elif data_type == 'aerial':
channels = 3
if arch == 'lenet-5':
model = Sequential() #モデルの初期化
#畳み込み第１層
model.add(Conv2D(
32, 5, padding='same',
input_shape=(28, 28, channels))) #output_shape=(None,28,28,32)
#filters=32, kernel_size=(5,5), strides(1,1), use_bias=True
#dilidation_rate(膨張率)=(1,1), kernel_initializer='glorot_uniform', bias_initializer='zeros'
#padding='sane'は出力のshapeが入力と同じになるように調整
#output_shape=(None(60000),28,28,32)
model.add(Activation('relu'))
model.add(MaxPooling2D(padding='same')) #output_shape=(None,14,14,32)
#pool_size=(2,2), strides(2,2)
#畳み込み第２層
model.add(Conv2D(64, 5, padding='same')) #output_shape=(None,14,14,64)
model.add(Activation('relu'))
model.add(MaxPooling2D(padding='same')) #output_shape=(None,7,7,64)
#平坦化
model.add(Flatten()) #output_shape=(None,3136(7*7*64))
#全結合第１層
model.add(Dense(1024)) #output_shape=(None,1024)
model.add(Activation('relu'))
model.add(Dropout(0.5)) #無視する割合を記述(例えば、0.2と記述した場合、80%の結合が残る)
#全結合第２層
model.add(Dense(classes)) #output_shape=(None,classes)
model.add(Activation('softmax'))
return model
elif arch == 'alexnet':
model = Sequential() #モデルの初期化
def simple_model():
model = Sequential()
model.add(
Convolution2D(filters=32,
kernel_size=(3, 3),
strides=(1, 1),
input_shape=(IMAGE_SIZE, IMAGE_SIZE, 1)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(filters=64, kernel_size=(3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(256))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(LABEL_NUM))
model.add(Activation('softmax'))
return model
def __init__(self, input_shape, lr=0.01, n_layers=2, n_hidden=8, rate_dropout=0.2, loss=risk_estimation):
print("initializing..., learing rate %s, n_layers %s, n_hidden %s, dropout rate %s." % (
lr, n_layers, n_hidden, rate_dropout))
self.model = Sequential()
self.model.add(Dropout(rate=rate_dropout, input_shape=(input_shape[0], input_shape[1])))
for i in range(0, n_layers - 1):
self.model.add(LSTM(n_hidden * 4, return_sequences=True, activation='tanh',
recurrent_activation='hard_sigmoid', kernel_initializer='glorot_uniform',
recurrent_initializer='orthogonal', bias_initializer='zeros',
dropout=rate_dropout, recurrent_dropout=rate_dropout))
self.model.add(LSTM(n_hidden, return_sequences=False, activation='tanh',
recurrent_activation='hard_sigmoid', kernel_initializer='glorot_uniform',
recurrent_initializer='orthogonal', bias_initializer='zeros',
dropout=rate_dropout, recurrent_dropout=rate_dropout))
self.model.add(Dense(1, kernel_initializer=initializers.glorot_uniform()))
# self.model.add(BatchNormalization(axis=-1, moving_mean_initializer=Constant(value=0.5),
# moving_variance_initializer=Constant(value=0.25)))
self.model.add(BatchNormalization(axis=-1))
self.model.add(Activation("relu(alpha=0., max_value=1.0)"))
opt = RMSprop(lr=lr)
self.model.compile(loss=loss,
optimizer=opt,
metrics=['accuracy'])
def create_VGG16(num_fc_neurons,
dropout_rate,
num_classes=24,
top_model_weights_path=None,
img_height=224,
img_width=224,
include_loc='all',
activation='softmax'):
# load pre-trained convolutional model
base_model = VGG16(weights='imagenet',
include_top=False,
input_shape=get_input_shape(img_height, img_width))
# build a classifier model to put on top of the convolutional model
top_model = Sequential()
top_model.add(Flatten(input_shape=base_model.output_shape[1:]))
for i in range(6, 8):
top_model.add(Dense(num_fc_neurons, name='fc' + str(i)))
#top_model.add(BatchNormalization(axis=1, name='fc'+str(i)+'_bn'))
top_model.add(Activation('relu', name='fc' + str(i) + '_ac'))
top_model.add(Dropout(dropout_rate))
top_model.add(Dense(num_classes, activation=activation,
name='predictions'))
if top_model_weights_path != None:
top_model.load_weights(top_model_weights_path)
if include_loc == 'base':
model = base_model
elif include_loc == 'top':
model = top_model
elif include_loc == 'all': # add the model on top of the convolutional base
model = Model(inputs=base_model.input,
outputs=top_model(base_model.output))
else:
raise ValueError('Only "base", "top" and "all" can be included.')
return model
"""
from tensorflow.contrib.keras.python.keras.models import Sequential
from tensorflow.contrib.keras.python.keras.layers import Dense, Dropout, Activation
from tensorflow.contrib.keras.python.keras.optimizers import SGD
from tensorflow.contrib.keras.python.keras.utils import to_categorical
import numpy as np
# Generate dummy data
x_train = np.random.random((1000, 20))
y_train = np.random.randint(2, size=(1000, 1))
x_test = np.random.random((100, 20))
y_test = np.random.randint(2, size=(100, 1))
model = Sequential()
model.add(Dense(64, input_dim=20, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(64, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', # binary classification
optimizer='rmsprop',
metrics=['accuracy'])
hist = model.fit(x_train, y_train,
validation_split=0.2,
epochs=1,
batch_size=128)
hist.history
score = model.evaluate(x_test, y_test, batch_size=128)
def MobileNet(
input_shape=None, # pylint: disable=invalid-name
alpha=1.0,
depth_multiplier=1,
dropout=1e-3,
include_top=True,
weights='imagenet',
input_tensor=None,
pooling=None,
classes=1000):
"""Instantiates the MobileNet architecture.
Note that only TensorFlow is supported for now,
therefore it only works with the data format
`image_data_format='channels_last'` in your Keras config
at `~/.keras/keras.json`.
To load a MobileNet model via `load_model`, import the custom
objects `relu6` and `DepthwiseConv2D` and pass them to the
`custom_objects` parameter.
E.g.
model = load_model('mobilenet.h5', custom_objects={
'relu6': mobilenet.relu6,
'DepthwiseConv2D': mobilenet.DepthwiseConv2D})
Arguments:
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(224, 224, 3)` (with `channels_last` data format)
or (3, 224, 224) (with `channels_first` data format).
It should have exactly 3 input channels,
and width and height should be no smaller than 32.
E.g. `(200, 200, 3)` would be one valid value.
alpha: controls the width of the network.
- If `alpha` < 1.0, proportionally decreases the number
of filters in each layer.
- If `alpha` > 1.0, proportionally increases the number
of filters in each layer.
- If `alpha` = 1, default number of filters from the paper
are used at each layer.
depth_multiplier: depth multiplier for depthwise convolution
(also called the resolution multiplier)
dropout: dropout rate
include_top: whether to include the fully-connected
layer at the top of the network.
weights: `None` (random initialization) or
`imagenet` (ImageNet weights)
input_tensor: optional Keras tensor (i.e. output of
`layers.Input()`)
to use as image input for the model.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model
will be the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a
2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
Returns:
A Keras model instance.
Raises:
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
RuntimeError: If attempting to run this model with a
backend that does not support separable convolutions.
"""
if K.backend() != 'tensorflow':
raise RuntimeError('Only TensorFlow backend is currently supported, '
'as other backends do not support '
'depthwise convolution.')
if weights not in {'imagenet', None}:
raise ValueError('The `weights` argument should be either '
'`None` (random initialization) or `imagenet` '
'(pre-training on ImageNet).')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as ImageNet with `include_top` '
'as true, `classes` should be 1000')
# Determine proper input shape.
if input_shape is None:
default_size = 224
else:
if K.image_data_format() == 'channels_first':
rows = input_shape[1]
cols = input_shape[2]
else:
rows = input_shape[0]
cols = input_shape[1]
if rows == cols and rows in [128, 160, 192, 224]:
default_size = rows
else:
default_size = 224
input_shape = _obtain_input_shape(input_shape,
default_size=default_size,
min_size=32,
data_format=K.image_data_format(),
require_flatten=include_top,
weights=weights)
if K.image_data_format() == 'channels_last':
row_axis, col_axis = (0, 1)
else:
row_axis, col_axis = (1, 2)
rows = input_shape[row_axis]
cols = input_shape[col_axis]
if weights == 'imagenet':
if depth_multiplier != 1:
raise ValueError('If imagenet weights are being loaded, '
'depth multiplier must be 1')
if alpha not in [0.25, 0.50, 0.75, 1.0]:
raise ValueError('If imagenet weights are being loaded, '
'alpha can be one of'
'`0.25`, `0.50`, `0.75` or `1.0` only.')
if rows != cols or rows not in [128, 160, 192, 224]:
raise ValueError('If imagenet weights are being loaded, '
'input must have a static square shape (one of '
'(128,128), (160,160), (192,192), or (224, 224)).'
' Input shape provided = %s' % (input_shape, ))
if K.image_data_format() != 'channels_last':
warnings.warn('The MobileNet family of models is only available '
'for the input data format "channels_last" '
'(width, height, channels). '
'However your settings specify the default '
'data format "channels_first" (channels, width, height).'
' You should set `image_data_format="channels_last"` '
'in your Keras config located at ~/.keras/keras.json. '
'The model being returned right now will expect inputs '
'to follow the "channels_last" data format.')
K.set_image_data_format('channels_last')
old_data_format = 'channels_first'
else:
old_data_format = None
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
x = _conv_block(img_input, 32, alpha, strides=(2, 2))
x = _depthwise_conv_block(x, 64, alpha, depth_multiplier, block_id=1)
x = _depthwise_conv_block(x,
128,
alpha,
depth_multiplier,
strides=(2, 2),
block_id=2)
x = _depthwise_conv_block(x, 128, alpha, depth_multiplier, block_id=3)
x = _depthwise_conv_block(x,
256,
alpha,
depth_multiplier,
strides=(2, 2),
block_id=4)
x = _depthwise_conv_block(x, 256, alpha, depth_multiplier, block_id=5)
x = _depthwise_conv_block(x,
512,
alpha,
depth_multiplier,
strides=(2, 2),
block_id=6)
x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=7)
x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=8)
x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=9)
x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=10)
x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=11)
x = _depthwise_conv_block(x,
1024,
alpha,
depth_multiplier,
strides=(2, 2),
block_id=12)
x = _depthwise_conv_block(x, 1024, alpha, depth_multiplier, block_id=13)
if include_top:
if K.image_data_format() == 'channels_first':
shape = (int(1024 * alpha), 1, 1)
else:
shape = (1, 1, int(1024 * alpha))
x = GlobalAveragePooling2D()(x)
x = Reshape(shape, name='reshape_1')(x)
x = Dropout(dropout, name='dropout')(x)
x = Conv2D(classes, (1, 1), padding='same', name='conv_preds')(x)
x = Activation('softmax', name='act_softmax')(x)
x = Reshape((classes, ), name='reshape_2')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='mobilenet_%0.2f_%s' % (alpha, rows))
# load weights
if weights == 'imagenet':
if K.image_data_format() == 'channels_first':
raise ValueError('Weights for "channels_last" format '
'are not available.')
if alpha == 1.0:
alpha_text = '1_0'
elif alpha == 0.75:
alpha_text = '7_5'
elif alpha == 0.50:
alpha_text = '5_0'
else:
alpha_text = '2_5'
if include_top:
model_name = 'mobilenet_%s_%d_tf.h5' % (alpha_text, rows)
weigh_path = BASE_WEIGHT_PATH + model_name
weights_path = get_file(model_name,
weigh_path,
cache_subdir='models')
else:
model_name = 'mobilenet_%s_%d_tf_no_top.h5' % (alpha_text, rows)
weigh_path = BASE_WEIGHT_PATH + model_name
weights_path = get_file(model_name,
weigh_path,
cache_subdir='models')
model.load_weights(weights_path)
if old_data_format:
K.set_image_data_format(old_data_format)
return model
test_label_mat = "label_foot.mat",
seq_len=seq_len)
N_train = len(train)
vector_dim = train.shape[2]
train = train.reshape(-1, seq_len, vector_dim, 1)
test = test.reshape(-1, seq_len, vector_dim, 1)
'''
construction model
'''
inputs = Input(shape=(train.shape[1:]))
x = Conv2D(16, (8, 1), padding='same', use_bias=False)(inputs)
x = Conv2D(16, (8, 1), padding='same', use_bias=False, activation='relu')(x)
x = BatchNormalization(axis=1)(x)
x = Dropout(0.3)(x)
x = Conv2D(32, (8, 1), padding='same', use_bias=False)(inputs)
x = Conv2D(32, (8, 1), padding='same', use_bias=False, activation='relu')(x)
x = BatchNormalization(axis=1)(x)
x = Dropout(0.3)(x)
x = Conv2D(64, (8, 1), padding='same', use_bias=False)(inputs)
x = Conv2D(64, (8, 1), padding='same', use_bias=False, activation='relu')(x)
x = BatchNormalization(axis=1)(x)
x = Dropout(0.3)(x)
x = Conv2D(64, (1, 5), padding='valid', use_bias=False, activation='relu')(x)
x = Reshape((128, 64))(x)
x = Bidirectional(LSTM(20, dropout=0.2, return_sequences=False))(x)
- model.fit(batch_size), model.fit_generator() are to train a small batch at a time until all dataset are trained
"""
from tensorflow.contrib.keras.python.keras.layers import Dropout, BatchNormalization, Input, Dense
from tensorflow.contrib.keras.python.keras.models import Model
import numpy as np
from tensorflow.contrib.keras.python.keras import backend as K
from tensorflow.contrib.keras.python.keras import losses
x = np.random.random((10, 10)) * 2
y = np.random.randint(2, size=(10, 1))
input_tensor = Input(shape=(10, ))
bn_tensor = BatchNormalization()(input_tensor)
dp_tensor = Dropout(0.7)(bn_tensor)
final_tensor = Dense(1)(dp_tensor)
model = Model(input_tensor, final_tensor)
model.compile(optimizer='SGD', loss='mse')
# x, y won't get into batches for training here
loss_on_batch = model.train_on_batch(x, y) # dive in for details
"""
('Runs a single gradient update on a single batch of data.\n'
'\n'
'Arguments:\n'
' x: Numpy array of training data,\n'
' or list of Numpy arrays if the model has multiple inputs.\n'
' If all inputs in the model are named,\n'
' you can also pass a dictionary\n'
- BatchNormalization layer output arrays on test and train mode - Dropout layer output arrays on test and train mode """ from tensorflow.contrib.keras.python.keras.layers import Dropout, BatchNormalization, Input, Dense from tensorflow.contrib.keras.python.keras.models import Model, Sequential, load_model import numpy as np from tensorflow.contrib.keras.python.keras import backend as K input_array_small = np.random.random((500, 10)) * 2 target_small = np.random.random((500, 1)) input_tensor = Input(shape=(10, )) bn_tensor = BatchNormalization()(input_tensor) dp_tensor = Dropout(0.7)(input_tensor) #### Access BatchNormalization layer's output as arrays in both test, train mode # test mode from Model method model_bn = Model(input_tensor, bn_tensor) bn_array = model_bn.predict(input_array_small) # test and train mode from K.function method k_bn = K.function([input_tensor, K.learning_phase()], [bn_tensor]) bn_array_test = k_bn([input_array_small, 0])[0] bn_array_train = k_bn([input_array_small, 1])[0] # are test mode the same? and test mode array differ from train mode array (bn_array == bn_array_test).sum() bn_array.shape # compare to see for equality
# test = np.r_[test1, test2]
# test_label = np.r_[test_label1, test_label2]
#train = np.r_[train, test]
#train_label = np.r_[train_label, test_label]
epochs = 100
batch_size = 256
f_dim = train.shape[2]
lstm1_dim = 20
lstm2_dim = 20
model = Sequential()
model.add(Convolution1D(8, 4, input_shape=(seq_len, f_dim)))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(MaxPooling1D(3))
model.add(Convolution1D(12, 4, kernel_regularizer=l2(0.05)))
model.add(Activation('relu'))
model.add(Dropout(0.4))
model.add(MaxPooling1D(3))
model.add(Convolution1D(18, 4, kernel_regularizer=l2(0.04)))
model.add(Activation('relu'))
model.add(Dropout(0.3))
model.add(MaxPooling1D(3))
model.add(Bidirectional(LSTM(lstm1_dim, dropout=0.2, return_sequences=True)))
model.add(Bidirectional(LSTM(lstm2_dim, dropout=0.2)))
model.add(BatchNormalization(axis=1))
model.add(Dense(30, kernel_regularizer=l2(0.02)))
model.add(BatchNormalization())
model.add(Activation('relu'))
'labels_' + str(lendata - 1) + '_' + str(lendata) + '.npy'),
labels[lendata - 1:lendata])
tokenizer = None
data = None
labels = None
embedding_layer = Embedding(len(word_index) + 1,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=False)
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH, ), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
x = Dropout(0.5)(embedded_sequences)
x = Conv1D(filters=128, kernel_size=3, padding='same', activation='relu')(x)
x = MaxPooling1D(pool_size=2)(x)
x = LSTM(100, dropout=0.2, recurrent_dropout=0.2, return_sequences=True)(x)
x = LSTM(100, dropout=0.2, recurrent_dropout=0.2)(x)
x = Dense(32, activation='relu')(x)
preds = Dense(len(labels_index), activation='softmax')(x)
model = Model(sequence_input, preds)
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['acc'])
embedding_matrix = None
embeddings_index = None
def create_AlexNet(num_fc_neurons,
dropout_rate,
num_classes=24,
img_height=224,
img_width=224,
include_loc='all',
activation='softmax'):
weight_decay = 0.0005
kernel_regularizer = regularizers.l2(weight_decay)
bias_regularizer = regularizers.l2(weight_decay)
# build a convolutional model
base_model = Sequential()
base_model.add(
Conv2D(96, (11, 11),
strides=(4, 4),
padding='valid',
activation='relu',
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
name='conv1',
input_shape=get_input_shape(img_height, img_width)))
base_model.add(LRN2D(name='lrn1'))
base_model.add(MaxPooling2D((3, 3), strides=(2, 2), name='pool1'))
base_model.add(
Conv2D(256, (5, 5),
strides=(1, 1),
padding='same',
activation='relu',
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
name='conv2'))
base_model.add(LRN2D(name='lrn2'))
base_model.add(MaxPooling2D((3, 3), strides=(2, 2), name='pool2'))
base_model.add(
Conv2D(384, (3, 3),
strides=(1, 1),
padding='same',
activation='relu',
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
name='conv3'))
base_model.add(
Conv2D(384, (3, 3),
strides=(1, 1),
padding='same',
activation='relu',
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
name='conv4'))
base_model.add(
Conv2D(256, (3, 3),
strides=(1, 1),
padding='same',
activation='relu',
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
name='conv5'))
base_model.add(MaxPooling2D((3, 3), strides=(2, 2), name='pool3'))
# build a classifier model to put on top of the convolutional model
top_model = Sequential()
top_model.add(Flatten(input_shape=base_model.output_shape[1:]))
for i in range(6, 8):
top_model.add(
Dense(num_fc_neurons,
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
name='fc' + str(i)))
#top_model.add(BatchNormalization(axis=1, name='fc'+str(i)+'_bn'))
top_model.add(Activation('relu', name='fc' + str(i) + '_ac'))
top_model.add(Dropout(dropout_rate))
top_model.add(
Dense(num_classes,
activation=activation,
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
name='predictions'))
if include_loc == 'base':
model = base_model
elif include_loc == 'top':
model = top_model
elif include_loc == 'all': # add the model on top of the convolutional base
model = Model(inputs=base_model.input,
outputs=top_model(base_model.output))
else:
raise ValueError('Only "base", "top" and "all" can be included.')
return model
tokenizer = None
data= None
labels=None
embedding_layer = Embedding(len(word_index) + 1,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=False)
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
x = Conv1D(128, 5, activation='relu')(embedded_sequences)
x = MaxPooling1D(5)(x)
x = Dropout (0.2)(x)
x = Conv1D(128, 5, activation='relu')(x)
x = MaxPooling1D(5)(x)
x = Conv1D(128, 5, activation='relu')(x)
x = MaxPooling1D(5)(x)
x = Conv1D(128, 5, activation='relu')(x)
x = MaxPooling1D(35)(x) # global max pooling
x = Dropout (0.5)(x)
x = Flatten()(x)
x = Dense(512, activation='relu')(x)
preds = Dense(len(labels_index), activation='softmax')(x)
model = Model(sequence_input, preds)
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['acc'])
def cnn_sentiment_model(inputs,
nb_words,
embedding_dim=300,
static_embedding=True,
embedding_weights=None,
filter_hs=None,
nb_filters=100,
emb_size=100,
hidden_dropout=0.2,
is_training=True,
augmentation_function=None,
l2_weight=1e-4,
img_shape=None,
new_shape=None,
image_summary=False,
batch_norm_decay=0.99,
seed=0,
embedding_dropout=0.2):
from tensorflow.contrib.keras.python.keras.layers import Embedding, Input, Convolution1D, MaxPooling1D, Flatten, \
Dense, Dropout, Activation
from tensorflow.contrib.keras.python.keras.initializers import glorot_uniform
from tensorflow.contrib.keras.python.keras.layers.merge import Concatenate
from tensorflow.contrib.keras.python.keras import backend as K
K.set_learning_phase(1 if is_training else 0)
sequence_length = img_shape[0]
if filter_hs is None:
filter_hs = [3, 4, 5]
model = inputs
def ci(shape, dtype=None, partition_info=None):
assert shape[0] == embedding_weights.shape[0] and shape[
1] == embedding_weights.shape[
1], 'Shapes are not equal required={} init value={}'.format(
shape, embedding_weights.shape)
return embedding_weights
model = Embedding(nb_words,
embedding_dim,
input_length=sequence_length,
trainable=(not static_embedding),
embeddings_initializer='uniform'
if embedding_weights is None else ci)(model)
if embedding_dropout > 0.0:
model = Dropout(embedding_dropout, seed=seed)(model,
training=is_training)
convs = list()
for fsz in filter_hs:
conv = Convolution1D(
filters=nb_filters,
kernel_size=fsz,
padding='valid',
activation='relu',
kernel_initializer=glorot_uniform(seed=seed))(model)
pool = MaxPooling1D(pool_size=sequence_length - fsz + 1)(conv)
flatten = Flatten()(pool)
convs.append(flatten)
if len(filter_hs) > 0:
graph_out = Concatenate()(convs)
else:
graph_out = convs[0]
model = graph_out
model = Dense(emb_size,
kernel_initializer=glorot_uniform(seed=seed))(model)
model = Dropout(hidden_dropout, seed=seed)(model, training=is_training)
model = Activation('relu')(model)
return model
def fit(self, eventlog_name):
import tensorflow as tf
from tensorflow.contrib.keras.python.keras.engine import Input, Model
from tensorflow.contrib.keras.python.keras.layers import Dense, Dropout, GRU, Embedding, merge, Masking
features, targets = self.dataset.load(eventlog_name, train=True)
inputs = []
layers = []
with tf.device('/cpu:0'):
# split attributes
features = [features[:, :, i] for i in range(features.shape[2])]
for i, t in enumerate(features):
voc_size = np.array(self.dataset.attribute_dims[i]) + 1 # we start at 1, hence +1
emb_size = np.floor(voc_size / 2.0).astype(int)
i = Input(shape=(None, *t.shape[2:]))
x = Embedding(input_dim=voc_size, output_dim=emb_size, input_length=t.shape[1], mask_zero=True)(i)
inputs.append(i)
layers.append(x)
# merge layers
x = merge.concatenate(layers)
x = GRU(64, implementation=2)(x)
# shared hidden layer
x = Dense(512, activation=tf.nn.relu)(x)
x = Dense(512, activation=tf.nn.relu)(Dropout(0.5)(x))
# hidden layers per attribute
outputs = []
for i, l in enumerate(targets):
o = Dense(256, activation=tf.nn.relu)(Dropout(0.5)(x))
o = Dense(256, activation=tf.nn.relu)(Dropout(0.5)(o))
o = Dense(l.shape[1], activation=tf.nn.softmax)(Dropout(0.5)(o))
outputs.append(o)
self.model = Model(inputs=inputs, outputs=outputs)
# compile model
# old setting : optimizers from tensorflow
# self.model.compile(
# optimizer=tf.train.AdamOptimizer(learning_rate=0.0001),
# loss='categorical_crossentropy'
# )
# new setting : optimizers from keras
self.model.compile(
optimizer='Adadelta',
loss='categorical_crossentropy'
)
# train model
self.model.fit(
features,
targets,
batch_size=100,
epochs=100,
validation_split=0.2,
)
batch_size = 1024
f_dim = train.shape[2]
lstm1_dim = 20
lstm2_dim = 20
model = Sequential()
model.add(
Bidirectional(LSTM(lstm1_dim, dropout=0.2, return_sequences=True),
input_shape=(seq_len, f_dim)))
model.add(Bidirectional(LSTM(lstm2_dim, dropout=0.2)))
model.add(BatchNormalization(axis=1))
model.add(Dense(30, kernel_regularizer=l2(0.02)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(Dropout(0.1))
model.add(Dense(2, kernel_regularizer=l2(0.01)))
model.add(BatchNormalization())
model.add(Activation('softmax'))
model.compile(optimizer='Nadam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.summary()
csv_logger = CSVLogger('training.log')
hist = model.fit(train,
train_label,
epochs=epochs,
batch_size=batch_size,
validation_split=2 / 7,
callbacks=[csv_logger],
np.save(os.path.join(SAVE_DIR, 'gens_'+str(i)+'_'+str(lendata)+'.npy'), data_gen[i:lendata])
np.save(os.path.join(SAVE_DIR, 'labels_' + str(i) + '_' + str(lendata) + '.npy'), labels[i:lendata])
np.save(os.path.join(SAVE_DIR, 'data_' + str(lendata-1) + '_' + str(lendata) + '.npy'), data[lendata-1:lendata])
np.save(os.path.join(SAVE_DIR, 'gens_' + str(lendata-1) + '_' + str(lendata) + '.npy'), data_gen[lendata-1:lendata])
np.save(os.path.join(SAVE_DIR, 'labels_' + str(lendata-1) + '_' + str(lendata) + '.npy'), labels[lendata-1:lendata])
tokenizer = None
data= None
labels=None
embedding_layer_gen = Embedding(len(word_index_gen) + 1,
300,
input_length=2)
sequence_input_gen = Input(shape=(2,), dtype='int32')
embedded_sequences_gen = embedding_layer_gen(sequence_input_gen)
x_gen = Dropout (0.8)(embedded_sequences_gen)
x_gen = Flatten()(x_gen)
x_gen = Dense(512, activation='relu')(x_gen)
embedding_layer = Embedding(len(word_index) + 1,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=False)
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
x = Conv1D(128, 15, activation='relu')(embedded_sequences)
x = MaxPooling1D(5)(x)
epochs = 100
batch_size = 256
f_dim = train.shape[2]
model = Sequential()
model.add(
Convolution2D(32, (2, 2),
2,
padding='same',
input_shape=train.shape[1:],
kernel_regularizer=l2(0.005),
use_bias=False))
model.add(BatchNormalization(axis=3))
model.add(Dropout(0.3))
model.add(
Convolution2D(64, (2, 2),
2,
padding='same',
kernel_regularizer=l2(0.005),
use_bias=False))
model.add(BatchNormalization(axis=3))
model.add(Dropout(0.3))
model.add(Activation("relu"))
model.add(
Convolution2D(128, (2, 2),
2,
padding='same',
kernel_regularizer=l2(0.005),
#hidden_initializer = random_uniform(seed=SEED)
tf.set_random_seed(32)
hidden_initializer = tf.random_uniform([1])
dropout_rate = 0.2
savefile = './models.h5'
# create model
if os.path.isfile(savefile):
model = load_model(savefile)
else:
model = Sequential()
model.add(Dense(128, input_dim=input_dimension, kernel_initializer=hidden_initializer, activation='relu'))
#model.add(Dense(128, input_dim=input_dimension, activation='relu'))
model.add(Dropout(dropout_rate))
model.add(Dense(64, kernel_initializer=hidden_initializer, activation='relu'))
model.add(Dense(2, kernel_initializer=hidden_initializer, activation='softmax'))
#model.add(Dense(64, activation='relu'))
#model.add(Dense(2, activation='softmax'))
sgd = SGD(lr=learning_rate, momentum=momentum)
model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['acc'])
model.fit(X_train, y_train, epochs=50, batch_size=128)
predictions = model.predict_proba(X_test)
ans = pd.DataFrame(predictions,columns=['target','dmy'])
print ans
outdf = pd.concat([outdf,ans['target']],axis=1)
outdf.to_csv('./submit_keras.csv',index=False)
from tensorflow.contrib.keras.python.keras.layers import Embedding
from tensorflow.contrib.keras.python.keras.layers import LSTM
from tensorflow.contrib.keras.python.keras import backend as K
model = Sequential()
model.add(Embedding(input_dim=64, output_dim=256, input_length=10))
# input_dim: Size of the vocabulary
# input_length: Length of input sequences, length of each sentences
# output_dim: Dimension of the dense embedding
model.input # (?, 10),
model.output # (?, 10, 256)
model.add(LSTM(128)) # unit=128, dimensionality of the output space
model.output
model.add(Dropout(0.5)) # percent to drop out
# model.ouput # will cause error on Dropout
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
import numpy as np
x_train = np.random.random((1000, 10))
y_train = np.random.randint(2, size=(1000, 1))
x_test = np.random.random((100, 10))
y_test = np.random.randint(2, size=(100, 1))
hist=model.fit(x_train, y_train, validation_split=0.2, batch_size=16, epochs=1)
hist.history
