def VGG16_Places365(include_top=True,
weights='places',
input_tensor=None,
input_shape=None,
pooling=None,
classes=365):
"""Instantiates the VGG16-places365 architecture.
Optionally loads weights pre-trained
on Places. Note that when using TensorFlow,
for best performance you should set
`image_data_format="channels_last"` in your Keras config
at ~/.keras/keras.json.
The model and the weights are compatible with both
TensorFlow and Theano. The data format
convention used by the model is the one
specified in your Keras config file.
# Arguments
include_top: whether to include the 3 fully-connected
layers at the top of the network.
weights: one of `None` (random initialization),
'places' (pre-training on Places),
or the path to the weights file to be loaded.
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(224, 224, 3)` (with `channels_last` data format)
or `(3, 224, 244)` (with `channels_first` data format).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 48.
E.g. `(200, 200, 3)` would be one valid value.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
# Returns
A Keras model instance.
# Raises
ValueError: in case of invalid argument for `weights`, or invalid input shape
"""
if not (weights in {'places', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `places` '
'(pre-training on Places), '
'or the path to the weights file to be loaded.')
if weights == 'places' and include_top and classes != 365:
raise ValueError('If using `weights` as places with `include_top`'
' as true, `classes` should be 365')
# Determine proper input shape
input_shape = _obtain_input_shape(input_shape,
default_size=224,
min_size=48,
data_format=K.image_data_format(),
require_flatten=include_top)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
# Block 1
x = Conv2D(filters=64,
kernel_size=3,
strides=(1, 1),
padding='same',
kernel_regularizer=l2(0.0002),
activation='relu',
name='block1_conv1')(img_input)
x = Conv2D(filters=64,
kernel_size=3,
strides=(1, 1),
padding='same',
kernel_regularizer=l2(0.0002),
activation='relu',
name='block1_conv2')(x)
x = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
name="block1_pool",
padding='valid')(x)
# Block 2
x = Conv2D(filters=128,
kernel_size=3,
strides=(1, 1),
padding='same',
kernel_regularizer=l2(0.0002),
activation='relu',
name='block2_conv1')(x)
x = Conv2D(filters=128,
kernel_size=3,
strides=(1, 1),
padding='same',
kernel_regularizer=l2(0.0002),
activation='relu',
name='block2_conv2')(x)
x = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
name="block2_pool",
padding='valid')(x)
# Block 3
x = Conv2D(filters=256,
kernel_size=3,
strides=(1, 1),
padding='same',
kernel_regularizer=l2(0.0002),
activation='relu',
name='block3_conv1')(x)
x = Conv2D(filters=256,
kernel_size=3,
strides=(1, 1),
padding='same',
kernel_regularizer=l2(0.0002),
activation='relu',
name='block3_conv2')(x)
x = Conv2D(filters=256,
kernel_size=3,
strides=(1, 1),
padding='same',
kernel_regularizer=l2(0.0002),
activation='relu',
name='block3_conv3')(x)
x = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
name="block3_pool",
padding='valid')(x)
# Block 4
x = Conv2D(filters=512,
kernel_size=3,
strides=(1, 1),
padding='same',
kernel_regularizer=l2(0.0002),
activation='relu',
name='block4_conv1')(x)
x = Conv2D(filters=512,
kernel_size=3,
strides=(1, 1),
padding='same',
kernel_regularizer=l2(0.0002),
activation='relu',
name='block4_conv2')(x)
x = Conv2D(filters=512,
kernel_size=3,
strides=(1, 1),
padding='same',
kernel_regularizer=l2(0.0002),
activation='relu',
name='block4_conv3')(x)
x = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
name="block4_pool",
padding='valid')(x)
# Block 5
x = Conv2D(filters=512,
kernel_size=3,
strides=(1, 1),
padding='same',
kernel_regularizer=l2(0.0002),
activation='relu',
name='block5_conv1')(x)
x = Conv2D(filters=512,
kernel_size=3,
strides=(1, 1),
padding='same',
kernel_regularizer=l2(0.0002),
activation='relu',
name='block5_conv2')(x)
x = Conv2D(filters=512,
kernel_size=3,
strides=(1, 1),
padding='same',
kernel_regularizer=l2(0.0002),
activation='relu',
name='block5_conv3')(x)
x = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
name="block5_pool",
padding='valid')(x)
if include_top:
# Classification block
x = Flatten(name='flatten')(x)
x = Dense(4096, activation='relu', name='fc1')(x)
x = Dropout(0.5, name='drop_fc1')(x)
x = Dense(4096, activation='relu', name='fc2')(x)
x = Dropout(0.5, name='drop_fc2')(x)
x = Dense(365, activation='softmax', name="predictions")(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='vgg16-places365')
# load weights
if weights == 'places':
if include_top:
weights_path = get_file(
'vgg16-places365_weights_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH,
cache_subdir='models')
else:
weights_path = get_file(
'vgg16-places365_weights_tf_dim_ordering_tf_kernels_notop.h5',
WEIGHTS_PATH_NO_TOP,
cache_subdir='models')
model.load_weights(weights_path)
if K.backend() == 'theano':
layer_utils.convert_all_kernels_in_model(model)
if K.image_data_format() == 'channels_first':
if include_top:
maxpool = model.get_layer(name='block5_pool')
shape = maxpool.output_shape[1:]
dense = model.get_layer(name='fc1')
layer_utils.convert_dense_weights_data_format(
dense, shape, 'channels_first')
if K.backend() == 'tensorflow':
warnings.warn('You are using the TensorFlow backend, yet you '
'are using the Theano '
'image data format convention '
'(`image_data_format="channels_first"`). '
'For best performance, set '
'`image_data_format="channels_last"` in '
'your Keras config '
'at ~/.keras/keras.json.')
elif weights is not None:
model.load_weights(weights)
return model
train_xception = extract_Xception(train_files)
valid_xception = extract_Xception(valid_files)
print("Xception shape", train_xception.shape[1:])
train_inceptionResNetV2 = extract_InceptionResNetV2(train_files)
valid_inceptionResNetV2 = extract_InceptionResNetV2(valid_files)
print("InceptionResNetV2 shape", train_inceptionResNetV2.shape[1:])
xception_branch, xception_input = input_branch(
input_shape=(train_xception.shape[1:]))
inceptionResNetV2_branch, inceptionResNetV2_input = input_branch(
input_shape=(train_inceptionResNetV2.shape[1:]))
concatenate_branches = Concatenate()(
[xception_branch, inceptionResNetV2_branch])
net = Dropout(0.3)(concatenate_branches)
net = Dense(1024, use_bias=False, kernel_initializer='uniform')(net)
net = BatchNormalization()(net)
net = Activation("relu")(net)
net = Dropout(0.3)(net)
net = Dense(120, kernel_initializer='uniform', activation="softmax")(net)
model = Model(inputs=[xception_input, inceptionResNetV2_input], outputs=[net])
model.compile(loss='categorical_crossentropy',
optimizer="sgd",
metrics=['accuracy'])
model.fit([train_xception, train_inceptionResNetV2],
train_targets,
validation_data=([valid_xception,
valid_inceptionResNetV2], valid_targets),
def get_nts_net(batch_size=2):
input_shape = (image_dimension, image_dimension, 3)
img_input = Input(shape=input_shape, name='img_input')
# input1 = Input((PROPOSAL_NUM,), name = 'ranking_loss_input')
base_model = ResNet50(
weights='imagenet',
include_top=False,
weight_decay=0, # add weight decay
# input_tensor=img_input,
# input_shape=input_shape,
pooling="avg")
layer_dict = {}
for layer in base_model.layers:
layer_dict[layer.name] = layer
# g = base_model(img_input)
global_conv_outputs = layer_dict['activation_49'].output
# global_conv_outputs = layer_dict['block5_conv3'].output
base_model_1 = Model(inputs=base_model.input,
outputs=global_conv_outputs,
name='resnet50') # shared model
global_conv_outputs = base_model_1(img_input)
global_features = GlobalAveragePooling2D()(global_conv_outputs)
global_conv_outputs_gradient_stopped = Lambda(
lambda x: K.stop_gradient(x),
name='stop_gradient')(global_conv_outputs)
global_features = Dropout(0.5)(global_features)
whole_image_pred = Dense(
num_classes,
activation="softmax",
kernel_regularizer=regularizers.l2(weight_decay),
bias_regularizer=regularizers.l2(weight_decay),
# activity_regularizer=regularizers.l2(weight_decay),
name="cls_pred_global")(global_features)
# proposal net
c1 = Conv2D(
128,
(3, 3),
activation='relu',
padding='same',
kernel_regularizer=regularizers.l2(weight_decay),
bias_regularizer=regularizers.l2(weight_decay),
# activity_regularizer=regularizers.l2(weight_decay),
name='c1')(global_conv_outputs_gradient_stopped)
c2 = Conv2D(
128,
(3, 3),
strides=2,
activation='relu',
padding='same',
kernel_regularizer=regularizers.l2(weight_decay),
bias_regularizer=regularizers.l2(weight_decay),
# activity_regularizer=regularizers.l2(weight_decay),
name='c2')(c1)
c3 = Conv2D(
128,
(3, 3),
strides=2,
activation='relu',
padding='same',
kernel_regularizer=regularizers.l2(weight_decay),
bias_regularizer=regularizers.l2(weight_decay),
# activity_regularizer=regularizers.l2(weight_decay),
name='c3')(c2)
p1 = Conv2D(
6,
(1, 1),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay),
bias_regularizer=regularizers.l2(weight_decay),
# activity_regularizer=regularizers.l2(weight_decay),
name='p1')(c1)
p2 = Conv2D(
6,
(1, 1),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay),
bias_regularizer=regularizers.l2(weight_decay),
# activity_regularizer=regularizers.l2(weight_decay),
name='p2')(c2)
p3 = Conv2D(
9,
(1, 1),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay),
bias_regularizer=regularizers.l2(weight_decay),
# activity_regularizer=regularizers.l2(weight_decay),
name='p3')(c3)
p1_reshape = Reshape((-1, ), name='p1_reshape')(p1)
p2_reshape = Reshape((-1, ), name='p2_reshape')(p2)
p3_reshape = Reshape((-1, ), name='p3_reshape')(p3)
rpn_score = Concatenate(
axis=-1, name='rpn_concat')([p1_reshape, p2_reshape, p3_reshape])
# anchors = Lambda(lambda x: K.variable(generate_default_anchor_maps()))(K.variable(1))
selected_indices = NMS(PROPOSAL_NUM=PROPOSAL_NUM)(rpn_score)
topN_scores = CollectScores(name='proposal_scores')(
[rpn_score, selected_indices])
crop_region = CollectRegions()(selected_indices)
# part image cls
img_input_padded = Lambda(lambda x: K.spatial_2d_padding(
x, padding=((pad_size, pad_size), (pad_size, pad_size))),
name='pad_image')(img_input)
part_imgs = CropImage(img_size=PART_RESIZE,
pad_size=pad_size,
nbox=PROPOSAL_NUM)([img_input_padded, crop_region])
part_imgs_stop_gradient = Lambda(lambda x: K.stop_gradient(x),
name='stop_gradient_part_img')(part_imgs)
part_conv_outputs = base_model_1(part_imgs_stop_gradient)
part_features = GlobalAveragePooling2D()(
part_conv_outputs) # (batch_size*PROPOSAL_NUM,channel)
part_features = Dropout(0.5)(part_features)
part_pred = Dense(
num_classes,
activation="softmax",
kernel_regularizer=regularizers.l2(weight_decay),
bias_regularizer=regularizers.l2(weight_decay),
# activity_regularizer=regularizers.l2(weight_decay),
name="cls_pred_part")(part_features)
# concat pred
part_feature_cat = Lambda(
lambda x: K.reshape(x, (-1, PROPOSAL_NUM, last_conv_channel)),
name='part_feature_reshape_1')(part_features)
part_feature_cat = Lambda(lambda x: x[:, :CAT_NUM, :],
name='part_feature_slice')(part_feature_cat)
part_feature_cat = Reshape((-1, ),
name='part_feature_reshape_2')(part_feature_cat)
concat_feature = Concatenate(
axis=-1, name='feature_concat')([global_features, part_feature_cat])
concat_feature = Dropout(0.5)(concat_feature)
concat_pred = Dense(
num_classes,
activation="softmax",
kernel_regularizer=regularizers.l2(weight_decay),
bias_regularizer=regularizers.l2(weight_decay),
# activity_regularizer=regularizers.l2(weight_decay),
name="cls_pred_concat")(concat_feature)
# in order to implement rank loss, we concat topN_scores and part_pred as one output
part_pred_reshape = Lambda(
lambda x: K.reshape(x, (-1, PROPOSAL_NUM, num_classes)),
name='cls_pred_part_reshape')(part_pred)
topN_scores_reshape = Lambda(lambda x: K.reshape(x, (-1, PROPOSAL_NUM, 1)),
name='topN_scores_reshape')(topN_scores)
rank_concat = Concatenate(axis=-1, name='rank_concat_0')(
[part_pred_reshape, topN_scores_reshape])
rank_concat = Lambda(lambda x: K.reshape(x, (-1, PROPOSAL_NUM *
(num_classes + 1))),
name='rank_concat')(rank_concat)
model = Model(
inputs=img_input,
outputs=[
whole_image_pred, # (batch_size,num_classes)
part_pred, # (batch_size,PROPOSAL_NUM*num_classes)
concat_pred, # (batch_size,num_classes)
rank_concat # topN_scores#(batch_size,PROPOSAL_NUM*(num_classes+1))
],
name='nts-net')
# for layer in model.layers: # this method does not work
# layer.kernel_regularizer = regularizers.l2(weight_decay)
# layer.bias_regularizer = regularizers.l2(weight_decay)
from keras.utils import plot_model
plot_model(model, to_file='model.png')
return model
conv_5 = ZeroPadding2D((1, 1))(conv_4)
conv_5 = merge([
Convolution2D(128, 3, 3, activation="relu", name='conv_5_' + str(i + 1))(
splittensor(ratio_split=2, id_split=i)(conv_5)) for i in range(2)
],
mode='concat',
concat_axis=1,
name="conv_5")
dense_1 = MaxPooling2D((3, 3), strides=(2, 2), name="convpool_5")(conv_5)
dense_1 = Flatten(name="flatten")(dense_1)
#model = Model(input=[inputs], output=[dense_1])
# I think this actually matches, just need to rerig the weights a bit
dense_1 = Dense(4096, activation='relu', name='fc6_gtanet')(dense_1)
dense_1 = Dropout(0.5)(dense_1)
dense_2 = Dense(4096, activation='relu', name='fc7_gtanet')(dense_1)
dense_2 = Dropout(0.5)(dense_2)
output = Dense(6, activation='linear', name='gtanet_fctop')(dense_2)
model = Model(input=[inputs], output=[output])
adam = Adam(lr=0.01)
model.compile(loss=['mse'], optimizer=adam, metrics=['mse'])
# 227 MB, as advertised
Wm = model.get_weights()
# 1st conv, different axes
Wm[0] = np.transpose(W[0], (3, 2, 0, 1))
Wm[1] = b[0]
(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features,
test_split=0.2)
print(len(X_train), 'train sequences')
print(len(X_test), 'test sequences')
print("Pad sequences (samples x time)")
X_train = sequence.pad_sequences(X_train, maxlen=maxlen)
X_test = sequence.pad_sequences(X_test, maxlen=maxlen)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
print('Build model...')
model = Sequential()
model.add(Embedding(max_features, 128, input_length=maxlen))
model.add(LSTM(128)) # try using a GRU instead, for fun
model.add(Dropout(0.5))
model.add(Dense(1))
model.add(Activation('sigmoid'))
# try using different optimizers and different optimizer configs
model.compile(loss='binary_crossentropy',
optimizer='adam',
class_mode="binary")
print("Train...")
model.fit(X_train,
y_train,
batch_size=batch_size,
nb_epoch=3,
validation_data=(X_test, y_test),
show_accuracy=True)
print (X_test.shape, in_shp) # (110000, 2, 1024) [2, 1024]
classes = ['BPSK', '8PSK', 'QPSK', 'PAM4', 'QAM16', 'QAM64', 'GFSK', 'CPFSK', 'WBFM', 'AM-DSB', 'AM-SSB']
# all_keys = {"bpsk":0,"8psk":0, "qpsk":0, "pam4":0, "qam16":0, "qam64":0, "gfsk":0, "cpfsk":0, "fm": 1, "am": 1, "amssb": 1 }
# Build VT-CNN2 Neural Net model using Keras primitives --
# - Reshape [N,2,1024] to [N,1,2,1024] on input
# - Pass through 2 2DConv/ReLu layers
# - Pass through 2 Dense layers (ReLu and Softmax)
# - Perform categorical cross entropy optimization
dr = 0.5 # dropout rate (%)
model = models.Sequential()
model.add(Reshape([1]+in_shp, input_shape=in_shp))
model.add(ZeroPadding2D((0, 2), data_format="channels_first"))
model.add(Convolution2D(256, (1, 3), activation="relu", name="conv1", data_format="channels_first"))
model.add(Dropout(dr))
model.add(ZeroPadding2D((0, 2), data_format="channels_first"))
model.add(Convolution2D(80, (2, 3), activation="relu", name="conv2", data_format="channels_first"))
model.add(Dropout(dr))
model.add(Flatten())
model.add(Dense(256, activation='relu', kernel_initializer='he_normal', name="dense1"))
model.add(Dropout(dr))
model.add(Dense( len(classes), kernel_initializer='he_normal', name="dense2" ))
model.add(Activation('softmax'))
model.add(Reshape([len(classes)]))
model.compile(loss='categorical_crossentropy', optimizer='adam')
model.summary()
# Set up some params
batch_size = 256
model.compile(loss='categorical_crossentropy',
optimizer=Adam(),
metrics=['accuracy'])
history = train_and_show_result(model)
plot_training_history(history)
# Dropout
print("Dropout\n")
model = Sequential()
model.add(Dense(input_dim=28*28, output_dim=500, activation='relu'))
model.add(Dropout(0.8))
model.add(Dense(output_dim=500, activation='relu'))
model.add(Dropout(0.8))
model.add(Dense(output_dim=500, activation='relu'))
model.add(Dropout(0.8))
model.add(Dense(output_dim=500, activation='relu'))
model.add(Dropout(0.8))
model.add(Dense(output_dim=500, activation='relu'))
model.add(Dropout(0.8))
model.add(Dense(output_dim=500, activation='relu'))
model.add(Dropout(0.8))
model.add(Dense(output_dim=500, activation='relu'))
model.add(Dropout(0.8))
model.add(Dense(output_dim=500, activation='relu'))
model.add(Dropout(0.8))
model.add(Dense(output_dim=500, activation='relu'))
ispretrain = True
batch_size = 100
latent_dim = 10
intermediate_dim = [500, 500, 2000]
theano.config.floatX = 'float32'
X, Y = load_data()
original_dim = 2000
n_centroid = 4
theta_p, u_p, lambda_p = gmm_para_init()
#===================
x = Input(batch_shape=(batch_size, original_dim))
h = Dense(intermediate_dim[0])(x)
h = Activation('relu')(h)
h = Dropout(0.2)(h)
h = Dense(intermediate_dim[1])(h)
h = Activation('relu')(h)
h = Dropout(0.2)(h)
h = Dense(intermediate_dim[2])(h)
h = Activation('relu')(h)
h = Dropout(0.2)(h)
z_mean = Dense(latent_dim)(h)
z_log_var = Dense(latent_dim)(h)
z = Lambda(sampling, output_shape=(latent_dim, ))([z_mean, z_log_var])
z = Lambda(sampling, output_shape=(latent_dim, ))([z_mean, z_log_var])
h_decoded = Dense(intermediate_dim[-1])(z)
h_decoded = Dense(intermediate_dim[-2])(h_decoded)
h_decoded = Dense(intermediate_dim[-3])(h_decoded)
x_decoded_mean = Dense(original_dim)(h_decoded)
Y_valid[i, :] = np.reshape(data[i][1], 15)
#%%
patience = 30
early_stopping = EarlyStopping(patience=patience, verbose=1)
checkpointer = ModelCheckpoint(filepath=sys.argv[4],
save_best_only=True,
verbose=1)
#%%
model = Sequential()
model.add(
Conv1D(kernel_size=5,
padding='same',
filters=25,
input_shape=(shape[0], shape[1])))
model.add(Dropout(0.4))
model.add(Conv1D(kernel_size=5, padding='same', filters=20))
model.add(Dropout(0.4))
model.add(MaxPooling1D(pool_size=4))
model.add(Conv1D(kernel_size=5, padding='same', filters=15))
model.add(Dropout(0.4))
model.add(BatchNormalization())
model.add(LSTM(units=750, return_sequences=True))
model.add(Dropout(0.4))
model.add(LSTM(units=500, return_sequences=True))
model.add(Dropout(0.4))
model.add(LSTM(units=350, return_sequences=False))
model.add(Dropout(0.4))
#model.add(Flatten())
model.add(Dense(200, activation='relu'))
model.add(Dropout(0.4))
# Conv1
model.add(Convolution2D(32, 3, 3, init='glorot_uniform'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D((2, 2)))
# Conv2
model.add(Convolution2D(64, 3, 3, init='glorot_uniform'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D((2, 2)))
# Conv3
model.add(Convolution2D(128, 3, 3, init='glorot_uniform'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D((2, 2)))
model.add(Dropout(p=0.1))
# Conv4
model.add(Convolution2D(256, 3, 3, init='glorot_uniform'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(p=0.1))
# FC5
model.add(Flatten())
model.add(Dense(256, init='glorot_uniform'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(p=0.4))
# FC6
model.add(Dense(128, init='glorot_uniform'))
model.add(BatchNormalization())
model.add(Activation('relu'))
def create_model(args, initial_mean_value, overal_maxlen, vocab):
import keras.backend as K
from keras.layers.embeddings import Embedding
from keras.models import Sequential, Model
from keras.layers.core import Dense, Dropout, Activation, Flatten
from nea.my_layers import Attention, MeanOverTime, Conv1DWithMasking
###############################################################################################################################
## Recurrence unit type
#
if args.recurrent_unit == 'lstm':
from keras.layers.recurrent import LSTM as RNN
elif args.recurrent_unit == 'gru':
from keras.layers.recurrent import GRU as RNN
elif args.recurrent_unit == 'simple':
from keras.layers.recurrent import SimpleRNN as RNN
###############################################################################################################################
## Create Model
#
dropout_W = 0.5 # default=0.5
dropout_U = 0.1 # default=0.1
cnn_border_mode = 'same'
if initial_mean_value.ndim == 0:
initial_mean_value = np.expand_dims(initial_mean_value, axis=0)
num_outputs = len(initial_mean_value)
if args.model_type == 'cls':
raise NotImplementedError
elif args.model_type == 'reg':
logger.info('Building a REGRESSION model')
model = Sequential()
model.add(Embedding(args.vocab_size, args.emb_dim, mask_zero=True))
if args.cnn_dim > 0:
model.add(
Conv1DWithMasking(nb_filter=args.cnn_dim,
filter_length=args.cnn_window_size,
border_mode=cnn_border_mode,
subsample_length=1))
if args.rnn_dim > 0:
model.add(
RNN(args.rnn_dim,
return_sequences=False,
dropout=dropout_W,
recurrent_dropout=dropout_U))
if args.dropout_prob > 0:
model.add(Dropout(args.dropout_prob))
model.add(Dense(num_outputs))
if not args.skip_init_bias:
bias_value = (np.log(initial_mean_value) -
np.log(1 - initial_mean_value)).astype(K.floatx())
model.layers[-1].b.set_value(bias_value)
model.add(Activation('sigmoid'))
model.emb_index = 0
elif args.model_type == 'regp':
logger.info('Building a REGRESSION model with POOLING')
model = Sequential()
model.add(Embedding(args.vocab_size, args.emb_dim, mask_zero=True))
if args.cnn_dim > 0:
model.add(
Conv1DWithMasking(nb_filter=args.cnn_dim,
filter_length=args.cnn_window_size,
border_mode=cnn_border_mode,
subsample_length=1))
if args.rnn_dim > 0:
model.add(
RNN(args.rnn_dim,
return_sequences=True,
dropout=dropout_W,
recurrent_dropout=dropout_U))
if args.dropout_prob > 0:
model.add(Dropout(args.dropout_prob))
if args.aggregation == 'mot':
model.add(MeanOverTime(mask_zero=True))
elif args.aggregation.startswith('att'):
model.add(
Attention(op=args.aggregation,
activation='tanh',
init_stdev=0.01))
model.add(Dense(num_outputs))
if not args.skip_init_bias:
bias_value = (np.log(initial_mean_value) -
np.log(1 - initial_mean_value)).astype(K.floatx())
model.layers[-1].bias = bias_value
model.add(Activation('sigmoid'))
model.emb_index = 0
elif args.model_type == 'breg':
logger.info('Building a BIDIRECTIONAL REGRESSION model')
from keras.layers import Dense, Dropout, Embedding, LSTM, Input, merge
model = Sequential()
sequence = Input(shape=(overal_maxlen, ), dtype='int32')
output = Embedding(args.vocab_size, args.emb_dim,
mask_zero=True)(sequence)
if args.cnn_dim > 0:
output = Conv1DWithMasking(nb_filter=args.cnn_dim,
filter_length=args.cnn_window_size,
border_mode=cnn_border_mode,
subsample_length=1)(output)
if args.rnn_dim > 0:
forwards = RNN(args.rnn_dim,
return_sequences=False,
dropout=dropout_W,
recurrent_dropout=dropout_U)(output)
backwards = RNN(args.rnn_dim,
return_sequences=False,
dropout=dropout_W,
recurrent_dropout=dropout_U,
go_backwards=True)(output)
if args.dropout_prob > 0:
forwards = Dropout(args.dropout_prob)(forwards)
backwards = Dropout(args.dropout_prob)(backwards)
merged = merge([forwards, backwards], mode='concat', concat_axis=-1)
densed = Dense(num_outputs)(merged)
if not args.skip_init_bias:
raise NotImplementedError
score = Activation('sigmoid')(densed)
model = Model(input=sequence, output=score)
model.emb_index = 1
elif args.model_type == 'bregp':
logger.info('Building a BIDIRECTIONAL REGRESSION model with POOLING')
from keras.layers import Dense, Dropout, Embedding, LSTM, Input, merge
model = Sequential()
sequence = Input(shape=(overal_maxlen, ), dtype='int32')
output = Embedding(args.vocab_size, args.emb_dim,
mask_zero=True)(sequence)
if args.cnn_dim > 0:
output = Conv1DWithMasking(nb_filter=args.cnn_dim,
filter_length=args.cnn_window_size,
border_mode=cnn_border_mode,
subsample_length=1)(output)
if args.rnn_dim > 0:
forwards = RNN(args.rnn_dim,
return_sequences=True,
dropout=dropout_W,
recurrent_dropout=dropout_U)(output)
backwards = RNN(args.rnn_dim,
return_sequences=True,
dropout=dropout_W,
recurrent_dropout=dropout_U,
go_backwards=True)(output)
if args.dropout_prob > 0:
forwards = Dropout(args.dropout_prob)(forwards)
backwards = Dropout(args.dropout_prob)(backwards)
forwards_mean = MeanOverTime(mask_zero=True)(forwards)
backwards_mean = MeanOverTime(mask_zero=True)(backwards)
merged = merge([forwards_mean, backwards_mean],
mode='concat',
concat_axis=-1)
densed = Dense(num_outputs)(merged)
if not args.skip_init_bias:
raise NotImplementedError
score = Activation('sigmoid')(densed)
model = Model(input=sequence, output=score)
model.emb_index = 1
logger.info(' Done')
###############################################################################################################################
## Initialize embeddings if requested
#
if args.emb_path:
from w2vEmbReader import W2VEmbReader as EmbReader
logger.info('Initializing lookup table')
emb_reader = EmbReader(args.emb_path, emb_dim=args.emb_dim)
model.layers[model.emb_index].set_weights(
emb_reader.get_emb_matrix_given_vocab(
vocab, model.layers[model.emb_index].get_weights()))
logger.info(' Done')
return model
#필터가 10개이고, 필터 사이즈가 (3,3), Stride가 1인 Conv Layer를 추가합니다.
model.add(
Convolution2D(input_shape=(28, 28, 1),
filters=10,
kernel_size=(3, 3),
strides=1,
padding='same'))
#Activation Function 으로 Relu를 사용하였습니다.
model.add(Activation('relu'))
#필터가 20개이고, 필터 사이즈가 (3,3), Stride가 1인 Conv Layer를 추가합니다.
model.add(Convolution2D(filters=20, kernel_size=(3, 3), strides=1))
model.add(Activation('relu'))
#이미지의 차원을 줄이기 위해서 Pooling Layer를 넣었습니다.
model.add(MaxPooling2D(pool_size=(2, 2)))
#0.25의 확률의 Dropout 시행
model.add(Dropout(0.25))
#FC Layer에 넣기 위해서 2차원 배열로 되어 있는 이미지 인풋을 Flattern 해 줍니다.
model.add(Flatten())
#Output Node 가 128개인 FC Layer
model.add(Dense(128))
model.add(Activation('relu'))
#Output Node 가 10개인 FC Layer
model.add(Dense(10))
#Output 벡터의 합을 1로 만들어주기 위해서 softmax 함수를 사용했습니다.
model.add(Activation('softmax'))
# #Keras Documentation 을 참고하여 Adam Optimizer 를 이용하였습니다.
model.compile(loss='mean_squared_error',
optimizer='sgd',
metrics=['accuracy'])
#CNN 모델을 학습시킵니다. 학습 결과는 History object에 저장됩니다.
history = model.fit(x_train, y_train, epochs=30, batch_size=100, verbose=1)
init='glorot_normal',
border_mode='same'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
# model.add(Dropout(.5))
model.add(Convolution2D(32, 5, 5, init='glorot_normal', border_mode='same'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
# model.add(Dropout(.5))
model.add(Convolution2D(32, 5, 5, init='glorot_normal', border_mode='same'))
model.add(Activation('relu'))
# model.add(Dropout(.5))
model.add(Flatten())
model.add(Dense(256))
model.add(Activation('relu'))
model.add(Dropout(.5))
model.add(Dense(1))
model.add(Activation('sigmoid'))
adam = Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
model.compile(loss='mean_squared_error', optimizer=adam)
from cninit import init_model
init_model(model, X_train, Y_train)
h = model.fit(X_train,
Y_train,
batch_size=32,
nb_epoch=200,
show_accuracy=True,
validation_data=(X_valid, Y_valid))
def create_model(self, n_dim, r):
# load inputs
X, _, _ = self.inputs
K.set_session(self.sess)
with tf.name_scope('generator'):
x = X
L = self.layers
# dim/layer: 4096, 2048, 1024, 512, 256, 128, 64, 32,
# n_filters = [ 64, 128, 256, 384, 384, 384, 384, 384]
n_filters = [ 128, 256, 512, 512, 512, 512, 512, 512]
# n_filters = [ 256, 512, 512, 512, 512, 1024, 1024, 1024]
# n_filtersizes = [129, 65, 33, 17, 9, 9, 9, 9]
# n_filtersizes = [31, 31, 31, 31, 31, 31, 31, 31]
n_filtersizes = [65, 33, 17, 9, 9, 9, 9, 9, 9]
downsampling_l = []
print('building model...')
# downsampling layers
for l, nf, fs in zip(list(range(L)), n_filters, n_filtersizes):
with tf.name_scope('downsc_conv%d' % l):
x = (Convolution1D(nb_filter=nf, filter_length=fs,
activation=None, border_mode='same', init=orthogonal_init,
subsample_length=2))(x)
# if l > 0: x = BatchNormalization(mode=2)(x)
x = LeakyReLU(0.2)(x)
print('D-Block: ', x.get_shape())
downsampling_l.append(x)
# bottleneck layer
with tf.name_scope('bottleneck_conv'):
x = (Convolution1D(nb_filter=n_filters[-1], filter_length=n_filtersizes[-1],
activation=None, border_mode='same', init=orthogonal_init,
subsample_length=2))(x)
x = Dropout(p=0.5)(x)
# x = BatchNormalization(mode=2)(x)
x = LeakyReLU(0.2)(x)
# upsampling layers
for l, nf, fs, l_in in reversed(list(zip(list(range(L)), n_filters, n_filtersizes, downsampling_l))):
with tf.name_scope('upsc_conv%d' % l):
# (-1, n/2, 2f)
x = (Convolution1D(nb_filter=2*nf, filter_length=fs,
activation=None, border_mode='same', init=orthogonal_init))(x)
# x = BatchNormalization(mode=2)(x)
x = Dropout(p=0.5)(x)
x = Activation('relu')(x)
# (-1, n, f)
x = SubPixel1D(x, r=2)
# (-1, n, 2f)
x = merge([x, l_in], mode='concat', concat_axis=-1)
print('U-Block: ', x.get_shape())
# final conv layer
with tf.name_scope('lastconv'):
x = Convolution1D(nb_filter=2, filter_length=9,
activation=None, border_mode='same', init=normal_init)(x)
x = SubPixel1D(x, r=2)
print(x.get_shape())
g = merge([x, X], mode='sum')
return g
def creat_Model_BiLSTM_CRF(charvocabsize, wordvocabsize, targetvocabsize,
char_W, word_W,
input_seq_lenth,
output_seq_lenth,
hidden_dim, emd_dim):
posi_input = Input(shape=(input_seq_lenth, 4), dtype='float32')
char_input = Input(shape=(input_seq_lenth,), dtype='int32')
char_embedding = Embedding(input_dim=charvocabsize + 1,
output_dim=emd_dim,
input_length=input_seq_lenth,
mask_zero=False,
trainable=True,
weights=[char_W])(char_input)
char_embedding_dropout = Dropout(0.5)(char_embedding)
# cnn2_word = Conv1D(100, 2, activation='relu', strides=1, padding='same')(char_embedding)
# cnn3_word = Conv1D(100, 3, activation='relu', strides=1, padding='same')(char_embedding)
# cnn4_word = Conv1D(50, 4, activation='relu', strides=1, padding='same')(char_embedding)
# cnn5_word = Conv1D(50, 5, activation='relu', strides=1, padding='same')(char_embedding)
#
# cnn_word = concatenate([char_embedding, cnn2_word, cnn3_word, cnn4_word, cnn5_word], axis=-1)
# char_embedding_dropout = Dropout(0.5)(cnn_word)
# word_input = Input(shape=(input_seq_lenth,), dtype='int32')
# word_embedding = Embedding(input_dim=wordvocabsize + 1,
# output_dim=emd_dim,
# input_length=input_seq_lenth,
# mask_zero=False,
# trainable=True,
# weights=[word_W])(word_input)
# word_embedding_dropout = Dropout(0.5)(word_embedding)
#
# cnn3_word = Conv1D(50, 3, activation='relu', strides=1, padding='valid')(word_embedding_dropout)
# cnn3_word_pool = GlobalMaxPooling1D()(cnn3_word)
# cnn3_word_repeat = RepeatVector(input_seq_lenth)(cnn3_word_pool)
# embedding = concatenate([char_embedding_dropout, cnn3_word_repeat], axis=-1)
# embedding_dropout = Dropout(0.5)(embedding)
BiLSTM = Bidirectional(LSTM(200, return_sequences=True), merge_mode = 'concat')(char_embedding_dropout)
# BiLSTM = Bidirectional(LSTM(hidden_dim, return_sequences=True))(word_embedding_dropout)
BiLSTM = Dropout(0.5)(BiLSTM)
# BiLSTM_dropout = BatchNormalization(axis=1)(BiLSTM)
attention = Dense(1, activation='tanh')(char_embedding_dropout)
attention = Flatten()(attention)
attention = Activation('softmax')(attention)
attention = RepeatVector(400)(attention)
attention = Permute([2, 1])(attention)
# apply the attention
representation = multiply([BiLSTM, attention])
# representation = Lambda(lambda xin: K.sum(xin, axis=1))(representation)
BiLSTM_attention = Dropout(0.5)(representation)
# embedding_dropout = Dropout(0.5)(embedding)
cnn2 = Conv1D(100, 2, activation='relu', strides=1, padding='same')(char_embedding_dropout)
cnn3 = Conv1D(100, 3, activation='relu', strides=1, padding='same')(char_embedding_dropout)
cnn4 = Conv1D(100, 4, activation='relu', strides=1, padding='same')(char_embedding_dropout)
cnn5 = Conv1D(100, 5, activation='relu', strides=1, padding='same')(char_embedding_dropout)
features = concatenate([BiLSTM_attention, cnn2, cnn3, cnn4, cnn5], axis=-1)
features_dropout = Dropout(0.5)(features)
# features_dropout = BatchNormalization(axis=1)(features_dropout)
TimeD = TimeDistributed(Dense(targetvocabsize+1))(features_dropout)
TimeD = Dropout(0.2)(TimeD)
# model = Activation('softmax')(TimeD)
crflayer = CRF(targetvocabsize+1, sparse_target=False)
model = crflayer(TimeD)#0.8746633147782367
# # model = crf(BiLSTM_dropout)#0.870420501714492
Models = Model([char_input], model)
# Models.compile(loss=loss, optimizer='adam', metrics=['acc'])
Models.compile(loss=crflayer.loss_function, optimizer='adam', metrics=[crflayer.accuracy])
# Models.compile(loss=crf.loss_function, optimizer=optimizers.RMSprop(lr=0.01), metrics=[crf.accuracy])
return Models
# ПЕРЕВОД В Ч/Б ФОРМАТ
X_train = X_train.reshape(X_train.shape[0], 20, 20, 1)
X_test = X_test.reshape(X_test.shape[0], 20, 20, 1)
X_train = X_train.astype("float32")
X_test = X_test.astype("float32")
X_train /= 255
X_test /= 255
# ПОСТРОЕНИЕ МОДЕЛИ
model = Sequential() # Линейное соединение слоев
model.add(Conv2D(20, (3, 3), padding='same', input_shape=(IMG_ROWS, IMG_COLS, 1))) # Сверточный слой
model.add(Activation('relu')) # Функция активации ReLu
model.add(Conv2D(20, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2))) # Пулинговый слой
model.add(Dropout(0.25)) # Dropout слой
model.add(Conv2D(40, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(Conv2D(40, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Dense(512)) # Полносвязный слой
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Flatten()) # Сглаживание ввода (изменение формы данных в обычный массив)
model.add(Dense(NB_CLASSES)) # Выходной слой
model.add(Activation('softmax')) # Функция активации SoftMax
model.summary() # Вывод краткого представления модели
model.compile(loss='categorical_crossentropy', optimizer=OPTIM, metrics=['accuracy']) # Конфигурация модели
def build_model(shape):
model = Sequential()
model.add(
Convolution2D(32,
kernel_size=(3, 3),
input_shape=shape,
activation="relu",
padding='same',
kernel_initializer='glorot_normal'))
model.add(
Convolution2D(32,
kernel_size=(3, 3),
activation="relu",
padding='same',
kernel_initializer='glorot_normal'))
model.add(MaxPooling2D((2, 2)))
model.add(Dropout(0.1))
model.add(BatchNormalization())
model.add(
Convolution2D(64,
kernel_size=(3, 3),
activation="relu",
padding='same',
kernel_initializer='glorot_normal'))
model.add(
Convolution2D(64,
kernel_size=(3, 3),
activation="relu",
padding='same',
kernel_initializer='glorot_normal'))
model.add(
Convolution2D(64,
kernel_size=(3, 3),
activation="relu",
padding='same',
kernel_initializer='glorot_normal'))
model.add(MaxPooling2D((2, 2)))
model.add(Dropout(0.3))
model.add(BatchNormalization())
model.add(
Convolution2D(128,
kernel_size=(3, 3),
activation="relu",
padding='same',
kernel_initializer='glorot_normal'))
model.add(
Convolution2D(128,
kernel_size=(3, 3),
activation="relu",
padding='same',
kernel_initializer='glorot_normal'))
model.add(MaxPooling2D((2, 2)))
model.add(Dropout(0.4))
model.add(BatchNormalization())
model.add(Flatten())
model.add(
Dense(1024, activation="relu", kernel_initializer='glorot_normal'))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(
Dense(1024, activation="relu", kernel_initializer='glorot_normal'))
model.add(Dropout(0.5))
model.add(
Dense(7, activation="softmax", kernel_initializer='glorot_normal'))
return model
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-num_hidden_units', type=int, default=1024)
parser.add_argument('-num_hidden_layers', type=int, default=3)
parser.add_argument('-dropout', type=float, default=0.5)
parser.add_argument('-activation', type=str, default='tanh')
parser.add_argument('-language_only', type=bool, default= False)
args = parser.parse_args()
questions_train = open('../data/preprocessed/questions_train2014.txt', 'r').read().decode('utf8').splitlines()
answers_train = open('../data/preprocessed/answers_train2014.txt', 'r').read().decode('utf8').splitlines()
images_train = open('../data/preprocessed/images_train2014.txt', 'r').read().decode('utf8').splitlines()
vgg_model_path = '../features/coco/vgg_feats.mat'
maxAnswers = 1000
questions_train, answers_train, images_train = selectFrequentAnswers(questions_train,answers_train,images_train, maxAnswers)
#encode the remaining answers
labelencoder = preprocessing.LabelEncoder()
labelencoder.fit(answers_train)
nb_classes = len(list(labelencoder.classes_))
joblib.dump(labelencoder,'../models/labelencoder.pkl')
features_struct = scipy.io.loadmat(vgg_model_path)
VGGfeatures = features_struct['feats']
print 'loaded vgg features'
image_ids = open('../features/coco_vgg_IDMap.txt').read().splitlines()
id_map = {}
for ids in image_ids:
id_split = ids.split()
id_map[id_split[0]] = int(id_split[1])
nlp = English()
print 'loaded word2vec features...'
img_dim = 4096
word_vec_dim = 300
model = Sequential()
if args.language_only:
model.add(Dense(args.num_hidden_units, input_dim=word_vec_dim, init='uniform'))
else:
model.add(Dense(args.num_hidden_units, input_dim=img_dim+word_vec_dim, init='uniform'))
model.add(Activation(args.activation))
if args.dropout>0:
model.add(Dropout(args.dropout))
for i in xrange(args.num_hidden_layers-1):
model.add(Dense(args.num_hidden_units, init='uniform'))
model.add(Activation(args.activation))
if args.dropout>0:
model.add(Dropout(args.dropout))
model.add(Dense(nb_classes, init='uniform'))
model.add(Activation('softmax'))
json_string = model.to_json()
if args.language_only:
model_file_name = '../models/mlp_language_only_num_hidden_units_' + str(args.num_hidden_units) + '_num_hidden_layers_' + str(args.num_hidden_layers)
else:
model_file_name = '../models/mlp_num_hidden_units_' + str(args.num_hidden_units) + '_num_hidden_layers_' + str(args.num_hidden_layers)
open(model_file_name + '.json', 'w').write(json_string)
print 'Compiling model...'
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
print 'Compilation done...'
batchSize = 128
print 'Training started...'
numEpochs = 100
save_epoch_interval = 10
for k in xrange(numEpochs):
#shuffle the data points before going through them
index_shuf = range(len(questions_train))
shuffle(index_shuf)
questions_train = [questions_train[i] for i in index_shuf]
answers_train = [answers_train[i] for i in index_shuf]
images_train = [images_train[i] for i in index_shuf]
progbar = generic_utils.Progbar(len(questions_train))
for qu_batch,an_batch,im_batch in zip(grouper(questions_train, batchSize, fillvalue=questions_train[0]), grouper(answers_train, batchSize, fillvalue=answers_train[0]), grouper(images_train, batchSize, fillvalue=images_train[0])):
X_q_batch = get_questions_matrix_sum(qu_batch, nlp)
if args.language_only:
X_batch = X_q_batch
else:
X_i_batch = get_images_matrix(im_batch, id_map, VGGfeatures)
X_batch = np.hstack((X_q_batch, X_i_batch))
Y_batch = get_answers_matrix(an_batch, labelencoder)
loss = model.train_on_batch(X_batch, Y_batch)
progbar.add(batchSize, values=[("train loss", loss)])
#print type(loss)
if k%save_epoch_interval == 0:
model.save_weights(model_file_name + '_epoch_{:02d}'.format(k))
model.save_weights(model_file_name + '_epoch_{:02d}.hdf5'.format(k))
from keras import backend as K
K.set_session(sess)
''' Import keras to build a DL model '''
from keras.models import Sequential
from keras.layers.core import Dense, Activation, Dropout
from keras import regularizers
print('Building a model whose optimizer=adam, activation function=softplus')
model_adam = Sequential()
model_adam.add(
Dense(128,
input_dim=X_train[0].shape[0],
kernel_regularizer=regularizers.l2(0.1)))
model_adam.add(Activation('relu'))
model_adam.add(Dropout(0.5))
model_adam.add(Dense(256))
model_adam.add(Activation('relu'))
model_adam.add(Dense(1))
model_adam.add(Activation('sigmoid'))
''' Setting optimizer as Adam '''
from keras.optimizers import SGD, Adam, RMSprop, Adagrad
model_adam.compile(loss='mean_squared_error',
optimizer='Adam',
metrics=['accuracy'])
with tf.device('/gpu:0'):
'''Fit models and use validation_split=0.1 '''
history_adam = model_adam.fit(X_train,
Y_train,
batch_size=batch_size,
# 出力はグレースケール1チャンネル
output_channel_count = 1
# 一番初めのConvolutionフィルタ枚数は64
first_layer_filter_count = 64
network = UNet(input_channel_count, output_channel_count, first_layer_filter_count)
model = network.get_model()
"""
inputs = Input((IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS))
c1 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(inputs)
c1 = Dropout(0.1)(c1)
c1 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c1)
p1 = MaxPooling2D((2, 2))(c1)
c2 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(p1)
c2 = Dropout(0.1)(c2)
c2 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c2)
def fit():
batch_size = 128
nb_epoch = 1
chunk_size = 15000
# input image dimensions
img_rows, img_cols = 28, 28
# number of convolutional filters to use
nb_filters = 32
# size of pooling area for max pooling
nb_pool = 2
# convolution kernel size
nb_conv = 3
#load all the labels for the train and test sets
y_train = np.loadtxt('labels_train.csv')
y_test = np.loadtxt('labels_test.csv')
fnames_train = [
'train/train' + str(i) + '.png' for i in xrange(len(y_train))
]
fnames_test = ['test/test' + str(i) + '.png' for i in xrange(len(y_test))]
nb_classes = len(np.unique(y_train))
# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train.astype(int), nb_classes)
Y_test = np_utils.to_categorical(y_test.astype(int), nb_classes)
model = Sequential()
model.add(
Convolution2D(nb_filters,
nb_conv,
nb_conv,
border_mode='valid',
input_shape=(1, img_rows, img_cols)))
model.add(Activation('relu'))
model.add(Convolution2D(nb_filters, nb_conv, nb_conv))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adadelta',
metrics=['accuracy'])
#model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch,
# verbose=1, validation_data=(X_test, Y_test))
model.fit_generator(myGenerator(Y_train, chunk_size, batch_size,
fnames_train),
samples_per_epoch=y_train.shape[0],
nb_epoch=nb_epoch,
verbose=2,
callbacks=[],
validation_data=None,
class_weight=None) # show_accuracy=True, nb_worker=1
'''
i = 0
pred = np.zeros((len(fnames_test), Y_train.shape[1]))
for X, y in myGenerator(Y_test, chunk_size, batch_size, fnames_test):
print('chunk '+str(i))
pred[i*chunk_size:(i+1)*chunk_size, :] = model.predict(X, samples_per_epoch = y_train.shape[0], nb_epoch = nb_epoch, verbose=2,callbacks=[], validation_data=None, class_weight=None) # show_accuracy=True, nb_worker=1
i += 1
print(pred[0:10])
'''
pred = model.predict_generator(
myGenerator(None, chunk_size, 100, fnames_test),
len(fnames_test)) # show_accuracy=True, nb_worker=1
#score = model.evaluate(X_test, Y_test, verbose=0)
#print('Test score:', score[0])
#print('Test accuracy:', score[1])
print('Test accuracy:',
np.mean(np.argmax(pred, axis=1) == np.argmax(Y_test, axis=1)))
return pred, Y_test
# 规范化
X_train /= 255
X_test /= 255
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')
# convert class vectors to binary class matrices
# 将分类向量转化为二元分类矩阵(one hot)
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)
# 模型定义
model = Sequential() # 序列化模型
model.add(Dense(512, input_shape=(784, ))) # 全连接层，784维输入，512维输出
model.add(Activation('relu')) # relu 激活函数
model.add(Dropout(0.2)) # Dropout 舍弃部分因曾结点的权重，防止过拟合
model.add(Dense(512)) # 由一个全连接层，512维输出
model.add(Activation('relu')) # relu 激活函数
model.add(Dropout(0.2)) # Dropout 舍弃部分因曾结点的权重，防止过拟合
model.add(Dense(10)) # 由一个全连接层，10维输出
model.add(Activation('softmax')) # softmax 激活函数用于分类
model.summary() # 模型概述
# 定义模型的损失函数，优化器，评估矩阵
model.compile(loss='categorical_crossentropy',
optimizer=RMSprop(),
metrics=['accuracy'])
# 训练，迭代 nb_epoch 次，使用验证集验证
history = model.fit(X_train,
DX_embed = MaskedPooling()(DX_embed)
input_PR = Input(shape = (len(PRs),))
PR_embed = Embedding(input_dim=len(code_cat), output_dim=code_embed_dim, mask_zero=True, embeddings_initializer=embed_initializer,
name='PR_embed')(input_PR)
if model_name=='setsum_nn' or 'setsum_lr':
PR_embed = MaskedDense(md_width, activation='relu')(PR_embed)
PR_embed = MaskedSum()(PR_embed)
elif model_name=='embed_sum':
PR_embed = MaskedSum()(PR_embed)
elif model_name=='embed_pool':
PR_embed = MaskedPooling()(PR_embed)
input_other = Input(shape=(other_mat_train.shape[1], ))
merged = Concatenate(axis=1)([DX_embed, PR_embed, input_other])
if model_name=='setsum_nn':
merged = Dense(fc_width, activation='relu')(merged)
merged = Dropout(dropout)(merged)
prediction = Dense(2, activation='softmax')(merged)
model = Model(inputs=[input_DX, input_PR, input_other], outputs=prediction)
for l in model.layers:
if l.name=='DX_embed' or l.name=='PR_embed':
l.trainable = False
adam = Adam(lr=lr1)
model.compile(optimizer=adam, loss='categorical_crossentropy')
auccheckpoint = AUCCheckPoint(filepath=model_path+'embeding_nn_sub_temp1_'+str(job_index)+'.h5', validation_y=Y_val[:, 1],
validation_x=[DX_mat_val, PR_mat_val, other_mat_val])
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=10, min_lr=K.epsilon())
earlystop = EarlyStopping(monitor='val_loss', patience=30)
class_weight = {0:1., 1:minor_class_weight}
def creat_Model_BiLSTM_CNN_hybrid(sourcevocabsize, targetvocabsize, poslabelvobsize, entlabelvobsize, source_W, poslabel_W,
entlabel_W, input_seq_lenth,
output_seq_lenth,
hidden_dim, emd_dim, loss='categorical_crossentropy', optimizer='rmsprop'):
# BiLSTM(w,p)+CNN(w,p)--timestep of LSTM concat CNN-GlobalMaxPool--softmax
# BiLSTM(w,p)+CNN(w,p)--timestep of LSTM multiply CNN--softmax
word_input = Input(shape=(input_seq_lenth,), dtype='int32')
posl_input = Input(shape=(input_seq_lenth,), dtype='int32')
entl_input = Input(shape=(input_seq_lenth,), dtype='int32')
l_A_embedding = Embedding(input_dim=sourcevocabsize + 1, output_dim=emd_dim, input_length=input_seq_lenth,
mask_zero=True, trainable=True, weights=[source_W])(word_input)
l_A_embedding_CNN = Embedding(input_dim=sourcevocabsize + 1, output_dim=emd_dim, input_length=input_seq_lenth,
mask_zero=False, trainable=True, weights=[source_W])(word_input)
poslable_embeding = Embedding(input_dim=poslabelvobsize + 1, output_dim=poslabelvobsize + 1, input_length=input_seq_lenth,
mask_zero=True, trainable=True, weights=[poslabel_W])(posl_input)
poslable_embeding_CNN = Embedding(input_dim=poslabelvobsize + 1, output_dim=poslabelvobsize + 1, input_length=input_seq_lenth,
mask_zero=False, trainable=True, weights=[poslabel_W])(posl_input)
entlable_embeding = Embedding(input_dim=entlabelvobsize + 1, output_dim=entlabelvobsize + 1, input_length=input_seq_lenth,
mask_zero=True, trainable=True, weights=[entlabel_W])(entl_input)
entlable_embeding_CNN = Embedding(input_dim=entlabelvobsize + 1, output_dim=entlabelvobsize + 1, input_length=input_seq_lenth,
mask_zero=False, trainable=True, weights=[entlabel_W])(entl_input)
concat_input = concatenate([l_A_embedding, poslable_embeding, entlable_embeding], axis=-1)
concat_input = Dropout(0.3)(concat_input)
concat_input_CNN1 = concatenate([l_A_embedding_CNN, poslable_embeding_CNN, entlable_embeding_CNN], axis=-1)
concat_input_CNN1 = Dropout(0.3)(concat_input_CNN1)
# concat_input_CNN2 = concatenate([l_A_embedding_CNN, poslable_embeding_CNN, entlable_embeding_CNN], axis=-1)
# concat_input_CNN2 = Dropout(0.3)(concat_input_CNN2)
#
# concat_input_CNN3 = concatenate([l_A_embedding_CNN, poslable_embeding_CNN, entlable_embeding_CNN], axis=-1)
# concat_input_CNN3 = Dropout(0.3)(concat_input_CNN3)
#
# BiLSTM = Bidirectional(LSTM(hidden_dim, return_sequences=True, dropout=0.1), merge_mode='ave')(concat_input )
# cnn1 = Conv1D(hidden_dim, 3, activation='relu', strides=1, padding='same')(concat_input_CNN1)
#
# concat_LC1 = multiply([BiLSTM, cnn1])
# concat_LC1 = Dropout(0.2)(concat_LC1)
#
# cnn2 = Conv1D(hidden_dim, 3, activation='relu', strides=1, padding='same')(concat_input_CNN2)
# maxpool = GlobalMaxPooling1D()(cnn2)
# repeat_maxpool = RepeatVector(input_seq_lenth)(maxpool)
#
#
# concat_LC2 = concatenate([concat_LC1, repeat_maxpool], axis=-1)
# concat_LC2 = Dropout(0.2)(concat_LC2)
BiLSTM = Bidirectional(LSTM(hidden_dim, return_sequences=True, dropout=0.1), merge_mode='concat')(concat_input)
cnn_f2 = Conv1D(100, 2, activation='relu', strides=1, padding='same')(concat_input_CNN1)
maxpool_f2 = GlobalMaxPooling1D()(cnn_f2)
repeat_maxpool_f2 = RepeatVector(input_seq_lenth)(maxpool_f2)
# repeat_maxpool_f2 = Dropout(0.1)(repeat_maxpool_f2)
cnn_f3 = Conv1D(200, 3, activation='relu', strides=1, padding='same')(concat_input_CNN1)
maxpool_f3 = GlobalMaxPooling1D()(cnn_f3)
repeat_maxpool_f3 = RepeatVector(input_seq_lenth)(maxpool_f3)
# repeat_maxpool_f3 = Dropout(0.1)(repeat_maxpool_f3)
# cnn_f4 = Conv1D(100, 4, activation='relu', strides=1, padding='same')(concat_input_CNN3)
# maxpool_f4 = GlobalMaxPooling1D()(cnn_f4)
# repeat_maxpool_f4 = RepeatVector(input_seq_lenth)(maxpool_f4)
# repeat_maxpool_f4 = Dropout(0.1)(repeat_maxpool_f4)
concat_cnns =concatenate([repeat_maxpool_f2,repeat_maxpool_f3], axis=-1)
concat_cnns = Dropout(0.3)(concat_cnns)
concat_LC2 = concatenate([BiLSTM, concat_cnns], axis=-1)
concat_LC2 = Dropout(0.2)(concat_LC2)
TimeD = TimeDistributed(Dense(targetvocabsize + 1))(concat_LC2)
# model = Activation('softmax')(TimeD)
crf = CRF(targetvocabsize + 1, sparse_target=False)
model = crf(TimeD)
Models = Model([word_input, posl_input, entl_input], model)
# Models.compile(loss=loss, optimizer=optimizer, metrics=['acc'])
# Models.compile(loss=crf.loss_function, optimizer='adam', metrics=[crf.accuracy])
# Models.compile(loss=crf.loss_function, optimizer=optimizers.RMSprop(lr=0.1), metrics=[crf.accuracy])
Models.compile(loss=crf.loss_function, optimizer='adam', metrics=[crf.accuracy])
return Models
def __init__(self,
X_train,
y_train,
n_hidden,
n_epochs=40,
normalize=False,
tau=1.,
X_test=None,
y_test=None):
"""
Constructor for the class implementing a Bayesian neural network
trained with the probabilistic back propagation method.
@param X_train Matrix with the features for the training data.
@param y_train Vector with the target variables for the
training data.
@param n_hidden Vector with the number of neurons for each
hidden layer.
@param n_epochs Numer of epochs for which to train the
network. The recommended value 40 should be
enough.
@param normalize Whether to normalize the input features. This
is recommended unles the input vector is for
example formed by binary features (a
fingerprint). In that case we do not recommend
to normalize the features.
"""
# We normalize the training data to have zero mean and unit standard
# deviation in the training set if necessary
if normalize:
self.std_X_train = np.std(X_train, 0)
self.std_X_train[self.std_X_train == 0] = 1
self.mean_X_train = np.mean(X_train, 0)
else:
self.std_X_train = np.ones(X_train.shape[1])
self.mean_X_train = np.zeros(X_train.shape[1])
X_train = (X_train - np.full(X_train.shape, self.mean_X_train)) / \
np.full(X_train.shape, self.std_X_train)
self.mean_y_train = np.mean(y_train)
self.std_y_train = np.std(y_train)
y_train_normalized = (y_train - self.mean_y_train) / self.std_y_train
y_train_normalized = np.array(y_train_normalized, ndmin=2).T
# We construct the network
N = X_train.shape[0]
dropout = 0.01
batch_size = 32
lengthscale = 1e-2
reg = lambda: l2(lengthscale**2 * (1 - dropout) / (2. * N * tau))
model = Sequential()
model.add(Dropout(dropout))
model.add(Dense(X_train.shape[1], n_hidden[0], W_regularizer=reg()))
model.add(Activation('relu'))
for i in xrange(len(n_hidden) - 1):
model.add(Dropout(dropout))
model.add(Dense(n_hidden[i], n_hidden[i + 1], W_regularizer=reg()))
model.add(Activation('relu'))
model.add(Dropout(dropout))
model.add(
Dense(n_hidden[-1],
y_train_normalized.shape[1],
W_regularizer=reg()))
# In Keras we have:
# lr = self.lr * (1.0 / (1.0 + self.decay * self.iterations))
# for SGD
# optimiser = SGD(lr=lr, decay=SGD_decay, momentum=0.9, nesterov=False)
optimiser = 'adam'
model.compile(loss='mean_squared_error', optimizer=optimiser)
# We iterate the learning process
start_time = time.time()
X_test = np.array(X_test, ndmin=2)
y_test = np.array(y_test, ndmin=2).T
X_test = (X_test - np.full(X_test.shape, self.mean_X_train)) / \
np.full(X_test.shape, self.std_X_train)
# modeltest = ModelTest(X_test, y_test, test_every_X_epochs=500, verbose=0, T=10,
# loss='euclidean', mean_y_train=self.mean_y_train, std_y_train=self.std_y_train, tau=tau)
# this doesn't seem to reduce error variance:
# scheduler = LearningRateScheduler(lambda epoch: 0.005 / (epoch+1)**0.5)
model.fit(X_train,
y_train_normalized,
batch_size=batch_size,
nb_epoch=n_epochs,
verbose=1) #,
# callbacks=[modeltest]) #scheduler,
self.model = model
self.tau = tau
self.running_time = time.time() - start_time
def creat_Model_BiLSTM_CnnDecoder(charvocabsize, wordvocabsize, targetvocabsize,
char_W, word_W,
input_seq_lenth,
output_seq_lenth,
hidden_dim, emd_dim):
posi_input = Input(shape=(input_seq_lenth, 4), dtype='float32')
char_input = Input(shape=(input_seq_lenth,), dtype='int32')
char_embedding = Embedding(input_dim=charvocabsize + 1,
output_dim=emd_dim,
input_length=input_seq_lenth,
mask_zero=False,
trainable=True,
weights=[char_W])(char_input)
char_embedding_dropout = Dropout(0.5)(char_embedding)
word_input = Input(shape=(input_seq_lenth,), dtype='int32')
word_embedding = Embedding(input_dim=wordvocabsize + 1,
output_dim=emd_dim,
input_length=input_seq_lenth,
mask_zero=False,
trainable=True,
weights=[word_W])(word_input)
word_embedding_dropout = Dropout(0.5)(word_embedding)
embedding = concatenate([char_embedding_dropout, posi_input], axis=-1)
embedding = Dropout(0.5)(embedding)
BiLSTM = Bidirectional(LSTM(int(hidden_dim / 2), return_sequences=True), merge_mode='concat')(embedding)
BiLSTM = BatchNormalization()(BiLSTM)
# BiLSTM = Dropout(0.3)(BiLSTM)
# decodelayer1 = LSTM(50, return_sequences=False, go_backwards=True)(concat_LC_d)#!!!!!
# repeat_decodelayer1 = RepeatVector(input_seq_lenth)(decodelayer1)
# concat_decoder = concatenate([concat_LC_d, repeat_decodelayer1], axis=-1)#!!!!
# decodelayer2 = LSTM(hidden_dim, return_sequences=True)(concat_decoder)
# decodelayer = Dropout(0.5)(decodelayer2)
# decoderlayer1 = LSTM(50, return_sequences=True, go_backwards=False)(BiLSTM)
decoderlayer2 = Conv1D(50, 2, activation='relu', strides=1, padding='same')(BiLSTM)
decoderlayer3 = Conv1D(50, 3, activation='relu', strides=1, padding='same')(BiLSTM)
decoderlayer4 = Conv1D(50, 4, activation='relu', strides=1, padding='same')(BiLSTM)
decoderlayer5 = Conv1D(50, 5, activation='relu', strides=1, padding='same')(BiLSTM)
decodelayer = concatenate([decoderlayer2, decoderlayer3, decoderlayer4, decoderlayer5], axis=-1)
decodelayer = BatchNormalization()(decodelayer)
decodelayer = Dropout(0.5)(decodelayer)
TimeD = TimeDistributed(Dense(targetvocabsize + 1))(decodelayer)
# TimeD = Dropout(0.5)(TimeD)
model = Activation('softmax')(TimeD) # 0.8769744561783556
Models = Model([char_input, posi_input], model)
# Models.compile(loss=my_cross_entropy_Weight, optimizer='adam', metrics=['acc'])
Models.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc'])
# Models.compile(loss=loss, optimizer='adam', metrics=['acc'], sample_weight_mode="temporal")
# Models.compile(loss=loss, optimizer=optimizers.RMSprop(lr=0.01), metrics=['acc'])
# Models.compile(loss=crf.loss_function, optimizer='adam', metrics=[crf.accuracy])
# Models.compile(loss=crf.loss_function, optimizer=optimizers.RMSprop(lr=0.005), metrics=[crf.accuracy])
return Models
predict_features = pd.read_csv(predict_path +
'train_features.csv').astype(float)
dfs = [train_features, validate_features, predict_features]
for df in dfs:
df = df.applymap(lambda x: np.nan if x == -1. else x)
create_feature_map(
train_features.columns.tolist(),
'{0}_dl_{1}{2}'.format(model_path, exec_time, model_fmap_file))
print 'Keras Training'
model = Sequential()
model.add(Dense(60, input_dim=105, init='uniform', activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(60, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['binary_crossentropy'])
# Training
model.fit(validate_features.values,
validate_labels.values[:, 0],
nb_epoch=10,
batch_size=32,
verbose=1,
validation_data=(train_features.values, train_labels.values[:,
def creat_Model_BiLSTM_CNN_multiply(sourcevocabsize, targetvocabsize, source_W, input_seq_lenth,
output_seq_lenth,
hidden_dim, emd_dim, loss='categorical_crossentropy', optimizer='rmsprop'):
# BiLSTM(w,p)+CNN(w,p)--timestep of LSTM multiply CNN--softmax
word_input = Input(shape=(input_seq_lenth,), dtype='int32')
# word_embedding_RNN = Embedding(input_dim=sourcevocabsize + 1, output_dim=emd_dim, input_length=input_seq_lenth,
# mask_zero=True, trainable=True, weights=[source_W])(word_input)
# word_embedding_RNN = Dropout(0.5)(word_embedding_RNN)
word_embedding_CNN = Embedding(input_dim=sourcevocabsize + 1, output_dim=emd_dim, input_length=input_seq_lenth,
mask_zero=False, trainable=True, weights=[source_W])(word_input)
word_embedding_CNN = Dropout(0.5)(word_embedding_CNN)
BiLSTM = Bidirectional(LSTM(hidden_dim, return_sequences=True), merge_mode='ave')(word_embedding_CNN)
# cnn = Conv1D(input_seq_lenth, 3, activation='relu', strides=1, padding='same')(word_embedding)
# pool1D = AveragePooling1D(input_seq_lenth)(cnn)
# reshape = Permute((2,1))(pool1D)#0.8716606498194945
cnn = Conv1D(1, 3, activation='relu', strides=1, padding='same')(word_embedding_CNN)
# pool1D = AveragePooling1D(input_seq_lenth)(cnn)
# reshape = Permute((2,1))(pool1D)
# # feature = reshape
# #
# # for i in range(0,input_seq_lenth-1):
# # feature = concatenate([feature, reshape])
# #
# # repeat_lstm = RepeatVector(input_seq_lenth)(BiLSTM)
concat_LC =multiply([BiLSTM, cnn])
# concat_LC = BatchNormalization(axis=1)(concat_LC)
concat_LC_d = Dropout(0.5)(concat_LC)
decodelayer1 = LSTM(50, return_sequences=False, go_backwards=True)(concat_LC_d)
repeat_decodelayer1 = RepeatVector(input_seq_lenth)(decodelayer1)
concat_decoder = concatenate([concat_LC_d, repeat_decodelayer1], axis=-1)
decodelayer2 = LSTM(hidden_dim, return_sequences=True)(concat_decoder)
decodelayer = Dropout(0.5)(decodelayer2)
# decodelayer = LSTMDecoderCell(output_dim=hidden_dim, hidden_dim=hidden_dim)(concat_LC)
# decodelayer = LSTM(hidden_dim, return_sequences=True)(concat_LC)#0.8770848440899202
# decodelayer = Dropout(0.5)(decodelayer)
# TimeD = TimeDistributed(Dense(int(hidden_dim / 2)))(concat_LC)
TimeD = TimeDistributed(Dense(targetvocabsize + 1))(decodelayer)
# TimeD = Dropout(0.5)(TimeD)
# model = Activation('softmax')(TimeD)#0.8769744561783556
#
crf = CRF(targetvocabsize + 1, sparse_target=False)
model = crf(TimeD)
Models = Model(word_input, model)
# Models.compile(loss=loss, optimizer='adam', metrics=['acc'])
# Models.compile(loss=loss, optimizer=optimizers.RMSprop(lr=0.01), metrics=['acc'])
Models.compile(loss=crf.loss_function, optimizer='adam', metrics=[crf.accuracy])
# Models.compile(loss=crf.loss_function, optimizer=optimizers.RMSprop(lr=0.005), metrics=[crf.accuracy])
return Models
cvfold = cvfold + 1
X_train_resampled, X_test_resampled = X_all_resampled[
train], X_all_resampled[test]
y_train_resampled, y_test_resampled = y_all_binary[
train], y_all_binary[test]
#apply preprocessing
# X_train = data_preprocess_train(X_train)
# X_test = data_preprocess_test(X_test)
model = Sequential()
for i, input_dim in enumerate(([n_in] + n_hiddens)[:-1]):
model.add(Dense(n_hiddens[i], input_dim=input_dim))
model.add(BatchNormalization()) # バッチごとの白色化
model.add(Activation(activation)) # ReLU
model.add(Dropout(p_keep)) # Dropout
model.add(Dense(n_out))
model.add(Activation('softmax'))
model.compile(
loss='categorical_crossentropy', # 交差entropy
optimizer=SGD(lr=0.01),
metrics=['accuracy'])
model.fit(X_train_resampled,
y_train_resampled,
epochs=epochs,
batch_size=batch_size,
verbose=1)
scores = model.evaluate(X_test_resampled, y_test_resampled, verbose=0)
def trainGestureRNN(numLayers,
numNodesPerLayer,
useGRU,
batchSize,
numEpochs,
learningRate,
l1Reg,
l2Reg,
dropoutI,
dropoutH,
sequences,
classes,
trainRange,
valRange,
testRange,
numClasses,
numObservations,
numSequences,
numFeatures,
modelFile,
callbacks=None,
outDirectory='',
trainMode='continue'):
"""
Returns True if training was completed, False if interrupted.
"""
trainModes = ['continue', 'overwrite', 'skip']
if trainMode.lower() not in trainModes:
raise ValueError(
"Parameter 'trainMode' must be one of 'continue', 'overwrite', or 'skip'"
)
if dropoutI < 0 or dropoutH < 0 or l2Reg < 0 or l1Reg < 0:
raise ValueError('Regularization parameters must be non-negative.')
if outDirectory is not None and outDirectory != '':
outDirectory = outDirectory + '\\'
else:
outDirectory = ''
# initialize, compile, and train model
#finish preparing data
#class labels must be made into binary arrays
binaryClasses = np.zeros((numObservations, numSequences, numClasses))
# tell cost function which timesteps to ignore
sampleWeights = np.ones((numObservations, numSequences))
#eh...just use for loops
for i in range(numObservations):
for j in range(numSequences):
if classes[i, j] >= 0:
binaryClasses[i, j, classes[i, j]] = 1
else:
sampleWeights[i, j] = 0
sequences = sequences.transpose((1, 0, 2))
binaryClasses = binaryClasses.transpose((1, 0, 2))
sampleWeights = sampleWeights.T
trainData = [
sequences[trainRange, :, :], binaryClasses[trainRange, :, :],
sampleWeights[trainRange, :]
]
valData = [
sequences[valRange, :, :], binaryClasses[valRange, :, :],
sampleWeights[valRange, :]
]
testData = [
sequences[testRange, :, :], binaryClasses[testRange, :, :],
sampleWeights[testRange, :]
]
modelFile = outDirectory + 'Keras' + modelFile
weightsFile = modelFile + '_Weights'
completedEpochs = 0
if (trainMode
== 'overwrite') or (not os.path.isfile(modelFile + '.json')
or not os.path.isfile(weightsFile + '.h5')):
model = Sequential()
#add masking layer to indicate dummy timesteps
model.add(Masking(0, input_shape=(numObservations, numFeatures)))
if dropoutI:
model.add(Dropout(dropoutI))
for i in range(numLayers):
if useGRU:
model.add(
GRU(output_dim=numNodesPerLayer,
return_sequences=True,
W_regularizer=l2(l2Reg)))
else:
model.add(
LSTM(output_dim=numNodesPerLayer,
return_sequences=True,
W_regularizer=l2(l2Reg)))
if dropoutH:
model.add(Dropout(dropoutH))
model.add(
TimeDistributed(
Dense(output_dim=numClasses,
activation='softmax',
W_regularizer=l2(l2Reg))))
else:
model = model_from_json(open(modelFile + '.json', 'rb').read())
model.load_weights(weightsFile + '.h5')
#compile model and training objective function
sgd = SGD(lr=learningRate)
rms = RMSprop(lr=learningRate)
adagrad = Adagrad(lr=learningRate)
model.compile(loss='categorical_crossentropy',
optimizer=rms,
sample_weight_mode='temporal',
metrics=['accuracy'])
checkp = [ModelCheckpoint(weightsFile + '.h5', save_best_only=True)]
if callbacks is None:
callbacks = checkp
else:
callbacks += checkp
try:
if trainMode != 'skip':
completedEpochs = model.fit(x=trainData[0],
y=trainData[1],
sample_weight=trainData[2],
validation_data=valData,
batch_size=batchSize,
nb_epoch=numEpochs,
callbacks=callbacks,
verbose=2)
completedEpochs = len(completedEpochs.history['loss'])
except KeyboardInterrupt:
if (not queryUser('Training interrupted. Compute test statistics?')):
return 0, float('nan'), float('nan'), float('nan')
#retrieve the best weights based upon validation set loss
if os.path.isfile(weightsFile + '.h5'):
model.load_weights(weightsFile + '.h5')
scores = model.test_on_batch(x=testData[0],
y=testData[1],
sample_weight=testData[2])
predictedClasses = model.predict_classes(x=testData[0])
scores[1] = accuracy(classes[:, testRange].T, predictedClasses)
scores.append(balancedAccuracy(classes[:, testRange].T, predictedClasses))
scores.append(
weightedAccuracy(classes[:, testRange].T,
predictedClasses,
forgetFactor=0))
print(
"Test loss of %.5f\nFrame-wise accuracy of %.5f\nSequence-wise accuracy of %.5f\nFinal frame accuracy of %0.5f"
% (scores[0], scores[1], scores[2], scores[3]))
if trainMode != 'skip':
modelString = model.to_json()
open(modelFile + '.json', 'wb').write(modelString)
model.save_weights(weightsFile + '.h5', overwrite=True)
print('Model and weights saved to %s and %s.' %
(modelFile + '.json', weightsFile + '.h5'))
return completedEpochs, scores[0], scores[1], scores[2], scores[3]
