def model_factory(n_classes, n_dims):
print("Building model...")
lmbd1 = 0
lmbd2 = 0
model = Sequential()
model.add(Dense(n_dims, 1024, init='glorot_uniform',
W_regularizer=l1l2(lmbd1, lmbd2)))
model.add(PReLU((1024,)))
model.add(BatchNormalization((1024,)))
model.add(Dropout(0.5))
model.add(Dense(1024, 512, init='glorot_uniform',
W_regularizer=l1l2(lmbd1, lmbd2)))
model.add(PReLU((512,)))
model.add(BatchNormalization((512,)))
model.add(Dropout(0.5))
model.add(Dense(512, 256, init='glorot_uniform',
W_regularizer=l1l2(lmbd1, lmbd2)))
model.add(PReLU((256,)))
model.add(BatchNormalization((256,)))
model.add(Dropout(0.5))
model.add(Dense(256, n_classes, init='glorot_uniform'))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer="adam")
return model
def rnn_1(DIM = 0, HIDDEN_SIZE = 0 , DROPOUT = 0, LAYERS = 1, UNIT = 'lstm', MAX_Q = 0, MAX_A = 0, FINETUNE = False, embedding_weights = None): RNN = RNN_UNIT[UNIT] model = Graph() if FINETUNE: model.add_input(name = 'q', input_shape = (None,), dtype = 'int64') model.add_input(name = 'a', input_shape = (MAX_A,), dtype = 'int64') VOCAB = embedding_weights.shape[0] EMB_HIDDEN_SIZE = embedding_weights.shape[1] model.add_node(Embedding(VOCAB,EMB_HIDDEN_SIZE,mask_zero=True, weights=[embedding_weights], input_length = MAX_Q ), name = 'q_e', input = 'q') model.add_node(Embedding(VOCAB,EMB_HIDDEN_SIZE,mask_zero=True, weights=[embedding_weights], input_length = MAX_A ), name = 'a_e', input = 'a') prev_q = 'q_e' prev_a = 'a_e' else: model.add_input(name = 'q', input_shape = (None,DIM)) model.add_input(name = 'a', input_shape = (MAX_A,DIM)) prev_q = 'q' prev_a = 'a' for layer in xrange(LAYERS-1): model.add_node(RNN(HIDDEN_SIZE, return_sequences=True), name = 'q_rnn_' + str(layer), input = prev_q) prev_q = 'q_rnn_' + str(layer) model.add_node(RNN(HIDDEN_SIZE, return_sequences=False), name = 'q_rnn_' + str(LAYERS), input = prev_q) model.add_node(RepeatVector(MAX_A), input = 'q_rnn_' + str(LAYERS), name = 'q_rv') for layer in xrange(LAYERS-1): if layer == 0: model.add_node(RNN(HIDDEN_SIZE, return_sequences=True), name = 'a_rnn_0', inputs = ['q_rv' , prev_a], merge_mode = 'concat', concat_axis = -1) prev_a = 'a_rnn_0' continue model.add_node(RNN(HIDDEN_SIZE, return_sequences=True), name = 'a_rnn_' + str(layer), input = prev_a) prev_a = 'a_rnn_' + str(layer) if LAYERS == 1: model.add_node(RNN(HIDDEN_SIZE, return_sequences=False), name = 'a_rnn_0', inputs = ['q_rv' , prev_a], merge_mode = 'concat', concat_axis = -1) else: model.add_node(RNN(HIDDEN_SIZE, return_sequences=False),name = 'a_rnn_' + str(LAYERS-1), input = prev_a) model.add_node(Dense(HIDDEN_SIZE, activation = 'relu', W_regularizer = l1l2(l1 = 0.00001, l2 = 0.00001)), name = 'dense_0', input = 'a_rnn_' + str(LAYERS-1)) model.add_node(Dropout(DROPOUT), name = 'dropout_0', input = 'dense_0') prev_d = 'dropout_0' for layer in xrange(LAYERS-1): model.add_node(Dense(HIDDEN_SIZE, activation = 'relu', W_regularizer = l1l2(l1 = 0.00001, l2 = 0.00001)), name = 'dense_' + str(layer+1), input = prev_d) model.add_node(Dropout(DROPOUT), name = 'dropout_' + str(layer+1), input = 'dense_' + str(layer+1)) prev_d = 'dropout_' + str(layer+1) model.add_node(Dense(1, activation = 'sigmoid'), name = 'sigmoid', input = prev_d) model.add_output(name = 'o', input = 'sigmoid') return model
def make_network(
n_dims,
output_activation="linear",
hidden_activation="relu",
hidden_layer_sizes=None,
dropout_probability=0,
optimizer="adam",
init="glorot_normal",
l1_penalty=0,
l2_penalty=0):
if not hidden_layer_sizes:
# start with a layer larger than the input vector and its
# mask of missing values and then transform down to a layer
# which is smaller than the input -- a bottleneck to force
# generalization
hidden_layer_sizes = [
min(2000, 8 * n_dims),
min(500, 2 * n_dims),
int(np.ceil(0.5 * n_dims)),
]
print("Hidden layer sizes: %s" % (hidden_layer_sizes,))
nn = Sequential()
first_layer_size = hidden_layer_sizes[0]
nn.add(Dense(
first_layer_size,
input_dim=2 * n_dims,
activation=hidden_activation,
W_regularizer=l1l2(l1_penalty, l2_penalty),
init=init))
nn.add(Dropout(dropout_probability))
for layer_size in hidden_layer_sizes[1:]:
nn.add(Dense(
layer_size,
activation=hidden_activation,
W_regularizer=l1l2(l1_penalty, l2_penalty),
init=init))
nn.add(Dropout(dropout_probability))
nn.add(
Dense(
n_dims,
activation=output_activation,
W_regularizer=l1l2(l1_penalty, l2_penalty),
init=init))
loss_function = make_reconstruction_loss(
n_dims,
mask_indicates_missing_values=True)
nn.compile(optimizer=optimizer, loss=loss_function)
return nn
def model_generator(latent_dim,
nch=512,
dropout=0.5,
reg=lambda: l1l2(l1=1e-7, l2=1e-7)):
model = Sequential(name="decoder")
h = 5
model.add(
Dense(input_dim=latent_dim,
output_dim=nch * 4 * 4,
W_regularizer=reg()))
model.add(Reshape(dim_ordering_shape((nch, 4, 4))))
model.add(SpatialDropout2D(dropout))
model.add(LeakyReLU(0.2))
model.add(
Convolution2D(nch / 2, h, h, border_mode='same', W_regularizer=reg()))
model.add(SpatialDropout2D(dropout))
model.add(LeakyReLU(0.2))
model.add(UpSampling2D(size=(2, 2)))
model.add(
Convolution2D(nch / 2, h, h, border_mode='same', W_regularizer=reg()))
model.add(SpatialDropout2D(dropout))
model.add(LeakyReLU(0.2))
model.add(UpSampling2D(size=(2, 2)))
model.add(
Convolution2D(nch / 4, h, h, border_mode='same', W_regularizer=reg()))
model.add(SpatialDropout2D(dropout))
model.add(LeakyReLU(0.2))
model.add(UpSampling2D(size=(2, 2)))
model.add(Convolution2D(3, h, h, border_mode='same', W_regularizer=reg()))
model.add(Activation('sigmoid'))
return model
def model_discriminator():
nch = 256
h = 5
reg = lambda: l1l2(l1=1e-7, l2=1e-7)
c1 = Convolution2D(nch / 4, h, h, border_mode='same', W_regularizer=reg(),
input_shape=dim_ordering_shape((3, 32, 32)))
c2 = Convolution2D(nch / 2, h, h, border_mode='same', W_regularizer=reg())
c3 = Convolution2D(nch, h, h, border_mode='same', W_regularizer=reg())
c4 = Convolution2D(1, h, h, border_mode='same', W_regularizer=reg())
def m(dropout):
model = Sequential()
model.add(c1)
model.add(SpatialDropout2D(dropout))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(LeakyReLU(0.2))
model.add(c2)
model.add(SpatialDropout2D(dropout))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(LeakyReLU(0.2))
model.add(c3)
model.add(SpatialDropout2D(dropout))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(LeakyReLU(0.2))
model.add(c4)
model.add(AveragePooling2D(pool_size=(4, 4), border_mode='valid'))
model.add(Flatten())
model.add(Activation('sigmoid'))
return model
return m
def model_discriminator(latent_dim, input_shape, output_dim=1, hidden_dim=2048,
reg=lambda: l1l2(1e-7, 1e-7), batch_norm_mode=1, dropout=0.5):
z = Input((latent_dim,))
x = Input(input_shape, name="x")
h = merge([z, Flatten()(x)], mode='concat')
h1 = Dense(hidden_dim, name="discriminator_h1", W_regularizer=reg())
b1 = BatchNormalization(mode=batch_norm_mode)
h2 = Dense(hidden_dim, name="discriminator_h2", W_regularizer=reg())
b2 = BatchNormalization(mode=batch_norm_mode)
h3 = Dense(hidden_dim, name="discriminator_h3", W_regularizer=reg())
b3 = BatchNormalization(mode=batch_norm_mode)
y = Dense(output_dim, name="discriminator_y", activation="sigmoid", W_regularizer=reg())
# training model uses dropout
_h = h
_h = Dropout(dropout)(LeakyReLU(0.2)((b1(h1(_h)))))
_h = Dropout(dropout)(LeakyReLU(0.2)((b2(h2(_h)))))
_h = Dropout(dropout)(LeakyReLU(0.2)((b3(h3(_h)))))
ytrain = y(_h)
mtrain = Model([z, x], ytrain, name="discriminator_train")
# testing model does not use dropout
_h = h
_h = LeakyReLU(0.2)((b1(h1(_h))))
_h = LeakyReLU(0.2)((b2(h2(_h))))
_h = LeakyReLU(0.2)((b3(h3(_h))))
ytest = y(_h)
mtest = Model([z, x], ytest, name="discriminator_test")
return mtrain, mtest
def test_W_reg():
for reg in [regularizers.identity(), regularizers.l1(), regularizers.l2(), regularizers.l1l2()]:
model = create_model(weight_reg=reg)
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
model.fit(X_train, Y_train, batch_size=batch_size,
nb_epoch=nb_epoch, verbose=0)
model.evaluate(X_test[test_ids, :], Y_test[test_ids, :], verbose=0)
def make_network(n_dims,
output_activation="linear",
hidden_activation="relu",
hidden_layer_sizes=None,
dropout_probability=0,
optimizer="adam",
init="glorot_normal",
l1_penalty=0,
l2_penalty=0):
if not hidden_layer_sizes:
# start with a layer larger than the input vector and its
# mask of missing values and then transform down to a layer
# which is smaller than the input -- a bottleneck to force
# generalization
hidden_layer_sizes = [
min(2000, 8 * n_dims),
min(500, 2 * n_dims),
int(np.ceil(0.5 * n_dims)),
]
print("Hidden layer sizes: %s" % (hidden_layer_sizes, ))
nn = Sequential()
first_layer_size = hidden_layer_sizes[0]
nn.add(
Dense(first_layer_size,
input_dim=2 * n_dims,
activation=hidden_activation,
W_regularizer=l1l2(l1_penalty, l2_penalty),
init=init))
nn.add(Dropout(dropout_probability))
for layer_size in hidden_layer_sizes[1:]:
nn.add(
Dense(layer_size,
activation=hidden_activation,
W_regularizer=l1l2(l1_penalty, l2_penalty),
init=init))
nn.add(Dropout(dropout_probability))
nn.add(
Dense(n_dims,
activation=output_activation,
W_regularizer=l1l2(l1_penalty, l2_penalty),
init=init))
loss_function = make_reconstruction_loss(
n_dims, mask_indicates_missing_values=True)
nn.compile(optimizer=optimizer, loss=loss_function)
return nn
def create_model(nr, nr2, dr1, dr2, dr3, lr, wll1, w1l2):
# create model
#L=WinnerTakeAll1D_GaborMellis(spatial=1, OneOnX=WTAX)
input_grid = Input(shape=(16, ))
x1 = Dense(nr,
init='normal',
activation='relu',
W_regularizer=l1l2(l1=w1l1, l2=w1l2))(input_grid)
x2 = Dropout(dr1)(x1)
x3 = Dense(nr, activation='relu')(x2)
x3 = Dense(nr, activation='relu')(x3)
x4 = merge([x3, x2], mode='sum')
x4 = Dropout(dr2)(x4)
x5 = Dense(nr, activation='relu')(x4)
x5 = Dense(nr, activation='relu')(x5)
x6 = merge([x5, x4], mode='sum')
x6 = Dropout(dr2)(x6)
x7 = Dense(nr, activation='relu')(x6)
x7 = Dense(nr, activation='relu')(x6)
x8 = merge([x7, x6], mode='sum')
x8 = Dropout(dr2)(x8)
x9 = Dense(nr, activation='relu')(x8)
x9 = Dense(nr, activation='relu')(x9)
x10 = merge([x9, x8], mode='sum')
x10 = Dropout(dr2)(x10)
x11 = Dense(nr, activation='relu')(x10)
x11 = Dense(nr, activation='relu')(x11)
x12 = merge([x11, x10], mode='sum')
x12 = Dropout(dr2)(x12)
x13 = Dense(nr, activation='relu')(x12)
x13 = Dense(nr, activation='relu')(x13)
x13 = merge([x13, x12], mode='sum')
x12 = Dropout(dr2)(x13)
x14 = Dense(nr, activation='relu')(x13)
x14 = Dense(nr, activation='relu')(x14)
x14 = merge([x14, x13], mode='sum')
x14 = Dropout(dr2)(x14)
x15 = Dense(nr2, activation='relu')(x14)
x15 = Dropout(dr3)(x15)
output = Dense(2, activation='softmax')(x15)
model = Model(input_grid, output)
admax = Adamax(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0)
model.compile(optimizer=admax,
loss='binary_crossentropy',
metrics=['accuracy']) # Gradient descent
return model
def rnn_2(DIM = 0, HIDDEN_SIZE = 0 , DROPOUT = 0, LAYERS = 1, UNIT = 'lstm', MAX_Q = 0, MAX_A = 0, FINETUNE = False, embedding_weights = None): LAYERS = max(LAYERS,min(2,LAYERS)) RNN = RNN_UNIT[UNIT] model = Graph() if FINETUNE: model.add_input(name = 'q', input_shape = (None,), dtype = 'int64') model.add_input(name = 'a', input_shape = (None,), dtype = 'int64') VOCAB = embedding_weights.shape[0] EMB_HIDDEN_SIZE = embedding_weights.shape[1] model.add_node(Embedding(VOCAB,EMB_HIDDEN_SIZE,mask_zero=True, weights=[embedding_weights]), name = 'q_e', input = 'q') model.add_node(Embedding(VOCAB,EMB_HIDDEN_SIZE,mask_zero=True, weights=[embedding_weights]), name = 'a_e', input = 'a') prev_q = 'q_e' prev_a = 'a_e' else: model.add_input(name = 'q', input_shape = (None,DIM)) model.add_input(name = 'a', input_shape = (None,DIM)) prev_q = 'q' prev_a = 'a' model.add_node(RNN(HIDDEN_SIZE, return_sequences = True), name='recurrent_context', input = prev_a) for layer in xrange(LAYERS - 2): model.add_node(RNN(HIDDEN_SIZE, return_sequences=True), name = 'q_rnn_' + str(layer+1), input = prev_q) prev_q = 'q_rnn_' + str(layer+1) model.add_node(RNN(HIDDEN_SIZE, return_sequences = True), name = 'encoder_context', input = prev_q) model.add_node(TimeDistributedAttention(prev_dim = HIDDEN_SIZE, att_dim = HIDDEN_SIZE, return_sequences = True, prev_context = False), name='attention', inputs=['encoder_context','recurrent_context'], merge_mode = 'join_att') prev_a = 'attention' for layer in xrange(LAYERS - 2): model.add_node(RNN(HIDDEN_SIZE, return_sequences=True), name = 'a_rnn_' + str(layer+1), input = prev_a) prev_a = 'a_rnn_' + str(layer+1) model.add_node(RNN(HIDDEN_SIZE, return_sequences=False),name = 'a_rnn_' + str(LAYERS), input = prev_a) model.add_node(Dense(HIDDEN_SIZE, activation = 'relu', W_regularizer = l1l2(l1 = 0.00001, l2 = 0.00001)), name = 'dense_0', input = 'a_rnn_' + str(LAYERS)) model.add_node(Dropout(DROPOUT), name = 'dropout_0', input = 'dense_0') prev_d = 'dropout_0' for layer in xrange(LAYERS-1): model.add_node(Dense(HIDDEN_SIZE, activation = 'relu', W_regularizer = l1l2(l1 = 0.00001, l2 = 0.00001)), name = 'dense_' + str(layer+1), input = prev_d) model.add_node(Dropout(DROPOUT), name = 'dropout_' + str(layer+1), input = 'dense_' + str(layer+1)) prev_d = 'dropout_' + str(layer+1) model.add_node(Dense(1, activation = 'sigmoid'), name = 'sigmoid', input = prev_d) model.add_output(name = 'o', input = 'sigmoid') return model
def regularizer(params):
if 'l1' in params and 'l2' in params:
return regularizers.l1l2(params['l1'], params['l2'])
elif 'l1' in params:
return regularizers.l1(params['l1'])
elif 'l2' in params:
return regularizers.l2(params['l2'])
else:
return None
def get_regularizer(lambda_l1=None, lambda_l2=None):
regularizer = None
if lambda_l1 is None and lambda_l2 is not None:
regularizer = l2(l=lambda_l2)
elif lambda_l1 is not None and lambda_l2 is None:
regularizer = l1(l=lambda_l1)
elif lambda_l1 is not None and lambda_l2 is not None:
regularizer = l1l2(l1=lambda_l1, l2=lambda_l2)
return regularizer
def build_model(in_dim, out_dim=1,
n_hidden=100, l1_norm=0.0,
l2_norm=0,
n_deep=5, drop=0.1,
learning_rate=0.1):
model = Sequential()
# Input layer
model.add(Dense(
input_dim=in_dim,
output_dim=n_hidden,
init='glorot_normal',
activation='tanh',
W_regularizer=l1l2(l1=l1_norm, l2=l2_norm)))
# do X layers
for layer in range(n_deep-1):
model.add(Dropout(drop))
model.add(Dense(
output_dim=np.round(n_hidden/2**(layer+1)),
init='glorot_normal',
activation='tanh',
W_regularizer=l1l2(l1=l1_norm, l2=l2_norm)))
# Output layer
if out_dim == 1:
activation = 'tanh'
else:
activation = 'softmax'
model.add(Dense(out_dim,
init='glorot_normal',
activation=activation))
# Optimization algorithms
opt = Adadelta(lr=learning_rate)
if out_dim == 1:
model.compile(loss='binary_crossentropy',
optimizer=opt)
else:
model.compile(loss='categorical_crossentropy',
optimizer=opt)
return model
def model_discriminator(latent_dim, output_dim=1, hidden_dim=512,
reg=lambda: l1l2(1e-7, 1e-7)):
z = Input((latent_dim,))
h = z
h = Dense(hidden_dim, name="discriminator_h1", W_regularizer=reg())(h)
h = LeakyReLU(0.2)(h)
h = Dense(hidden_dim, name="discriminator_h2", W_regularizer=reg())(h)
h = LeakyReLU(0.2)(h)
y = Dense(output_dim, name="discriminator_y", activation="sigmoid", W_regularizer=reg())(h)
return Model(z, y)
def test_W_reg():
for reg in [regularizers.l1(), regularizers.l2(), regularizers.l1l2()]:
model = create_model(weight_reg=reg)
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
model.fit(X_train,
Y_train,
batch_size=batch_size,
nb_epoch=nb_epoch,
verbose=0)
model.evaluate(X_test[test_ids, :], Y_test[test_ids, :], verbose=0)
def model_discriminator(latent_dim, output_dim=1, hidden_dim=512,
reg=lambda: l1l2(1e-7, 1e-7)):
z = Input((latent_dim,))
h = z
h = Dense(hidden_dim, name="discriminator_h1", W_regularizer=reg())(h)
h = LeakyReLU(0.2)(h)
h = Dense(hidden_dim, name="discriminator_h2", W_regularizer=reg())(h)
h = LeakyReLU(0.2)(h)
y = Dense(output_dim, name="discriminator_y", activation="sigmoid", W_regularizer=reg())(h)
return Model(z, y)
def create_model_test():
model = Sequential()
model.add(Dense(280, input_dim=16, init='normal', activation='relu' ,W_regularizer=l1l2(l1=5e-06, l2=5e-06), activity_regularizer=l1l2(l1=0, l2=1e-5))) #W_regularizer=l1(0.000001), activity_regularizer=activity_l1(0.000001)))
model.add(Dropout(0.25))
model.add(Dense(370, activation ='relu',activity_regularizer=l1l2(l1=0, l2=5e-5)))
model.add(Dropout(0.5))
model.add(Dense(120, activation ='relu',W_regularizer=l1l2(l1=0, l2=5e-06)))
model.add(Dropout(0.55))
model.add(Dense(1))
model.add(Activation('sigmoid'))
admax = Adamax(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)#decay ? 0.002
model.compile(optimizer=admax, loss='binary_crossentropy', metrics=['accuracy']) # Gradient descent
return model
def test_W_reg():
(X_train, Y_train), (X_test, Y_test), test_ids = get_data()
for reg in [regularizers.l1(),
regularizers.l2(),
regularizers.l1l2()]:
model = create_model(weight_reg=reg)
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
assert len(model.losses) == 1
model.fit(X_train, Y_train, batch_size=batch_size,
nb_epoch=nb_epoch, verbose=0)
model.evaluate(X_test[test_ids, :], Y_test[test_ids, :], verbose=0)
def network(regl1, regl2, weight_init, dropout, optimize):
#create network architecture
model = Sequential()
model.add(Convolution2D(16, 7, 7,input_shape=(1, 256, 192),W_regularizer=l1l2(l1=regl1, l2=regl2),init=weight_init))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(dropout))
model.add(Convolution2D(32, 6, 6, W_regularizer=l1l2(l1=regl1, l2=regl2),init=weight_init))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(dropout))
model.add(Convolution2D(64, 3, 3, W_regularizer=l1l2(l1=regl1, l2=regl2),init=weight_init))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(dropout))
model.add(Convolution2D(64, 2, 2, W_regularizer=l1l2(l1=regl1, l2=regl2),init=weight_init))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(dropout))
model.add(Flatten())
model.add(Dense(50,W_regularizer=l1l2(l1=regl1, l2=regl2),init=weight_init))
#model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(output_dim=1))
model.add(Activation('sigmoid'))
model.compile(optimizer=optimize, loss='binary_crossentropy', metrics=['accuracy'])
return model
def __init__(self, **settings):
# max_features, embedding_dim, seqlen,
# nb_filter, filter_size, activation, dropout_p,
# l1reg, l2reg, batchnorm,
self.settings = settings
self.settings['verbosity'] = 2
seqlen = self.settings['seqlen']
l1reg = self.settings['l1reg']
l2reg = self.settings['l2reg']
self.nn = Sequential()
self.nn.add(Embedding(input_dim=self.settings['max_features'],
output_dim=self.settings['embedding_dim'],
input_length=seqlen))
self.nn.add(Dropout(self.settings['dropout_p']))
self.nn.add(Convolution1D(self.settings['nb_filter'],
self.settings['filter_size'],
activation=self.settings['activation']))
self.nn.add(Convolution1D(self.settings['nb_filter'],
self.settings['filter_size'],
activation=self.settings['activation']))
self.nn.add(MaxPooling1D(pool_length=2))
self.nn.add(Convolution1D(self.settings['nb_filter'],
self.settings['filter_size'],
activation=self.settings['activation']))
self.nn.add(Convolution1D(self.settings['nb_filter'],
self.settings['filter_size'],
activation=self.settings['activation']))
self.nn.add(MaxPooling1D(pool_length=2))
self.nn.add(Convolution1D(self.settings['nb_filter'],
self.settings['filter_size'],
activation=self.settings['activation']))
self.nn.add(Convolution1D(self.settings['nb_filter'],
self.settings['filter_size'],
activation=self.settings['activation']))
self.nn.add(MaxPooling1D(pool_length=2))
self.nn.add(Flatten())
self.nn.add(Dropout(self.settings['dropout_p']))
self.nn.add(Dense(10))
self.nn.add(Activation(self.settings['activation']))
self.nn.add(Dense(10))
self.nn.add(Activation(self.settings['activation']))
if (l1reg is not None and l1reg is float and l2reg is not
None and l2reg is float):
self.nn.add(Dense(1), W_regularizer=l1l2(l1reg, l2reg))
elif (l2reg is not None and l2reg is float):
self.nn.add(Dense(1), W_regularizer=l2(l2reg))
elif (l1reg is not None and l1reg is float):
self.nn.add(Dense(1), W_regularizer=l1(l1reg))
else:
self.nn.add(Dense(1))
if (self.settings['batchnorm'] is True):
self.nn.add(BatchNormalization())
self.nn.add(Activation('sigmoid'))
def create_model():
model = Sequential()
model.add(
Dense(280,
input_dim=16,
init='normal',
activation='relu',
W_regularizer=l1l2(l1=5e-06, l2=5e-06),
activity_regularizer=l1l2(l1=0, l2=1e-5))
) #W_regularizer=l1(0.000001), activity_regularizer=activity_l1(0.000001)))
model.add(Dropout(0.25))
model.add(
Dense(370, activation='relu', activity_regularizer=l1l2(l1=0,
l2=5e-5)))
model.add(Dropout(0.5))
model.add(Dense(120, activation='relu', W_regularizer=l1l2(l1=0,
l2=5e-06)))
model.add(Dropout(0.55))
model.add(Dense(1))
model.add(Activation('sigmoid'))
admax = Adamax(lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0) #decay ? 0.002
"""
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.97, patience=1, min_lr=0.00001)
callbacks = [
EarlyStopping(monitor='val_loss', patience=25, verbose=0),
ModelCheckpoint("/home/admin-7036/Documents/Projet python/bosongit/weigh.hdf", monitor='val_loss', save_best_only=True, verbose=0),
reduce_lr
]
"""
model.compile(optimizer=admax,
loss='binary_crossentropy',
metrics=['accuracy']) # Gradient descent
return model
def __init__(self, filename, labels, model_type, feature_params=None, model=None,
layers=1, layer_dim=100, activation="tanh", normalize=False,
init="glorot_normal", max_num_labels=100, batch_size=10,
minibatch_size=200, nb_epochs=5, dropout=0,
optimizer="adam", loss="categorical_crossentropy",
regularizer="l2", regularization=1e-8):
"""
Create a new untrained NN or copy the weights from an existing one
:param labels: a list of labels that can be updated later to add a new label
:param feature_params: dict of feature type name -> FeatureInformation
:param model: if given, copy the weights (from a trained model)
:param layers: number of hidden layers
:param layer_dim: size of hidden layer
:param activation: activation function at hidden layers
:param normalize: perform batch normalization after each layer?
:param init: initialization type for hidden layers
:param max_num_labels: since model size is fixed, set maximum output size
:param batch_size: fit model every this many items
:param minibatch_size: batch size for SGD
:param nb_epochs: number of epochs for SGD
:param dropout: dropout to apply to input layer
:param optimizer: algorithm to use for optimization
:param loss: objective function to use for optimization
:param regularizer: regularization type (None, l1, l2 or l1l2)
:param regularization: regularization parameter lambda
"""
super(NeuralNetwork, self).__init__(model_type=model_type, filename=filename,
labels=labels, model=model)
assert feature_params is not None or model is not None
if self.is_frozen:
self.model = model
else:
self.max_num_labels = max_num_labels
self._layers = layers
self._layer_dim = layer_dim
self._activation = (lambda x: x*x*x) if activation == "cube" else activation
self._normalize = normalize
self._init = init
self._num_labels = self.num_labels
self._minibatch_size = minibatch_size
self._nb_epochs = nb_epochs
self._dropout = dropout
self._optimizer = optimizer
self._loss = (lambda t, p: K.sum(K.maximum(0., 1.-p*t+p*(1.-t)))) if loss == "max_margin" else loss
self._regularizer = (lambda: None) if regularizer is None else \
(lambda: regularizers.l1l2(regularization, regularization)) if regularizer == "l1l2" else \
(lambda: regularizers.get(regularizer, {"l": regularization}))
self.feature_params = feature_params
self.model = None
self._batch_size = batch_size
self._item_index = 0
self._iteration = 0
def test_dense(self):
input_data = np.random.random_sample([1, 10])
layer = Dense(2, init='one', activation="relu",
input_shape=(10, ), W_regularizer=l1l2(l1=0.01, l2=0.02))
self.modelTestSingleLayer(input_data, layer, dump_weights=True)
layer2 = Dense(2, init='one', activation="softplus",
input_shape=(10, ), b_regularizer=l2(0.02))
self.modelTestSingleLayer(input_data, layer2, dump_weights=True)
layer3 = Dense(2, init='one', input_shape=(10, ),
W_regularizer=keras.regularizers.WeightRegularizer(l1=0.1))
self.modelTestSingleLayer(input_data, layer3, dump_weights=True)
layer4 = Dense(2, init='glorot_uniform', activation="hard_sigmoid", input_shape=(10, ))
self.modelTestSingleLayer(input_data, layer4, dump_weights=True)
def build_model(params):
n_hidden_layers = int(np.round(params['n_hidden_layers'][0]))
n_neurons = int(np.round(params['n_neurons'][0]))
log_l1_weight_reg = np.float32(params['log_l1_weight_reg'][0])
log_l2_weight_reg = np.float32(params['log_l2_weight_reg'][0])
#prob_drop_out = float(params['prob_drop_out'][ 0 ].astype('float32'))
prob_drop_out = np.float32(params['prob_drop_out'][0])
log_l_rate = np.float32(params['log_learning_rate'][0])
print n_hidden_layers
print n_neurons
print log_l1_weight_reg
print log_l2_weight_reg
print prob_drop_out
print log_l_rate
model = Sequential()
model.add(Dense(n_neurons, input_shape = (784,), W_regularizer=l1l2(l1 = np.exp(log_l1_weight_reg), \
l2 = np.exp(log_l2_weight_reg))))
model.add(Activation('relu'))
prob_drop_out = float(prob_drop_out)
model.add(Dropout(prob_drop_out))
#model.add(Dropout(0.35))
for i in range(n_hidden_layers - 1):
model.add(Dense(n_neurons, W_regularizer=l1l2(l1 = np.exp(log_l1_weight_reg), \
l2 = np.exp(log_l2_weight_reg))))
model.add(Activation('relu'))
model.add(Dropout(prob_drop_out))
#model.add(Dropout(0.35))
n_classes = 10
model.add(Dense(n_classes))
model.add(Activation('softmax'))
adam = Adam(lr=np.exp(log_l_rate), beta_1=0.9, beta_2=0.999, epsilon=1e-08)
model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=['accuracy'])
return model
def model4(model, ENT_EMB, REL_EMB, MAXLEN, HIDDEN_SIZE): from keras.layers.averagelayer import Average from keras.regularizers import l1l2, l2 model.add_shared_node(Embedding(ENT_EMB,HIDDEN_SIZE, input_length = MAXLEN),name = 'embedding', inputs = ['input1','input2']) model.add_shared_node(Average(), name='avg', inputs=['embedding']) prev = 'avg' for layer in xrange(3): model.add_shared_node(Dense(HIDDEN_SIZE, activation = 'relu', W_regularizer = l1l2(l1 = 0.00001, l2 = 0.00001)), name='dense'+str(layer+1), inputs=[prev]) model.add_shared_node(Dropout(0.25),name='dense'+str(layer+1) + '_d', inputs = ['dense'+str(layer+1)] ) prev = 'dense'+str(layer+1)+'_d' model.add_shared_node(Layer(), name='merge_siam', inputs=[prev], merge_mode = 'concat', concat_axis = -1) return model
def model3():
model = Sequential()
model.add(Convolution2D(nb_filter=10,nb_row=5,nb_col=7,input_shape=(600,600,1),W_regularizer=l1l2()))
model.add(MaxPooling2D(pool_size=(3,3)))
model.add(PReLU())
model.add(Flatten())
model.add(Dropout(0.5))
model.add(Dense(output_dim=10,W_regularizer=l1l2()))
model.add(PReLU())
model.add(Dropout(0.5))
model.add(Dense(output_dim=8,activation='softmax'))
return model
def create_model():
model = Sequential()
model.add(
Dense(100,
input_dim=147,
init='normal',
activation='relu',
W_regularizer=l1l2(l1=1E-6, l2=1E-5),
activity_regularizer=l1l2(l1=0, l2=1e-7))
) #W_regularizer=l1(0.000001), activity_regularizer=activity_l1(0.000001)))
model.add(Dropout(0.1))
model.add(
Dense(540,
activation='relu',
W_regularizer=l1l2(l1=1e-07, l2=0),
activity_regularizer=l1l2(l1=0, l2=5e-7)))
model.add(Dropout(0.3))
model.add(Dense(310, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(450, activation='relu'))
model.add(Dropout(0.4))
model.add(Dense(1))
model.add(Activation('sigmoid'))
admax = Adamax(lr=0.005,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0) #decay ? 0.002
model.compile(optimizer=admax,
loss='binary_crossentropy',
metrics=['accuracy']) # Gradient descent
return model
def build_model(ncell, nmark, nfilter, coeff_l1, coeff_l2,
k, dropout, dropout_p, regression, n_classes, lr=0.01):
""" Builds the neural network architecture """
# the input layer
data_input = Input(shape=(ncell, nmark))
# the filters
conv = Convolution1D(nfilter, 1, activation='linear',
W_regularizer=l1l2(l1=coeff_l1, l2=coeff_l2),
name='conv1')(data_input)
conv = Activation('relu')(conv) ### filter responses?
# the cell grouping part
pooled = Lambda(select_top, output_shape=(nfilter,), arguments={'k':k})(conv)
# possibly add dropout
if dropout or ((dropout == 'auto') and (nfilter > 5)):
pooled = Dropout(p=dropout_p)(pooled)
# network prediction output
if not regression:
output = Dense(n_classes, activation='softmax',
W_regularizer=l1l2(l1=coeff_l1, l2=coeff_l2),
name='output')(pooled)
else:
output = Dense(1, activation='linear', W_regularizer=l1l2(l1=coeff_l1, l2=coeff_l2),
name='output')(pooled)
model = Model(input=data_input, output=output)
if not regression:
model.compile(optimizer=Adam(lr=lr),
loss='categorical_crossentropy',
metrics=['accuracy'])
else:
model.compile(optimizer=Adam(lr=lr),
loss='mean_squared_error')
return model
def build_model(train):
input_dim = train.shape[1]
model = Sequential()
model.add(
Dense(15,
input_dim=input_dim,
init='glorot_normal',
activation='tanh',
W_regularizer=l1l2(l1=1e-4, l2=1e-4)))
model.add(Dropout(0.15))
model.add(Dense(1, init='zero', activation='linear'))
model.compile(loss="mse", optimizer="adagrad")
return model
def timed_sgd_Elastic(x1,y1,x2=None,y2=None,
lambda1=0.01, lambda2=0.01, lr=0.1, decay=1e-2, nesterov=True, momentum=0.8, batch_size=100,nb_epoch=50):
if (x2 is None)^(y2 is None): raise ValueError("if you specify x2 or y2 you need to specify the other as well")
if x2 is None and y2 is None: x2=x1; y2=y1 #if no Cross-validation set, use original
print("Running SGD Elastic with lambda1: ", lambda1, " lambda2: ", lambda2, " lr: ", lr, " decay: ", decay, " Momentum: ", momentum, " Batch size: ", batch_size)
time0 = time() # Start timer
earlystopper = EarlyStopping(monitor='loss', patience=1)
sgd=SGD(lr=lr, decay=decay, nesterov=nesterov, momentum=momentum)
model = Sequential()
model.add(Dense(1, input_dim=K, activation='linear', W_regularizer=l1l2(lambda1,lambda2)))
model.compile(loss='mse', optimizer=sgd)
model.fit(x1, y1,nb_epoch=nb_epoch,batch_size=batch_size, show_accuracy=True, callbacks=[earlystopper], verbose=0)
score = model.evaluate(x2, y2, batch_size=20)
return time()-time0, score
def l1l2_penalty_reg(alpha=1.0, l1_ratio=0.5):
'''Calculate L1 and L2 penalties for a Keras layer
This follows the same formulation as in the R package glmnet and Sklearn
Args:
alpha ([float]): amount of regularization.
l1_ratio ([float]): portion of L1 penalty. Setting to 1.0 equals
Lasso.
'''
if l1_ratio == .0:
return l2(alpha)
elif l1_ratio == 1.:
return l1(alpha)
else:
return l1l2(l1_ratio * alpha, 1. / 2 * (1 - l1_ratio) * alpha)
def vgg16(input_shape=(224, 224, 3),
weights=None,
vgg_transfer=None,
activation_fn='relu',
l1=0.00001,
l2=0.00001,
dropout=0.5):
input_tensor = Input(shape=input_shape)
if vgg_transfer is not None:
vgg_transfer = 'imagenet'
vgg = VGG16(input_tensor=input_tensor,
include_top=False,
weights=vgg_transfer)
x = Flatten(name='flatten')(vgg.output)
x = Dense(4096,
activation=activation_fn,
name='fc1',
W_regularizer=l1l2(l1, l2),
b_regularizer=l1l2(l1, l2))(x)
x = Dropout(dropout)(x)
x = Dense(4096,
activation=activation_fn,
name='fc2',
W_regularizer=l1l2(l1, l2),
b_regularizer=l1l2(l1, l2))(x)
x = Dense(2, activation='softmax', name='predictions')(x)
model = Model(input=input_tensor, output=x)
if weights is not None:
model.load_weights(weights)
return model
def make_model(dropout, nb_filters, nb_conv, nb_pool,weight_initiation,activation_function,l1_reg,l2_reg):
'''Creates model comprised of 2 convolutional layers followed by dense layers
dense_layer_sizes: List of layer sizes. This list has one number for each layer
nb_filters: Number of convolutional filters in each convolutional layer
nb_conv: Convolutional kernel size
nb_pool: Size of pooling area for max pooling
'''
model = Sequential()
model.add(Convolution2D(8, 8, 8,
border_mode='valid',
input_shape=(1, img_rows, img_cols),subsample = (4,4),W_regularizer=l1l2(l1 = l1_reg,l2=l2_reg),b_regularizer=l1l2(l1 = l1_reg,l2=l2_reg),init=weight_initiation))
model.add(Activation(activation_function))
model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
model.add(Dropout(dropout))
model.add(Convolution2D(8, 5, 5,W_regularizer=l1l2(l1 = l1_reg,l2=l2_reg),b_regularizer=l1l2(l1 = l1_reg,l2=l2_reg),subsample = (2,2),init=weight_initiation))
model.add(Activation(activation_function))
model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
model.add(Dropout(dropout))
model.add(Convolution2D(16, nb_conv, nb_conv,W_regularizer=l1l2(l1=l1_reg,l2=l2_reg),b_regularizer=l1l2(l1 = l1_reg,l2=l2_reg),init=weight_initiation))
model.add(Activation(activation_function))
model.add(Dropout(dropout))
model.add(Flatten())
model.add(Dense(5,W_regularizer=l1l2(l1 = l1_reg,l2=l2_reg),b_regularizer=l1l2(l1 = l1_reg,l2=l2_reg),init=weight_initiation))
model.add(Activation(activation_function))
model.add(Dropout(dropout))
model.add(Dense(output_dim=1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adadelta')
return model
def dan(DIM = 0, HIDDEN_SIZE = 0 , DROPOUT = 0, LAYERS = 1, UNIT = 'lstm', MAX_Q = 0, MAX_A = 0, FINETUNE = False, embedding_weights = None): from keras.layers.averagelayer import Average model = Graph() if FINETUNE: model.add_input(name = 'q', input_shape = (None,), dtype = 'int64') model.add_input(name = 'a', input_shape = (None,), dtype = 'int64') VOCAB = embedding_weights.shape[0] EMB_HIDDEN_SIZE = embedding_weights.shape[1] model.add_node(Embedding(VOCAB,EMB_HIDDEN_SIZE,mask_zero=True, weights=[embedding_weights]), name = 'q_e', input = 'q') model.add_node(Embedding(VOCAB,EMB_HIDDEN_SIZE,mask_zero=True, weights=[embedding_weights]), name = 'a_e', input = 'a') prev_q = 'q_e' prev_a = 'a_e' else: model.add_input(name = 'q', input_shape = (None,DIM)) model.add_input(name = 'a', input_shape = (None,DIM)) prev_q = 'q' prev_a = 'a' EMB_HIDDEN_SIZE = DIM # model.add_node(Average(),name = 'avg_a', inputs = [prev_a], merge_mode = 't_ave') # model.add_node(Average(), name = 'avg_q', inputs = [prev_q], merge_mode = 't_ave') model.add_node(Average(),name = 'avg_a', input = prev_a) model.add_node(Average(), name = 'avg_q', input = prev_q) model.add_node(Dense(HIDDEN_SIZE, activation = 'relu', W_regularizer = l1l2(l1 = 0.00001, l2 = 0.00001)), name = 'd_0', inputs = ['avg_a','avg_q'], merge_mode = 'concat', concat_axis = -1) model.add_node(Dropout(DROPOUT), name = 'd_0_dr', input = 'd_0') prev = 'd_0_dr' for layer in xrange(LAYERS-1): model.add_node(Dense(HIDDEN_SIZE, activation = 'relu', W_regularizer = l1l2(l1 = 0.00001, l2 = 0.00001)), name = 'd_' + str(layer+1), input = prev) model.add_node(Dropout(DROPOUT), name = 'd_' + str(layer+1) + '_dr', input = 'd_' + str(layer+1)) prev = 'd_' + str(layer+1) + '_dr' model.add_node(Dense(1, activation = 'sigmoid'), name = 'sigmoid', input = prev) model.add_output(name = 'o', input = 'sigmoid') return model
def add_input(filed_name, input_len_field, layers_name, total_len = 0):
temp_len = int(np.ceil(Log(input_len_field)) + 1)
# Input layer
globals()["input_{F}".format(F=filed_name)] = Input(shape=(input_len_field,),
name="input_{F}".format(F=filed_name)
)
# "Emmbeding" layer
globals()["input_{F}_D".format(F=filed_name)] = Dense(temp_len,
activation='sigmoid',
init='uniform',
name="input_{F}_D".format(F=filed_name),
W_regularizer=l1l2(l1=l1_reglazation, l2=l2_reglazation)
)(globals()["input_{F}".format(F=filed_name)])
layers_name.append(globals()["input_{F}_D".format(F=filed_name)])
total_len[0] += temp_len
def train(self):
self.model = Sequential()
self.model.add(Dense(100, input_dim=self.N, W_regularizer=l1l2(), b_regularizer=l1l2(), activity_regularizer=activity_l1l2()))
self.model.add(Activation('relu'))
self.model.add(Dropout(0.25))
self.model.add(Dense(100, W_regularizer=l1l2(), b_regularizer=l1l2(), activity_regularizer=activity_l1l2()))
self.model.add(Activation('relu'))
self.model.add(Dropout(0.25))
self.model.add(Dense(100, W_regularizer=l1l2(), b_regularizer=l1l2(), activity_regularizer=activity_l1l2()))
self.model.add(Activation('relu'))
self.model.add(Dropout(0.25))
self.model.add(Dense(100, W_regularizer=l1l2(), b_regularizer=l1l2(), activity_regularizer=activity_l1l2()))
self.model.add(Activation('relu'))
self.model.add(Dropout(0.25))
self.model.add(Dense(1))
self.model.compile(loss='mean_squared_error', optimizer='rmsprop')
self.model.fit(np.array(self.X), np.array(self.Y), show_accuracy=True, batch_size=16, nb_epoch=2000, verbose=2)
def autoencoder(train, tests, valid, ws, compression_rate, max_epochs, activation, sparcity, optimizer):
# Compute the layer sizes.
layer1 = int(ws * compression_rate)
layer2 = int(layer1 * compression_rate)
# Construct a Keras model.
model = Sequential()
regular = None
if sparcity == 'l1':
regular = l1()
elif sparcity == 'l2':
regular = l2()
elif sparcity == 'l1l2':
regular = l1l2()
# Add the first set of connections to the network
model.add(Dense(layer1, input_dim=ws, activation=activation, W_regularizer=regular))
model.add(Dense(layer2, input_dim=layer1, activation=activation, W_regularizer=regular))
model.add(Dense(layer1, input_dim=layer2, activation=activation, W_regularizer=regular))
model.add(Dense(ws, input_dim=layer1, activation=activation, W_regularizer=regular))
# Compile the model using binary crossentropy and rmsprop optimization.
model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
# Train the model using early stopping.
best_accuracy = -1.0
keep_training = True
epochs_done = 0
while keep_training:
epochs_done += 5
print("____Epochs done = " + str(epochs_done))
# Fit the model using 64 additional epochs.
model.fit(train, train, batch_size=1, nb_epoch=5, verbose=0)
loss_tests, accuracy_tests = model.evaluate(tests, tests, batch_size=1, verbose=0)
# If the accuracy deteriorates or the epochs done exceeds the max epochs stop training.
if accuracy_tests <= best_accuracy or epochs_done >= max_epochs:
keep_training = False
else:
best_accuracy = accuracy_tests
# Compute the performance metrics and then compute their significance.
loss_train, accuracy_train = model.evaluate(train, train, batch_size=1, verbose=0)
loss_tests, accuracy_tests = model.evaluate(tests, tests, batch_size=1, verbose=0)
loss_valid, accuracy_valid = model.evaluate(valid, valid, batch_size=1, verbose=0)
print("____" + str(loss_train) + ";" + str(loss_tests) + ";" + str(loss_valid))
return loss_train, loss_tests, loss_valid
def discriminator_model(sample_dimension, layers, activation=None,
reg=lambda: l1l2(1e-5, 1e-5), dropout=0.25, batch_norm_mode=0):
model = Sequential()
model.add(Dense(sample_dimension, input_shape=(sample_dimension,)))
for layer_dims in layers:
model.add(Dense(layer_dims, W_regularizer=reg()))
if batch_norm_mode is not None:
model.add(BatchNormalization(mode=batch_norm_mode))
if activation is not None:
if type(activation) is str:
model.add(Activation(activation))
else:
model.add(Activation(activation()))
if dropout is not None and dropout > 0:
model.add(Dropout(dropout))
model.add(Dense(1, activation='sigmoid'))
return model
def test_dense(self):
input_data = np.random.random_sample([2, 10, 5, 7])
layer = Dense(2, init='one', activation="relu",
input_shape=(10, 5, 7), W_regularizer=l1l2(l1=0.01, l2=0.02))
self.modelTestSingleLayer(input_data, layer, dump_weights=True)
input_data2 = np.random.random_sample([2, 10])
layer2 = Dense(2, init='one', activation="softplus",
input_shape=(10, ), b_regularizer=l2(0.02))
self.modelTestSingleLayer(input_data2, layer2, dump_weights=True)
layer3 = Dense(2, init='one', input_shape=(10, ),
W_regularizer=keras.regularizers.WeightRegularizer(l1=0.1))
self.modelTestSingleLayer(input_data2, layer3, dump_weights=True)
layer4 = Dense(2, init='glorot_uniform', activation="hard_sigmoid", input_shape=(10, ))
self.modelTestSingleLayer(input_data2, layer4, dump_weights=True)
# Test for unsupported init_method. Should get a warning not an exception.
layer5 = Dense(4, init='he_uniform', input_shape=(10, ))
self.modelTestSingleLayer(input_data2, layer5, dump_weights=True)
def model_discriminator(latent_dim, input_shape, output_dim=1, hidden_dim=1024,
reg=lambda: l1l2(1e-4, 1e-4), batch_norm_mode=1):
z = Input((latent_dim,))
x = Input(input_shape, name="x")
h = merge([z, Flatten()(x)], mode='concat')
h = Dense(hidden_dim, name="discriminator_h1", W_regularizer=reg())(h)
h = BatchNormalization(mode=batch_norm_mode)(h)
h = LeakyReLU(0.2)(h)
h = Dropout(0.5)(h)
h = Dense(hidden_dim / 2, name="discriminator_h2", W_regularizer=reg())(h)
h = BatchNormalization(mode=batch_norm_mode)(h)
h = LeakyReLU(0.2)(h)
h = Dropout(0.5)(h)
h = Dense(hidden_dim / 4, name="discriminator_h3", W_regularizer=reg())(h)
h = BatchNormalization(mode=batch_norm_mode)(h)
h = LeakyReLU(0.2)(h)
h = Dropout(0.5)(h)
y = Dense(output_dim, name="discriminator_y", activation="sigmoid", W_regularizer=reg())(h)
return Model([z, x], y, name="discriminator")
def model_discriminator(input_shape, hidden_dim=1024, reg=lambda: l1l2(1e-5, 1e-5), dropout=0.5, batch_norm_mode=1,
output_activation="sigmoid"):
return Sequential([
Flatten(name="discriminator_flatten", input_shape=input_shape),
Dense(hidden_dim, name="discriminator_h1", W_regularizer=reg()),
batch_norm(batch_norm_mode),
LeakyReLU(0.2),
dropout_layer(dropout),
Dense(hidden_dim / 2, name="discriminator_h2", W_regularizer=reg()),
batch_norm(batch_norm_mode),
LeakyReLU(0.2),
dropout_layer(dropout),
Dense(hidden_dim / 4, name="discriminator_h3", W_regularizer=reg()),
batch_norm(batch_norm_mode),
LeakyReLU(0.2),
dropout_layer(dropout),
Dense(1, name="discriminator_y", W_regularizer=reg()),
Activation(output_activation)],
name="discriminator")
def createModel(input_shape, tf_ordering=True, second_phase = False):
print("Creating new model with input shape", input_shape)
axis = -1
if not(tf_ordering):
axis = 1
alpha = 0.1
w_reg = 0.0001
print("Hyperparameters: alpha=%f, w_reg=%f"%(alpha, w_reg))
path1 = Sequential()
path1.add(Convolution2D(64, 7, 7, border_mode='valid', input_shape = input_shape, W_regularizer=l1l2(l1 = w_reg, l2 = w_reg), trainable=not(second_phase)))
path1.add(Dropout(alpha))
path1.add(Activation('relu'))
path1.add(MaxPooling2D(pool_size=(4,4), strides=(1,1), border_mode='valid'))
path1.add(Convolution2D(64, 3, 3, border_mode='valid', W_regularizer=l1l2(l1 = w_reg, l2 = w_reg), trainable=not(second_phase)))
path1.add(Dropout(alpha))
path1.add(Activation('relu'))
path1.add(MaxPooling2D(pool_size=(2,2), strides=(1,1), border_mode='valid'))
path2 = Sequential()
path2.add(Convolution2D(160, 13, 13, border_mode='valid', input_shape = input_shape, W_regularizer=l1l2(l1 = w_reg, l2 = w_reg), trainable=not(second_phase)))
path2.add(Dropout(alpha))
path2.add(Activation('relu'))
classification_layer = Sequential()
classification_layer.add(Merge([path1, path2], mode='concat', concat_axis=axis))
classification_layer.add(Convolution2D(5, 21, 21, border_mode='valid', W_regularizer=l1l2(l1 = w_reg, l2 = w_reg)))
classification_layer.add(Dropout(alpha))
classification_layer.add(Flatten())
classification_layer.add(Activation('softmax'))
sgd = SGD(lr=0.005, decay = 1e-1, momentum=0.5, nesterov=True)
adam = Adam(lr=0.0005)
classification_layer.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['accuracy', dice])
classification_layer.summary()
return classification_layer
def comp_double(self):
'''
double model. Simialar to two-pathway, except takes in a 4x33x33 patch and it's center 4x5x5 patch. merges paths at flatten layer.
'''
print ('Compiling double model...')
single = Sequential()
single.add(Convolution2D(64, 7, 7, border_mode='valid', W_regularizer=l1l2(l1=0.01, l2=0.01), input_shape=(4,33,33)))
single.add(Activation('relu'))
single.add(BatchNormalization(mode=0, axis=1))
single.add(MaxPooling2D(pool_size=(2,2), strides=(1,1)))
single.add(Dropout(0.5))
single.add(Convolution2D(nb_filter=128, nb_row=5, nb_col=5, activation='relu', border_mode='valid', W_regularizer=l1l2(l1=0.01, l2=0.01)))
single.add(BatchNormalization(mode=0, axis=1))
single.add(MaxPooling2D(pool_size=(2,2), strides=(1,1)))
single.add(Dropout(0.5))
single.add(Convolution2D(nb_filter=256, nb_row=5, nb_col=5, activation='relu', border_mode='valid', W_regularizer=l1l2(l1=0.01, l2=0.01)))
single.add(BatchNormalization(mode=0, axis=1))
single.add(MaxPooling2D(pool_size=(2,2), strides=(1,1)))
single.add(Dropout(0.5))
single.add(Convolution2D(nb_filter=128, nb_row=3, nb_col=3, activation='relu', border_mode='valid', W_regularizer=l1l2(l1=0.01, l2=0.01)))
single.add(Dropout(0.25))
single.add(Flatten())
# add small patch to train on
five = Sequential()
five.add(Reshape((100,1), input_shape = (4,5,5)))
five.add(Flatten())
five.add(MaxoutDense(128, nb_feature=5))
five.add(Dropout(0.5))
model = Sequential()
# merge both paths
model.add(Merge([five, single], mode='concat', concat_axis=1))
model.add(Dense(5))
model.add(Activation('softmax'))
sgd = SGD(lr=0.001, decay=0.01, momentum=0.9)
model.compile(loss='categorical_crossentropy', optimizer='sgd')
print ('Done.')
return model
def comp_double(self):
'''
double model. Simialar to two-pathway, except takes in a 4x33x33 patch and it's center 4x5x5 patch. merges paths at flatten layer.
'''
print 'Compiling double model...'
single = Sequential()
single.add(Convolution2D(64, 7, 7, border_mode='valid', W_regularizer=l1l2(l1=0.01, l2=0.01), input_shape=(4,33,33)))
single.add(Activation('relu'))
single.add(BatchNormalization(mode=0, axis=1))
single.add(MaxPooling2D(pool_size=(2,2), strides=(1,1)))
single.add(Dropout(0.5))
single.add(Convolution2D(nb_filter=128, nb_row=5, nb_col=5, activation='relu', border_mode='valid', W_regularizer=l1l2(l1=0.01, l2=0.01)))
single.add(BatchNormalization(mode=0, axis=1))
single.add(MaxPooling2D(pool_size=(2,2), strides=(1,1)))
single.add(Dropout(0.5))
single.add(Convolution2D(nb_filter=256, nb_row=5, nb_col=5, activation='relu', border_mode='valid', W_regularizer=l1l2(l1=0.01, l2=0.01)))
single.add(BatchNormalization(mode=0, axis=1))
single.add(MaxPooling2D(pool_size=(2,2), strides=(1,1)))
single.add(Dropout(0.5))
single.add(Convolution2D(nb_filter=128, nb_row=3, nb_col=3, activation='relu', border_mode='valid', W_regularizer=l1l2(l1=0.01, l2=0.01)))
single.add(Dropout(0.25))
single.add(Flatten())
# add small patch to train on
five = Sequential()
five.add(Reshape((100,1), input_shape = (4,5,5)))
five.add(Flatten())
five.add(MaxoutDense(128, nb_feature=5))
five.add(Dropout(0.5))
model = Sequential()
# merge both paths
model.add(Merge([five, single], mode='concat', concat_axis=1))
model.add(Dense(5))
model.add(Activation('softmax'))
sgd = SGD(lr=0.001, decay=0.01, momentum=0.9)
model.compile(loss='categorical_crossentropy', optimizer='sgd')
print 'Done.'
return model
def model_discriminator(latent_dim,
input_shape,
output_dim=1,
hidden_dim=2048,
reg=lambda: l1l2(1e-7, 1e-7),
batch_norm_mode=1,
dropout=0.5):
z = Input((latent_dim, ))
x = Input(input_shape, name="x")
h = merge([z, Flatten()(x)], mode='concat')
h1 = Dense(hidden_dim, name="discriminator_h1", W_regularizer=reg())
b1 = BatchNormalization(mode=batch_norm_mode)
h2 = Dense(hidden_dim, name="discriminator_h2", W_regularizer=reg())
b2 = BatchNormalization(mode=batch_norm_mode)
h3 = Dense(hidden_dim, name="discriminator_h3", W_regularizer=reg())
b3 = BatchNormalization(mode=batch_norm_mode)
y = Dense(output_dim,
name="discriminator_y",
activation="sigmoid",
W_regularizer=reg())
# training model uses dropout
_h = h
_h = Dropout(dropout)(LeakyReLU(0.2)((b1(h1(_h)))))
_h = Dropout(dropout)(LeakyReLU(0.2)((b2(h2(_h)))))
_h = Dropout(dropout)(LeakyReLU(0.2)((b3(h3(_h)))))
ytrain = y(_h)
mtrain = Model([z, x], ytrain, name="discriminator_train")
# testing model does not use dropout
_h = h
_h = LeakyReLU(0.2)((b1(h1(_h))))
_h = LeakyReLU(0.2)((b2(h2(_h))))
_h = LeakyReLU(0.2)((b3(h3(_h))))
ytest = y(_h)
mtest = Model([z, x], ytest, name="discriminator_test")
return mtrain, mtest
def model_discriminator(latent_dim,
input_shape,
output_dim=1,
hidden_dim=512,
activation="tanh",
reg=lambda: l1l2(1e-3, 1e-3)):
z = Input((latent_dim, ))
x = Input(input_shape, name="x")
h = merge([z, Flatten()(x)], mode='concat')
h = Dense(hidden_dim,
name="discriminator_h1",
activation=activation,
W_regularizer=reg())(h)
h = Dense(hidden_dim,
name="discriminator_h2",
activation=activation,
W_regularizer=reg())(h)
y = Dense(output_dim,
name="discriminator_y",
activation="sigmoid",
W_regularizer=reg())(h)
return Model([z, x], y, name="discriminator")
def model_generator():
model = Sequential()
nch = 256
reg = lambda: l1l2(l1=1e-7, l2=1e-7)
h = 5
model.add(Dense(input_dim=100, output_dim=nch * 4 * 4,
W_regularizer=reg()))
model.add(BatchNormalization(mode=0))
model.add(Reshape(dim_ordering_shape((nch, 4, 4))))
model.add(
Convolution2D(int(nch / 2),
4,
4,
border_mode='same',
W_regularizer=reg()))
model.add(BatchNormalization(mode=0, axis=1))
model.add(LeakyReLU(0.2))
model.add(UpSampling2D(size=(2, 2)))
model.add(
Convolution2D(int(nch / 2),
h,
h,
border_mode='same',
W_regularizer=reg()))
model.add(BatchNormalization(mode=0, axis=1))
model.add(LeakyReLU(0.2))
model.add(UpSampling2D(size=(2, 2)))
model.add(
Convolution2D(int(nch / 4),
h,
h,
border_mode='same',
W_regularizer=reg()))
model.add(BatchNormalization(mode=0, axis=1))
model.add(LeakyReLU(0.2))
model.add(UpSampling2D(size=(2, 2)))
model.add(Convolution2D(3, h, h, border_mode='same', W_regularizer=reg()))
model.add(Activation('sigmoid'))
return model
def __init__(self, **settings):
# max_features, embedding_dim, seqlen,
# nb_filter, filter_widths, activation, dropout_p,
# l1reg, l2reg, batchnorm,
self.settings = settings
self.settings['verbosity'] = 2
seqlen = self.settings['seqlen']
l1reg = self.settings['l1reg']
l2reg = self.settings['l2reg']
conv_filters = []
for n_gram in settings['filter_widths']:
sequential = Sequential()
conv_filters.append(sequential)
sequential.add(Embedding(input_dim=self.settings['max_features'],
output_dim=self.settings['embedding_dim'],
input_length=seqlen))
sequential.add(Dropout(self.settings['dropout_p']))
sequential.add(Convolution1D(self.settings['nb_filter'],
n_gram,
activation=self.settings['activation']
))
sequential.add(MaxPooling1D(pool_length=seqlen - n_gram + 1))
sequential.add(Flatten())
self.nn = Sequential()
self.nn.add(Merge(conv_filters, mode='concat'))
self.nn.add(Dropout(self.settings['dropout_p']))
if (l1reg is not None and l1reg is float and l2reg is not
None and l2reg is float):
self.nn.add(Dense(1), W_regularizer=l1l2(l1reg, l2reg))
elif (l2reg is not None and l2reg is float):
self.nn.add(Dense(1), W_regularizer=l2(l2reg))
elif (l1reg is not None and l1reg is float):
self.nn.add(Dense(1), W_regularizer=l1(l1reg))
else:
self.nn.add(Dense(1))
if (self.settings['batchnorm'] is True):
self.nn.add(BatchNormalization())
self.nn.add(Activation('sigmoid'))
def build_model(self):
print('Building model...')
self.model = Sequential()
self.model.add(
Convolution2D(32,
3,
3,
border_mode='same',
input_shape=(3, 32, 32)))
self.model.add(LeakyReLU(0.1))
self.model.add(Convolution2D(32, 3, 3, border_mode='same'))
self.model.add(LeakyReLU(0.1))
self.model.add(MaxPooling2D(pool_size=(2, 2)))
self.model.add(Convolution2D(32, 3, 3, border_mode='same'))
self.model.add(LeakyReLU(0.1))
self.model.add(Convolution2D(32, 3, 3, border_mode='same'))
self.model.add(LeakyReLU(0.1))
self.model.add(MaxPooling2D(pool_size=(2, 2)))
self.model.add(Convolution2D(32, 3, 3, border_mode='same'))
self.model.add(LeakyReLU(0.1))
self.model.add(Convolution2D(32, 3, 3, border_mode='same'))
self.model.add(LeakyReLU(0.1))
self.model.add(MaxPooling2D(pool_size=(2, 2)))
self.model.add(Convolution2D(32, 3, 3, border_mode='same'))
self.model.add(LeakyReLU(0.1))
self.model.add(Convolution2D(32, 3, 3, border_mode='same'))
self.model.add(LeakyReLU(0.1))
self.model.add(MaxPooling2D(pool_size=(2, 2)))
self.model.add(Flatten())
self.model.add(Dense(100, W_regularizer=l1l2()))
self.model.add(LeakyReLU(0.1))
self.model.add(Dense(10))
self.model.add(Activation('softmax'))
def create_MP_model(params, input_dims, W_mask=None):
num_model = len(input_dims)
# Create Model
print('Start Build Model...')
model_BP = [Sequential() for i in range(num_model)]
for i in range(num_model):
model_BP[i].add(
Masking(mask_value=1.0,
input_shape=[params['maxlen'], input_dims[i]]))
model_BP[i].add(
TimeDistributedDenseWithWmask(
params['MP_per_model'],
input_dim=input_dims[i],
W_mask=W_mask,
W_regularizer=WeightRegularizerWithPmask(
l2=params['reg_weight'], p_mask=W_mask),
init='lecun_uniform',
activation='relu'))
#model_BP[i].add(MaskedMaxPooling1D(pool_length = 100, stride =10))
model_BP[i].add(TemporalPyramidMaxPooling(tp_layer=params['tp_layer']))
if (num_model > 1):
model = Sequential()
model.add(Merge(model_BP, mode='concat', concat_axis=-1))
else:
model = model_BP[0]
model.add(
Dense(params['nb_classes'],
W_regularizer=l1l2(l1=params['l1_weight'],
l2=params['reg_weight'])))
model.compile(loss=multiclass_hinge,
optimizer=get_optimizer(params),
metrics=["accuracy"])
return model
def create_MP_model(params,input_dims,W_mask=None):
num_model = len(input_dims)
# Create Model
print('Start Build Model...')
model_BP = [Sequential() for i in range(num_model)]
for i in range(num_model):
model_BP[i].add(Masking(mask_value=1.0,input_shape=[params['maxlen'], input_dims[i]]))
model_BP[i].add(TimeDistributedDenseWithWmask(params['MP_per_model'], input_dim=input_dims[i], W_mask=W_mask, W_regularizer=WeightRegularizerWithPmask(l2=params['reg_weight'],p_mask=W_mask),init='lecun_uniform',activation='relu'))
#model_BP[i].add(MaskedMaxPooling1D(pool_length = 100, stride =10))
model_BP[i].add(TemporalPyramidMaxPooling(tp_layer=params['tp_layer']))
if (num_model > 1):
model = Sequential()
model.add(Merge(model_BP,mode='concat',concat_axis = -1))
else:
model = model_BP[0]
model.add(Dense(params['nb_classes'], W_regularizer=l1l2(l1=params['l1_weight'], l2=params['reg_weight'])))
model.compile(loss=multiclass_hinge, optimizer=get_optimizer(params),metrics=["accuracy"])
return model
def train_model_l1(l1penalty, generator):
print('Build model...')
model = Sequential()
model.add(
Dense(1,
input_shape=(input_dim, ),
W_regularizer=l1l2(l1=l1penalty, l2=l1penalty)))
model.add(Activation('sigmoid'))
adam_optimizer = Adam(lr=0.01, decay=0.99)
# try using different optimizers and different optimizer configs
model.compile(loss='binary_crossentropy',
optimizer=adam_optimizer,
metrics=['accuracy'])
############################################################################
model.fit_generator(generator=generator,
nb_worker=1,
nb_epoch=10,
samples_per_epoch=samples_per_epoch)
####################################################
print("MODEL DONE")
return model
def comp_two_path(self):
'''
compiles two-path model, takes in a 4x33x33 patch and assesses global and local paths, then merges the results.
'''
print 'Compiling two-path model...'
model = Graph()
model.add_input(name='input', input_shape=(self.n_chan, 33, 33))
# local pathway, first convolution/pooling
model.add_node(Convolution2D(64, 7, 7, border_mode='valid', activation='relu', W_regularizer=l1l2(l1=0.01, l2=0.01)), name='local_c1', input= 'input')
model.add_node(MaxPooling2D(pool_size=(4,4), strides=(1,1), border_mode='valid'), name='local_p1', input='local_c1')
# local pathway, second convolution/pooling
model.add_node(Dropout(0.5), name='drop_lp1', input='local_p1')
model.add_node(Convolution2D(64, 3, 3, border_mode='valid', activation='relu', W_regularizer=l1l2(l1=0.01, l2=0.01)), name='local_c2', input='drop_lp1')
model.add_node(MaxPooling2D(pool_size=(2,2), strides=(1,1), border_mode='valid'), name='local_p2', input='local_c2')
# global pathway
model.add_node(Convolution2D(160, 13, 13, border_mode='valid', activation='relu', W_regularizer=l1l2(l1=0.01, l2=0.01)), name='global', input='input')
# merge local and global pathways
model.add_node(Dropout(0.5), name='drop_lp2', input='local_p2')
model.add_node(Dropout(0.5), name='drop_g', input='global')
model.add_node(Convolution2D(5, 21, 21, border_mode='valid', activation='relu', W_regularizer=l1l2(l1=0.01, l2=0.01)), name='merge', inputs=['drop_lp2', 'drop_g'], merge_mode='concat', concat_axis=1)
# Flatten output of 5x1x1 to 1x5, perform softmax
model.add_node(Flatten(), name='flatten', input='merge')
model.add_node(Dense(5, activation='softmax'), name='dense_output', input='flatten')
model.add_output(name='output', input='dense_output')
sgd = SGD(lr=0.005, decay=0.1, momentum=0.9)
model.compile('sgd', loss={'output':'categorical_crossentropy'})
print 'Done.'
return model
def compile_model(self):
'''
compiles standard single model with 4 convolitional/max-pooling layers.
'''
print 'Compiling single model...'
single = Sequential()
single.add(Convolution2D(self.n_filters[0], self.k_dims[0], self.k_dims[0], border_mode='valid', W_regularizer=l1l2(l1=self.w_reg, l2=self.w_reg), input_shape=(self.n_chan,33,33)))
single.add(Activation(self.activation))
single.add(BatchNormalization(mode=0, axis=1))
single.add(MaxPooling2D(pool_size=(2,2), strides=(1,1)))
single.add(Dropout(0.5))
single.add(Convolution2D(self.n_filters[1], self.k_dims[1], self.k_dims[1], activation=self.activation, border_mode='valid', W_regularizer=l1l2(l1=self.w_reg, l2=self.w_reg)))
single.add(BatchNormalization(mode=0, axis=1))
single.add(MaxPooling2D(pool_size=(2,2), strides=(1,1)))
single.add(Dropout(0.5))
single.add(Convolution2D(self.n_filters[2], self.k_dims[2], self.k_dims[2], activation=self.activation, border_mode='valid', W_regularizer=l1l2(l1=self.w_reg, l2=self.w_reg)))
single.add(BatchNormalization(mode=0, axis=1))
single.add(MaxPooling2D(pool_size=(2,2), strides=(1,1)))
single.add(Dropout(0.5))
single.add(Convolution2D(self.n_filters[3], self.k_dims[3], self.k_dims[3], activation=self.activation, border_mode='valid', W_regularizer=l1l2(l1=self.w_reg, l2=self.w_reg)))
single.add(Dropout(0.25))
single.add(Flatten())
single.add(Dense(5))
single.add(Activation('softmax'))
sgd = SGD(lr=0.001, decay=0.01, momentum=0.9)
single.compile(loss='categorical_crossentropy', optimizer='sgd')
print 'Done.'
return single
def build_model(in_dim, out_dim=1, n_hidden=100, l1_norm=0.0,
l2_norm=0, n_deep=5, drop=0.1,
learning_rate=0.1, optimizer='Adadelta',
activation='tanh'):
model = Sequential()
# Input layer
model.add(Dense(
input_dim=in_dim,
output_dim=n_hidden,
init='uniform',
activation=activation,
W_regularizer=l1l2(l1=l1_norm, l2=l2_norm)))
# do X layers
for layer in range(n_deep-1):
model.add(Dropout(drop))
model.add(Dense(
output_dim=np.round(n_hidden/2**(layer+1)),
init='uniform',
activation=activation))
# Output layer
if out_dim == 1:
activation = activation
else:
activation = 'softmax'
model.add(Dense(out_dim,
init='uniform',
activation=activation))
# Optimization algorithms
if optimizer == 'Adadelta':
if learning_rate is None:
opt = Adadelta()
else:
opt = Adadelta(lr=learning_rate)
elif optimizer =='SGD':
if learning_rate is None:
opt = SGD(lr=learning_rate)
else:
opt = SGD()
elif optimizer == 'RMSprop':
if learning_rate is None:
opt = RMSprop(lr=learning_rate)
else:
opt = RMSprop()
elif optimizer == 'Adagrad':
if learning_phase is None:
opt = Adagrad(lr=learning_rate)
else:
opt = Adagrad()
elif optimizer == 'Adam':
if learning_rate is None:
opt = Adam(lr=learning_rate)
else:
opt = Adam()
elif optimizer == 'Adamax':
if learning_rate is None:
opt = Adamax(lr=learning_rate)
else:
opt = Adamax()
else:
logger.info('Optimizer {} not defined, using Adadelta'.format(optimizer))
opt = Adadelta(lr=learning_rate)
if out_dim == 1:
model.compile(loss='binary_crossentropy',
optimizer=opt)
else:
model.compile(loss='categorical_crossentropy',
optimizer=opt)
return model
def build_model(train):
input_dim = train.shape[1]
model = Sequential()
model.add(Dense(15, input_dim = input_dim, init = 'glorot_normal', activation='tanh', W_regularizer=l1l2(l1 = 1e-4, l2 = 1e-4)))
model.add(Dropout(0.15))
model.add(Dense(1, init = 'zero', activation='linear'))
model.compile(loss = "mse", optimizer = "adagrad")
return model
def build_model(
in_dim,
out_dim=1,
n_hidden=100,
l1_norm=0.0,
l2_norm=0,
n_deep=5,
drop=0.1,
learning_rate=0.1,
optimizer="Adadelta",
activation="tanh",
n_class=1,
):
model = Sequential()
# Input layer
model.add(
Dense(
input_dim=in_dim,
output_dim=n_hidden,
init="uniform",
activation=activation,
W_regularizer=l1l2(l1=l1_norm, l2=l2_norm),
)
)
# do X layers
for layer in range(n_deep - 1):
model.add(Dropout(drop))
model.add(Dense(output_dim=n_hidden, init="uniform", activation=activation)) # np.round(n_hidden/2**(layer+1)),
# Output layer
if out_dim == 1:
activation = activation
elif n_class == 1 and self.n_label > 2:
activation = "softmax"
elif n_class > 1:
activation = "sigmoid"
model.add(Dense(out_dim, init="uniform", activation=activation))
# Optimization algorithms
if optimizer == "Adadelta":
if learning_rate is None:
opt = Adadelta()
else:
opt = Adadelta(lr=learning_rate)
elif optimizer == "SGD":
if learning_rate is None:
opt = SGD()
else:
opt = SGD(lr=learning_rate)
elif optimizer == "RMSprop":
if learning_rate is None:
opt = RMSprop()
else:
opt = RMSprop(lr=learning_rate)
elif optimizer == "Adagrad":
if learning_rate is None:
opt = Adagrad()
else:
opt = Adagrad(lr=learning_rate)
elif optimizer == "Adam":
if learning_rate is None:
opt = Adam()
else:
opt = Adam(lr=learning_rate)
elif optimizer == "Adamax":
if learning_rate is None:
opt = Adamax()
else:
opt = Adamax(lr=learning_rate)
else:
logger.info("Optimizer {} not defined, using Adadelta".format(optimizer))
opt = Adadelta(lr=learning_rate)
if out_dim == 1:
model.compile(loss="binary_crossentropy", optimizer=opt)
else:
model.compile(loss="categorical_crossentropy", optimizer=opt)
return model
X_train[row, n, dictionary.index(tri)] = 1.0
for row, (word, tag) in enumerate(test_tri):
for n, w in enumerate(word):
for tri in w:
X_test[row, n, dictionary.index(tri)] = 1.0
# Predictive model
y_train = np.zeros((len(train), 2), dtype=np.float32)
y_test = np.zeros((len(test), 2), dtype=np.float32)
for row, (tri, tag) in enumerate(train_tri):
y_train[row, 1 if tag=='pos' else 0] = 1.0
for row, (tri, tag) in enumerate(test_tri):
y_test[row, 1 if tag=='pos' else 0] = 1.0
# http://research.microsoft.com/apps/pubs/default.aspx?id=256230
model = Sequential()
model.add(Convolution1D(1000, 3, activation='tanh', W_regularizer=l1l2(l1=0.01, l2=0.01), b_regularizer=l1l2(l1=0.01, l2=0.01), init='uniform', border_mode='same', input_shape=(max_len,len(dictionary))))
model.add(MaxPooling1D(pool_length=max_len, border_mode='valid'))
model.add(Flatten())
model.add(Dense(300, activation='tanh', W_regularizer=l1l2(l1=0.01, l2=0.01), b_regularizer=l1l2(l1=0.01, l2=0.01)))
model.add(CosSim(2, activation='linear', bias=False, W_regularizer=l1l2(l1=0.01, l2=0.01)))
model.add(Activation(activation='softmax'))
def PredictiveLoss(y_true, y_pred):
return -K.mean(K.log(y_true * y_pred), axis=-1)
model.compile(optimizer='Adagrad',
loss=PredictiveLoss,
metrics=['accuracy'])
model.fit(X_train, y_train, validation_data=(X_test, y_test), batch_size=2, nb_epoch=30)
loss, acc = model.evaluate(X_test, y_test)
#plot(model, to_file='cdssm_pred_model.png')
def constructModel(self, *args, **kwargs):
print("Model Compilation Start...")
############## Initialization ################
convInit = "he_normal"
############## Regularization ################
reg = KR.l1l2(l1=0.0002, l2=0.0002)
#reg = None
############## Optimizer ################
baseLr = 0.001
#opt = KO.SGD (lr=0.0001, decay=1e-6, momentum=0.9, nesterov=True)
#opt = KO.RMSprop ()
#opt = KO.Adagrad ()
#opt = KO.Adadelta()
opt = KO.Adam (lr=baseLr, beta_1=0.9, beta_2=0.999, epsilon=1e-8, clipvalue=5)
#opt = KO.Adamax (lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-8)
opt.baseLr = baseLr
opt.lrDecay = 0.85
############### Structural #################
f1 = 48;
f2_s1 = 16; f2_e1 = 32; f2_e3 = 32;
f3_s1 = 16; f3_e1 = 32; f3_e3 = 32;
f4_s1 = 32; f4_e1 = 64; f4_e3 = 64;
f5_s1 = 32; f5_e1 = 64; f5_e3 = 64;
f6_s1 = 48; f6_e1 = 96; f6_e3 = 96;
f7_s1 = 48; f7_e1 = 96; f7_e3 = 96;
f8_s1 = 64; f8_e1 = 128; f8_e3 = 128;
f9_s1 = 64; f9_e1 = 128; f9_e3 = 128;
f10 = 2;
############### Model #################
model = KM.Graph()
model.add_input ("input", (3,192,192), (50,3,192,192))
model.add_node (KLCv.AveragePooling2D ((2,2), (2,2), "valid"), "input_medium", input="input")
model.add_node (KLCv.AveragePooling2D ((2,2), (2,2), "valid"), "input_coarse", input="input_medium")
model.add_node (KLCv.Convolution2D (f1, 7, 7, border_mode="same", subsample=(1,1), init=convInit, W_regularizer=reg), "conv1_fine/act", input="input")
model.add_node (KLCv.Convolution2D (f1, 7, 7, border_mode="same", subsample=(1,1), init=convInit, W_regularizer=reg), "conv1_medium/act", input="input_medium")
model.add_node (KLCv.Convolution2D (f1, 7, 7, border_mode="same", subsample=(1,1), init=convInit, W_regularizer=reg), "conv1_coarse/act", input="input_coarse")
model.add_node (KLCv.MaxPooling2D ((4,4), (4,4), "same"), "finepool/out", input="conv1_fine/act")
model.add_node (KLCv.MaxPooling2D ((2,2), (2,2), "same"), "mediumpool/out", input="conv1_medium/act")
model.add_node (KLN. BatchNormalization(axis=1), "bn1/out", inputs=["finepool/out", "mediumpool/out", "conv1_coarse/act"], concat_axis=1)
model.add_node (KLCo.Activation ("relu"), "conv1/out", input="bn1/out")
model.add_node (KLCv.Convolution2D (f2_s1, 1, 1, border_mode="same", init=convInit, W_regularizer=reg), "fire2/comp/act", input="conv1/out")
model.add_node (KLN. BatchNormalization(axis=1), "bn2c/out", input="fire2/comp/act")
model.add_node (KLCo.Activation ("relu"), "fire2/comp/out", input="bn2c/out")
model.add_node (KLCv.Convolution2D (f2_e1, 1, 1, border_mode="same", init=convInit, W_regularizer=reg), "fire2/exp1/act", input="fire2/comp/out")
model.add_node (KLCv.Convolution2D (f2_e3, 3, 3, border_mode="same", init=convInit, W_regularizer=reg), "fire2/exp3/act", input="fire2/comp/out")
model.add_node (KLCo.Activation ("relu"), "fire2/exp/out", inputs=["fire2/exp1/act", "fire2/exp3/act"], concat_axis=1)
model.add_node (KLCv.Convolution2D (f3_s1, 1, 1, border_mode="same", init=convInit, W_regularizer=reg), "fire3/comp/act", input="fire2/exp/out")
model.add_node (KLN. BatchNormalization(axis=1), "bn3c/out", input="fire3/comp/act")
model.add_node (KLCo.Activation ("relu"), "fire3/comp/out", input="bn3c/out")
model.add_node (KLCv.Convolution2D (f3_e1, 1, 1, border_mode="same", init=convInit, W_regularizer=reg), "fire3/exp1/act", input="fire3/comp/out")
model.add_node (KLCv.Convolution2D (f3_e3, 3, 3, border_mode="same", init=convInit, W_regularizer=reg), "fire3/exp3/act", input="fire3/comp/out")
model.add_node (KLCo.Activation ("relu"), "fire3/exp/out", inputs=["fire3/exp1/act", "fire3/exp3/act"], concat_axis=1)
model.add_node (KLCv.Convolution2D (f4_s1, 1, 1, border_mode="same", init=convInit, W_regularizer=reg), "fire4/comp/act", input="fire3/exp/out")
model.add_node (KLN. BatchNormalization(axis=1), "bn4c/out", input="fire4/comp/act")
model.add_node (KLCo.Activation ("relu"), "fire4/comp/out", input="bn4c/out")
model.add_node (KLCv.Convolution2D (f4_e1, 1, 1, border_mode="same", init=convInit, W_regularizer=reg), "fire4/exp1/act", input="fire4/comp/out")
model.add_node (KLCv.Convolution2D (f4_e3, 3, 3, border_mode="same", init=convInit, W_regularizer=reg), "fire4/exp3/act", input="fire4/comp/out")
model.add_node (KLCo.Activation ("relu"), "fire4/exp/out", inputs=["fire4/exp1/act", "fire4/exp3/act"], concat_axis=1)
model.add_node (KLCv.MaxPooling2D ((3,3), (2,2), "same"), "maxpool4/out", input="fire4/exp/out")
model.add_node (KLCv.Convolution2D (f5_s1, 1, 1, border_mode="same", init=convInit, W_regularizer=reg), "fire5/comp/act", input="maxpool4/out")
model.add_node (KLN. BatchNormalization(axis=1), "bn5c/out", input="fire5/comp/act")
model.add_node (KLCo.Activation ("relu"), "fire5/comp/out", input="bn5c/out")
model.add_node (KLCv.Convolution2D (f5_e1, 1, 1, border_mode="same", init=convInit, W_regularizer=reg), "fire5/exp1/act", input="fire5/comp/out")
model.add_node (KLCv.Convolution2D (f5_e3, 3, 3, border_mode="same", init=convInit, W_regularizer=reg), "fire5/exp3/act", input="fire5/comp/out")
model.add_node (KLCo.Activation ("relu"), "fire5/exp/out", inputs=["fire5/exp1/act", "fire5/exp3/act"], concat_axis=1)
model.add_node (KLCv.Convolution2D (f6_s1, 1, 1, border_mode="same", init=convInit, W_regularizer=reg), "fire6/comp/act", input="fire5/exp/out")
model.add_node (KLN. BatchNormalization(axis=1), "bn6c/out", input="fire6/comp/act")
model.add_node (KLCo.Activation ("relu"), "fire6/comp/out", input="bn6c/out")
model.add_node (KLCv.Convolution2D (f6_e1, 1, 1, border_mode="same", init=convInit, W_regularizer=reg), "fire6/exp1/act", input="fire6/comp/out")
model.add_node (KLCv.Convolution2D (f6_e3, 3, 3, border_mode="same", init=convInit, W_regularizer=reg), "fire6/exp3/act", input="fire6/comp/out")
model.add_node (KLCo.Activation ("relu"), "fire6/exp/out", inputs=["fire6/exp1/act", "fire6/exp3/act"], concat_axis=1)
model.add_node (KLCv.Convolution2D (f7_s1, 1, 1, border_mode="same", init=convInit, W_regularizer=reg), "fire7/comp/act", input="fire6/exp/out")
model.add_node (KLN. BatchNormalization(axis=1), "bn7c/out", input="fire7/comp/act")
model.add_node (KLCo.Activation ("relu"), "fire7/comp/out", input="bn7c/out")
model.add_node (KLCv.Convolution2D (f7_e1, 1, 1, border_mode="same", init=convInit, W_regularizer=reg), "fire7/exp1/act", input="fire7/comp/out")
model.add_node (KLCv.Convolution2D (f7_e3, 3, 3, border_mode="same", init=convInit, W_regularizer=reg), "fire7/exp3/act", input="fire7/comp/out")
model.add_node (KLCo.Activation ("relu"), "fire7/exp/out", inputs=["fire7/exp1/act", "fire7/exp3/act"], concat_axis=1)
model.add_node (KLCv.Convolution2D (f8_s1, 1, 1, border_mode="same", init=convInit, W_regularizer=reg), "fire8/comp/act", input="fire7/exp/out")
model.add_node (KLN. BatchNormalization(axis=1), "bn8c/out", input="fire8/comp/act")
model.add_node (KLCo.Activation ("relu"), "fire8/comp/out", input="bn8c/out")
model.add_node (KLCv.Convolution2D (f8_e1, 1, 1, border_mode="same", init=convInit, W_regularizer=reg), "fire8/exp1/act", input="fire8/comp/out")
model.add_node (KLCv.Convolution2D (f8_e3, 3, 3, border_mode="same", init=convInit, W_regularizer=reg), "fire8/exp3/act", input="fire8/comp/out")
model.add_node (KLCo.Activation ("relu"), "fire8/exp/out", inputs=["fire8/exp1/act", "fire8/exp3/act"], concat_axis=1)
model.add_node (KLCv.MaxPooling2D ((3,3), (2,2), "same"), "maxpool8/out", input="fire8/exp/out")
model.add_node (KLCv.Convolution2D (f9_s1, 1, 1, border_mode="same", init=convInit, W_regularizer=reg), "fire9/comp/act", input="maxpool8/out")
model.add_node (KLN. BatchNormalization(axis=1), "bn9c/out", input="fire9/comp/act")
model.add_node (KLCo.Activation ("relu"), "fire9/comp/out", input="bn9c/out")
model.add_node (KLCv.Convolution2D (f9_e1, 1, 1, border_mode="same", init=convInit, W_regularizer=reg), "fire9/exp1/act", input="fire9/comp/out")
model.add_node (KLCv.Convolution2D (f9_e3, 3, 3, border_mode="same", init=convInit, W_regularizer=reg), "fire9/exp3/act", input="fire9/comp/out")
model.add_node (KLCo.Activation ("relu"), "fire9/exp/out", inputs=["fire9/exp1/act", "fire9/exp3/act"], concat_axis=1)
model.add_node (KLCv.Convolution2D (f10 , 1, 1, border_mode="same", init=convInit, W_regularizer=reg), "conv10/act", input="fire9/exp/out")
model.add_node (KLN. BatchNormalization(axis=1), "bn10/out", input="conv10/act")
model.add_node (KLCo.Activation ("relu"), "conv10/out", input="bn10/out")
model.add_node (KLCv.AveragePooling2D ((12,12), (1,1), "valid"), "avgpool10/out", input="conv10/out")
model.add_node (KLCo.Reshape ((f10,)), "softmax/in", input="avgpool10/out")
model.add_node (KLCo.Activation ("softmax"), "softmax/out", input="softmax/in")
model.add_output("output", "softmax/out")
model.compile(loss={"output":'categorical_crossentropy'}, optimizer=opt)
#model.powerf = T.function()
KUV.plot(model, to_file='model.png', show_shape=True)
print("Model Compilation End.")
#pdb.set_trace()
return model
logger.info('samples_per_epoch:%s' % args.samples_per_epoch)
logger.info('use_per_class_threshold_tuning:%s' % args.use_per_class_threshold_tuning)
logger.info('evaluation_re_try_times:%s' % args.evaluation_re_try_times)
logger.info('evaluation_re_try_waiting_time:%s' % args.evaluation_re_try_waiting_time)
logger.info('fail_on_evlauation_failure:%s' % args.fail_on_evlauation_failure)
if args.weight_regularizer_hidden == 'none':
logger.info('weight_regularizer_hidden:No')
weight_regularizer_hidden = None
elif args.weight_regularizer_hidden == 'l1l2':
logger.info('weight_regularizer_hidden:%s, l1_regularizer_weight_hidden:%s,l2_regularizer_weight_hidden:%s' % (
args.weight_regularizer_hidden, args.l1_regularizer_weight_hidden, args.l2_regularizer_weight_hidden))
from keras.regularizers import l1l2
weight_regularizer_hidden = l1l2(l1=args.l1_regularizer_weight_hidden, l2=args.l2_regularizer_weight_hidden)
elif args.weight_regularizer_hidden == 'l1':
logger.info('weight_regularizer_hidden:%s, l1_regularizer_weight_hidden:%s' % (
args.weight_regularizer_hidden, args.l1_regularizer_weight_hidden))
from keras.regularizers import l1
weight_regularizer_hidden = l1(l=args.l1_regularizer_weight_hidden)
elif args.weight_regularizer_hidden == 'l2':
logger.info('weight_regularizer_hidden:%s, l2_regularizer_weight_hidden:%s' % (
args.weight_regularizer_hidden, args.l2_regularizer_weight_hidden))
from keras.regularizers import l2
weight_regularizer_hidden = l2(l=args.l2_regularizer_weight_hidden)
if weight_regularizer_hidden:
weight_regularizer_hidden = weight_regularizer_hidden.get_config()
#weight_regularizer_proj
