def model(data,
hidden_layers,
hidden_neurons,
output_file,
validation_split=0.9):
train_n = int(validation_split * len(data))
batch_size = 50
train_data = data[:train_n, :]
val_data = data[train_n:, :]
#data: data_num * data_dim
input_sh = Input(shape=(data.shape[1], ))
encoded = Dense(data.shape[1],
activation='relu',
activity_regularizer=regularizers.activity_l1l2(
10e-5, 10e-5))(input_sh)
encoded = noise.GaussianNoise(0.2)(encoded)
for i in range(hidden_layers):
encoded = Dense(hidden_neurons[i],
activation='relu',
activity_regularizer=regularizers.activity_l1l2(
10e-5, 10e-5))(encoded)
encoded = noise.GaussianNoise(0.2)(encoded)
for j in range(hidden_layers - 1, -1, -1):
decoded = Dense(hidden_neurons[i],
activation='relu',
activity_regularizer=regularizers.activity_l1l2(
10e-5, 10e-5))(encoded)
decoded = Dense(data.shape[1], activation='sigmoid')(decoded)
autoencoder = Model(input=input_sh, output=decoded)
autoencoder.compile(optimizer='adadelta', loss='mse')
checkpointer = ModelCheckpoint(filepath='data/bestmodel' + output_file +
".hdf5",
verbose=1,
save_best_only=True)
earlystopper = EarlyStopping(monitor='val_loss', patience=15, verbose=1)
train_generator = DataGenerator(batch_size)
train_generator.fit(train_data, train_data)
val_generator = DataGenerator(batch_size)
val_generator.fit(val_data, val_data)
autoencoder.fit_generator(train_generator,
samples_per_epoch=len(train_data),
nb_epoch=100,
validation_data=val_generator,
nb_val_samples=len(val_data),
max_q_size=batch_size,
callbacks=[checkpointer, earlystopper])
enco = Model(input=input_sh, output=encoded)
enco.compile(optimizer='adadelta', loss='mse')
reprsn = enco.predict(data)
return reprsn
def nips_conv(num_cells):
"""Hard-coded model for NIPS"""
layers = list()
input_shape = (40, 50, 50)
# injected noise strength
sigma = 0.1
# convolutional layer sizes
convlayers = [(16, 15), (8, 9)]
# l2_weight_regularization for every layer
l2_weight = 1e-3
# weight and activity regularization
W_reg = [(0., l2_weight), (0., l2_weight)]
act_reg = [(0., 0.), (0., 0.)]
# loop over convolutional layers
for (n, size), w_args, act_args in zip(convlayers, W_reg, act_reg):
args = (n, size, size)
kwargs = {
'border_mode': 'valid',
'subsample': (1, 1),
'init': 'normal',
'W_regularizer': l1l2(*w_args),
'activity_regularizer': activity_l1l2(*act_args),
}
if len(layers) == 0:
kwargs['input_shape'] = input_shape
# add convolutional layer
layers.append(Convolution2D(*args, **kwargs))
# add gaussian noise
layers.append(GaussianNoise(sigma))
# add ReLu
layers.append(Activation('relu'))
# flatten
layers.append(Flatten())
# Add a final dense (affine) layer
layers.append(Dense(num_cells, init='normal',
W_regularizer=l1l2(0., l2_weight),
activity_regularizer=activity_l1l2(1e-3, 0.)))
# Finish it off with a parameterized softplus
layers.append(ParametricSoftplus())
return layers
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)
model = Sequential()
model.add(
Convolution2D(
8,
nb_conv,
nb_conv,
border_mode='same',
input_shape=(1, img_rows, img_cols),
# W_regularizer = l1l2ld(l1 = 0., l2 = 0., ld = 0.),
W_regularizer=l2(l=0.),
b_regularizer=l2(l=0.),
activity_regularizer=activity_l1l2(l1=0., l2=0.)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
# model.add(AveragePooling2D(pool_size=(nb_pool, nb_pool)))
model.add(
Convolution2D(
16,
nb_conv,
nb_conv,
border_mode='same',
input_shape=(1, img_rows, img_cols),
# W_regularizer = l1l2ld(l1 = 0., l2 = 0., ld = 0.),
W_regularizer=l2(l=0.),
b_regularizer=l2(l=0.),
activity_regularizer=activity_l1l2(l1=0., l2=0.)))
model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
def main():
parser = argparser()
args = parser.parse_args()
weight_path = args.weight_path
layer = args.layer
img_rows, img_cols = 32, 32
# the CIFAR10 images are RGB
img_channels = 3
batch_size = 32
nb_classes = 10
model = Sequential()
print("Making model")
model = Sequential()
model.add(Dense(512, input_shape=(3072,), activity_regularizer = activity_l1l2(l1 = 0., l2 = 0.),
W_regularizer = l2(l = 0.), b_regularizer = l2(l = 0.)))
model.add(Activation('relu'))
# model.add(Dropout(0.2))
model.add(Dense(1024, activity_regularizer = activity_l1l2(l1 = 0., l2 = 0.),
W_regularizer = l2(l = 0.), b_regularizer = l2(l = 0.)))
model.add(Activation('relu'))
model.add(Dense(1024, activity_regularizer = activity_l1l2(l1 = 0., l2 = 0.),
W_regularizer = l2(l = 0.), b_regularizer = l2(l = 0.)))
model.add(Activation('relu'))
model.add(Dense(2048, activity_regularizer = activity_l1l2(l1 = 0., l2 = 0.),
W_regularizer = l2(l = 0.), b_regularizer = l2(l = 0.)))
model.add(Activation('relu'))
model.add(Dense(2048, activity_regularizer = activity_l1l2(l1 = 0., l2 = 0.),
W_regularizer = l2(l = 0.), b_regularizer = l2(l = 0.)))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(10, activity_regularizer = activity_l1l2(l1 = 0., l2 = 0.),
W_regularizer = l2(l = 0.), b_regularizer = l2(l = 0.)))
model.add(Activation('softmax'))
# let's train the model using SGD + momentum (how original).
print("Compiling model")
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
print("Going to visualize layer ", layer)
print(model.layers[layer].get_config())
# load learned weight
print("Loading weight")
model.load_weights(weight_path)
weight = model.layers[0].get_weights()
print("shape of weight: ", weight[0].shape)
# generate function to get output at layer to be visualized
for i in range(len(model.layers)):
print(i)
input = model.layers[0].input
output = model.layers[layer].output
func = K.function([K.learning_phase()] + [input], output)
(X_train, y_train), (X_test, y_test) = cifar10.load_data()
# im = X_train[100, :, :, :]
# im = np.swapaxes(im, 0, 2)
# im = np.swapaxes(im, 0, 1)
# plt.figure(1)
# plt.imshow(im)
# plt.show()
# sys.exit()
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_test = X_test.reshape(10000, 3072)
X_train /= 255
X_test /= 255
print(X_test.shape[0], 'test samples')
crop_x = X_test
# crop_x = random_crop(X_test, size = (9, 9), times = 10)
print("shape of crop_x: ", crop_x.shape)
im = crop_x[0, :]
# print("crop_x[0]", im)
im = im * 255
im = im.astype(np.uint8)
# print("im of uint8: ", im)
fig = plt.figure()
# plt.imshow(im)
# plt.show()
# sys.exit()
# get output from layer to be visualized
# print(X_test[50][1])
activation = func([0] + [crop_x])
print("shape of activation: ", activation.shape)
max_sample_index = np.argmax(activation, axis = 0)
max_sample_index = max_sample_index.squeeze()
np.savetxt("max_sample_index", max_sample_index, fmt = "%d")
print("shape of max_sample_index: ", max_sample_index.shape)
# print("max_29", activation[:, 29, :, :])
# for i in range(32):
# ax = fig.add_subplot(8, 4, i + 1, frameon=False)
# ax.set_xticks([])
# ax.set_yticks([])
# ax.xaxis.set_ticks_position('none')
# ax.yaxis.set_ticks_position('none')
# im = crop_x[max_sample_index[i], :]
# im = im.reshape((3, 32, 32))
# im = np.swapaxes(im, 0, 2)
# im = np.swapaxes(im, 1, 0)
# # print("shape of im: ", im.shape)
# im = im * 255
# im = im.astype(np.uint8)
# ax.imshow(im)
# plt.show()
if activation.ndim == 4:
num = activation.shape[0]
print("num: ", num)
col = activation.shape[1]
print("col: ", col)
map_size = activation.shape[2] * activation.shape[3]
print("map_size: ", map_size)
flatten_activation = np.reshape(activation, (num, col * map_size))
print("shape of flatten_activation: ", flatten_activation.shape)
trans_activation = flatten_activation.transpose()
print("shape of trans_activation: ", trans_activation.shape)
reshape_activation = np.reshape(trans_activation, (col, num * map_size))
print("shape of reshape_activation: ", reshape_activation.shape)
matrix_activation = reshape_activation.transpose()
print("shape of matrix_activation: ", matrix_activation.shape)
mean = np.mean(matrix_activation, axis = 0, keepdims = True)
# mean_p = T.printing.Print('mean')(mean)
std = np.std(matrix_activation, axis = 0, keepdims = True)
normalized_output = (matrix_activation - mean) / std
covariance = np.dot(np.transpose(normalized_output), normalized_output) / num / map_size
else:
num = activation.shape[0]
mean = np.mean(activation, axis = 0, keepdims = True)
# mean_p = T.printing.Print('mean')(mean)
std = np.std(activation, axis = 0, keepdims = True)
normalized_output = (activation - mean) / std
covariance = np.dot(np.transpose(normalized_output), normalized_output) / num
np.savetxt("mean", mean, fmt = "%f")
np.savetxt("std", std, fmt = "%f")
np.savetxt("covariance", covariance, fmt = "%f")
def AlexNet(outdim=1000, weights_path=None, heatmap=False, l1=0, l2=0, usemil=False, usemymil=False, k=1., usemysoftmil=False, softmink=1., softmaxk=1.,\
sparsemil=False, sparsemill1=0., sparsemill2=0., saveact=False):
l1factor = l1
l2factor = l2
if heatmap:
inputs = Input(shape=(3, None, None))
else:
inputs = Input(shape=(3, 227, 227))
conv_1 = Convolution2D(96,
11,
11,
subsample=(4, 4),
activation='relu',
W_regularizer=l1l2(l1=l1factor, l2=l2factor),
name='conv_1')(inputs)
conv_2 = MaxPooling2D((3, 3), strides=(2, 2))(conv_1)
conv_2 = crosschannelnormalization(name="convpool_1")(conv_2)
conv_2 = ZeroPadding2D((2, 2))(conv_2)
conv_2 = merge([
Convolution2D(128,
5,
5,
activation="relu",
name='conv_2_' + str(i + 1),
W_regularizer=l1l2(l1=l1factor, l2=l2factor))(
splittensor(ratio_split=2, id_split=i)(conv_2))
for i in range(2)
],
mode='concat',
concat_axis=1,
name="conv_2")
conv_3 = MaxPooling2D((3, 3), strides=(2, 2))(conv_2)
conv_3 = crosschannelnormalization()(conv_3)
conv_3 = ZeroPadding2D((1, 1))(conv_3)
conv_3 = Convolution2D(384,
3,
3,
activation='relu',
name='conv_3',
W_regularizer=l1l2(l1=l1factor,
l2=l2factor))(conv_3)
conv_4 = ZeroPadding2D((1, 1))(conv_3)
conv_4 = merge([
Convolution2D(192,
3,
3,
activation="relu",
name='conv_4_' + str(i + 1),
W_regularizer=l1l2(l1=l1factor, l2=l2factor))(
splittensor(ratio_split=2, id_split=i)(conv_4))
for i in range(2)
],
mode='concat',
concat_axis=1,
name="conv_4")
conv_5 = ZeroPadding2D((1, 1))(conv_4)
conv_5 = merge([
Convolution2D(128,
3,
3,
activation="relu",
name='conv_5_' + str(i + 1),
W_regularizer=l1l2(l1=l1factor, l2=l2factor))(
splittensor(ratio_split=2, id_split=i)(conv_5))
for i in range(2)
],
mode='concat',
concat_axis=1,
name="conv_5")
dense_1 = MaxPooling2D((3, 3), strides=(2, 2), name="convpool_5")(conv_5)
if heatmap:
dense_1 = Convolution2D(4096,
6,
6,
activation="relu",
name="dense_1",
W_regularizer=l1l2(l1=l1factor,
l2=l2factor))(dense_1)
dense_2 = Convolution2D(4096,
1,
1,
activation="relu",
name="dense_2",
W_regularizer=l1l2(l1=l1factor,
l2=l2factor))(dense_1)
dense_3 = Convolution2D(outdim,
1,
1,
name="dense_3",
W_regularizer=l1l2(l1=l1factor,
l2=l2factor))(dense_2)
prediction = Softmax4D(axis=1, name="softmax")(dense_3)
elif usemil:
dense_1 = Convolution2D(128,
1,
1,
activation='relu',
name='mil_1',
W_regularizer=l1l2(l1=l1factor,
l2=l2factor))(dense_1)
dense_2 = Convolution2D(128,
1,
1,
activation='relu',
name='mil_2',
W_regularizer=l1l2(l1=l1factor,
l2=l2factor))(dense_1)
dense_3 = Convolution2D(outdim,
1,
1,
name='mil_3',
W_regularizer=l1l2(l1=l1factor,
l2=l2factor))(dense_2)
prediction_1 = Softmax4D(axis=1, name='softmax')(dense_3)
#prediction = Flatten(name='flatten')(prediction_1)
#dense_3 = Dense(outdim,name='dense_3',W_regularizer=l1l2(l1=l1factor, l2=l2factor))(prediction)
#prediction = Activation("softmax",name="softmax2")(dense_3)
prediction_1 = MaxPooling2D((6, 6), name='output')(prediction_1)
prediction = Flatten(name='flatten')(prediction_1)
prediction = Recalc(axis=1, name='Recalcmil')(prediction)
elif usemymil:
dense_1 = Convolution2D(128,
1,
1,
activation='relu',
name='mil_1',
W_regularizer=l1l2(l1=l1factor,
l2=l2factor))(dense_1)
dense_2 = Convolution2D(128,
1,
1,
activation='relu',
name='mil_2',
W_regularizer=l1l2(l1=l1factor,
l2=l2factor))(dense_1)
dense_3 = Convolution2D(1,
1,
1,
activation='sigmoid',
name='mil_3',
W_regularizer=l1l2(l1=l1factor,
l2=l2factor))(dense_2)
#prediction_1 = Softmax4D(axis=1, name='softmax')(dense_3)
#prediction = ExtractDim(axis=1, name='extract')(prediction_1)
prediction = Flatten(name='flatten')(dense_3)
prediction = ReRank(k=k, label=1, name='output')(prediction)
elif usemysoftmil:
dense_1 = Convolution2D(128,
1,
1,
activation='relu',
name='mil_1',
W_regularizer=l1l2(l1=l1factor,
l2=l2factor))(dense_1)
dense_2 = Convolution2D(128,
1,
1,
activation='relu',
name='mil_2',
W_regularizer=l1l2(l1=l1factor,
l2=l2factor))(dense_1)
dense_3 = Convolution2D(1,
1,
1,
activation='sigmoid',
name='mil_3',
W_regularizer=l1l2(l1=l1factor,
l2=l2factor))(dense_2)
#prediction_1 = Softmax4D(axis=1, name='softmax')(dense_3)
#prediction = ExtractDim(axis=1, name='extract')(prediction_1)
prediction = Flatten(name='flatten')(dense_3)
prediction = SoftReRank(softmink=softmink,
softmaxk=softmaxk,
label=1,
name='output')(prediction)
elif sparsemil:
dense_1 = Convolution2D(128,
1,
1,
activation='relu',
name='mil_1',
W_regularizer=l1l2(l1=l1factor,
l2=l2factor))(dense_1)
dense_2 = Convolution2D(128,
1,
1,
activation='relu',
name='mil_2',
W_regularizer=l1l2(l1=l1factor,
l2=l2factor))(dense_1)
prediction_1 = Convolution2D(1,1,1,activation='sigmoid',name='mil_3',W_regularizer=l1l2(l1=l1factor, l2=l2factor),\
activity_regularizer=activity_l1l2(l1=sparsemill1, l2=sparsemill2))(dense_2)
# prediction_1 = Softmax4D(axis=1, name='softmax')(prediction_1)
#dense_3 = Convolution2D(outdim,1,1,name='mil_3',W_regularizer=l1l2(l1=l1factor, l2=l2factor))(dense_2)
#prediction_1 = Softmax4D(axis=1, name='softmax')(dense_3)
#prediction_1 = ActivityRegularizerOneDim(l1=sparsemill1, l2=sparsemill2)(prediction_1)
#prediction = MaxPooling2D((6,6), name='output')(prediction_1)
# prediction_1 = Convolution2D(1, 3, 3, activation='sigmoid', border_mode='same', name='smooth', \
# W_regularizer=l1l2(l1=l1factor, l2=l2factor), activity_regularizer=activity_l1l2(l1=sparsemill1, l2=sparsemill2))(prediction_1)
prediction = Flatten(name='flatten')(prediction_1)
if saveact:
model = Model(input=inputs, output=prediction)
return model
prediction = RecalcExpand(axis=1, name='Recalcmil')(prediction)
else:
dense_1 = Flatten(name="flatten")(dense_1)
dense_1 = Dense(4096,
activation='relu',
name='dense_1',
W_regularizer=l1l2(l1=l1factor, l2=l2factor))(dense_1)
dense_2 = Dropout(0.5)(dense_1)
dense_2 = Dense(4096,
activation='relu',
name='dense_2',
W_regularizer=l1l2(l1=l1factor, l2=l2factor))(dense_2)
dense_3 = Dropout(0.5)(dense_2)
dense_3 = Dense(outdim,
name='dense_3',
W_regularizer=l1l2(l1=l1factor, l2=l2factor))(dense_3)
prediction = Activation("softmax", name="softmax")(dense_3)
model = Model(input=inputs, output=prediction)
if weights_path:
model.load_weights(weights_path)
return model
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')
# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)
model = Sequential()
model.add(Convolution2D(8, nb_conv, nb_conv,
border_mode='same',
input_shape=(1, img_rows, img_cols),
# W_regularizer = l1l2ld(l1 = 0., l2 = 0., ld = 0.),
W_regularizer = l2(l = 0.),
b_regularizer = l2(l = 0.),
activity_regularizer = activity_l1l2(l1 = 0., l2 = 0.)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
# model.add(AveragePooling2D(pool_size=(nb_pool, nb_pool)))
model.add(Convolution2D(16, nb_conv, nb_conv,
border_mode='same',
input_shape=(1, img_rows, img_cols),
# W_regularizer = l1l2ld(l1 = 0., l2 = 0., ld = 0.),
W_regularizer = l2(l = 0.),
b_regularizer = l2(l = 0.),
activity_regularizer = activity_l1l2(l1 = 0., l2 = 0.)))
model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
model.add(Convolution2D(32, nb_conv, nb_conv,
border_mode='valid',
input_shape=(1, img_rows, img_cols),
# W_regularizer = l1l2ld(l1 = 0., l2 = 0., ld = 0.),
# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)
# model = Sequential()
# model.add(Dense(1024, input_shape=(3072,), W_regularizer = l2(0.), b_regularizer = l2(0.)))
# model.add(Activation('relu'))
# # model.add(Dropout(0.2))
# model.add(Dense(512, W_regularizer = l2(0.), b_regularizer = l2(0.)))
# model.add(Activation('relu'))
# # model.add(Dropout(0.2))
# model.add(Dense(10, W_regularizer = l2(0.), b_regularizer = l2(0.)))
# model.add(Activation('softmax'))
model = Sequential()
model.add(Dense(512, input_shape=(3072,), activity_regularizer = activity_l1l2(l1 = 0., l2 = 0.),
W_regularizer = l2(l = 0.), b_regularizer = l2(l = 0.)))
model.add(Activation('relu'))
# model.add(Dropout(0.2))
model.add(Dense(1024, activity_regularizer = activity_l1l2(l1 = 0., l2 = 0.),
W_regularizer = l2(l = 0.), b_regularizer = l2(l = 0.)))
model.add(Activation('relu'))
model.add(Dense(1024, activity_regularizer = activity_l1l2(l1 = 0., l2 = 0.),
W_regularizer = l2(l = 0.), b_regularizer = l2(l = 0.)))
model.add(Activation('relu'))
model.add(Dense(2048, activity_regularizer = activity_l1l2(l1 = 0., l2 = 0.),
W_regularizer = l2(l = 0.), b_regularizer = l2(l = 0.)))
model.add(Activation('relu'))
model.add(Dense(2048, activity_regularizer = activity_l1l2(l1 = 0., l2 = 0.),
W_regularizer = l2(l = 0.), b_regularizer = l2(l = 0.)))
model.add(Activation('relu'))
def _regularize(layer_idx):
"""Small helper function to define per layer regularization"""
return {
'W_regularizer': l1l2(l1_reg_weights[layer_idx], l2_reg_weights[layer_idx]),
'activity_regularizer': activity_l1l2(l1_reg_activity[layer_idx], l2_reg_activity[layer_idx]),
}
def get_model_classification_refinement(input_shape,
filename=None,
l1_value=10e-12,
l2_value=10e-14,
mode='valid'):
def binary_crossentropy(y_true, y_pred):
p1 = 0.412244897959
p0 = 1 - p1
w = p1 * K.cast(K.equal(y_true, 0), K.floatx()) + p0 * K.cast(
K.equal(y_true, 1), K.floatx())
return K.sum(w * K.binary_crossentropy(y_pred, y_true),
axis=[1, 2, 3]) / K.sum(w, axis=[1, 2, 3])
def binary_crossentropy_metric(y_true, y_pred):
return K.mean(binary_crossentropy(y_true, y_pred))
model = Sequential()
model.add(
Convolution2D(nb_filter=32,
nb_row=3,
nb_col=3,
border_mode=mode,
input_shape=(input_shape[0], input_shape[1], 1),
dim_ordering='tf',
W_regularizer=l1l2(l1=l1_value, l2=l2_value),
activity_regularizer=activity_l1l2(l1=l1_value,
l2=l2_value)))
model.add(Activation('relu'))
model.add(
Convolution2D(nb_filter=32,
nb_row=3,
nb_col=3,
border_mode=mode,
dim_ordering='tf',
W_regularizer=l1l2(l1=l1_value, l2=l2_value),
activity_regularizer=activity_l1l2(l1=l1_value,
l2=l2_value)))
model.add(Activation('relu'))
model.add(MaxPooling2D(dim_ordering='tf'))
model.add(
Convolution2D(nb_filter=64,
nb_row=3,
nb_col=3,
border_mode=mode,
dim_ordering='tf',
W_regularizer=l1l2(l1=l1_value, l2=l2_value),
activity_regularizer=activity_l1l2(l1=l1_value,
l2=l2_value)))
model.add(Activation('relu'))
model.add(
Convolution2D(nb_filter=64,
nb_row=3,
nb_col=3,
border_mode=mode,
dim_ordering='tf',
W_regularizer=l1l2(l1=l1_value, l2=l2_value),
activity_regularizer=activity_l1l2(l1=l1_value,
l2=l2_value)))
model.add(Activation('relu'))
model.add(MaxPooling2D(dim_ordering='tf'))
model.add(
Convolution2D(nb_filter=128,
nb_row=3,
nb_col=3,
border_mode=mode,
dim_ordering='tf',
W_regularizer=l1l2(l1=l1_value, l2=l2_value),
activity_regularizer=activity_l1l2(l1=l1_value,
l2=l2_value)))
model.add(Activation('relu'))
model.add(
Convolution2D(nb_filter=128,
nb_row=3,
nb_col=3,
border_mode=mode,
dim_ordering='tf',
W_regularizer=l1l2(l1=l1_value, l2=l2_value),
activity_regularizer=activity_l1l2(l1=l1_value,
l2=l2_value)))
model.add(Activation('relu'))
model.add(MaxPooling2D(dim_ordering='tf'))
model.add(
Convolution2D(nb_filter=512,
nb_row=16,
nb_col=16,
border_mode=mode,
dim_ordering='tf',
W_regularizer=l1l2(l1=l1_value, l2=l2_value),
activity_regularizer=activity_l1l2(l1=l1_value,
l2=l2_value)))
model.add(Activation('relu'))
model.add(
Convolution2D(nb_filter=512,
nb_row=1,
nb_col=1,
border_mode=mode,
dim_ordering='tf',
W_regularizer=l1l2(l1=l1_value, l2=l2_value),
activity_regularizer=activity_l1l2(l1=l1_value,
l2=l2_value)))
model.add(Activation('relu'))
model.add(
Convolution2D(nb_filter=1,
nb_row=1,
nb_col=1,
border_mode=mode,
dim_ordering='tf',
W_regularizer=l1l2(l1=l1_value, l2=l2_value),
activity_regularizer=activity_l1l2(l1=l1_value,
l2=l2_value)))
model.add(Activation('sigmoid'))
print("compiling model")
# load previously trained model
if filename is not None:
print("Loading weights from", filename)
model.load_weights(filename)
model.compile(
optimizer=Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08),
metrics=[dice_error_metric, binary_crossentropy_metric, 'accuracy'],
loss=binary_crossentropy)
print("Model padding:", (input_shape[0] - model.output_shape[1]) / 2,
(input_shape[1] - model.output_shape[2]) / 2)
return model
def get_model_classification(input_shape,
filename=None,
l1_value=10e-12,
l2_value=10e-14):
model = Sequential()
model.add(
Convolution2D(nb_filter=32,
nb_row=3,
nb_col=3,
border_mode='valid',
input_shape=(input_shape[0], input_shape[1], 1),
dim_ordering='tf',
W_regularizer=l1l2(l1=l1_value, l2=l2_value),
activity_regularizer=activity_l1l2(l1=l1_value,
l2=l2_value)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(
AtrousConv2D(nb_filter=64,
nb_row=3,
nb_col=3,
atrous_rate=(2, 2),
border_mode='valid',
dim_ordering='tf',
W_regularizer=l1l2(l1=l1_value, l2=l2_value),
activity_regularizer=activity_l1l2(l1=l1_value,
l2=l2_value)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(
AtrousConv2D(nb_filter=128,
nb_row=3,
nb_col=3,
atrous_rate=(4, 4),
border_mode='valid',
dim_ordering='tf',
W_regularizer=l1l2(l1=l1_value, l2=l2_value),
activity_regularizer=activity_l1l2(l1=l1_value,
l2=l2_value)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(
AtrousConv2D(nb_filter=512,
nb_row=3,
nb_col=3,
atrous_rate=(8, 8),
border_mode='valid',
dim_ordering='tf',
W_regularizer=l1l2(l1=l1_value, l2=l2_value),
activity_regularizer=activity_l1l2(l1=l1_value,
l2=l2_value)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(
Convolution2D(nb_filter=512,
nb_row=1,
nb_col=1,
border_mode='valid',
dim_ordering='tf',
W_regularizer=l1l2(l1=l1_value, l2=l2_value),
activity_regularizer=activity_l1l2(l1=l1_value,
l2=l2_value)))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(
Convolution2D(nb_filter=1,
nb_row=1,
nb_col=1,
border_mode='valid',
dim_ordering='tf',
W_regularizer=l1l2(l1=l1_value, l2=l2_value),
activity_regularizer=activity_l1l2(l1=l1_value,
l2=l2_value)))
model.add(Activation('sigmoid'))
print("compiling model")
# load previously trained model
if filename is not None:
print("Loading weights from", filename)
model.load_weights(filename)
model.compile(
optimizer=Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08),
metrics=[dice_error_metric, binary_crossentropy_metric, 'accuracy'],
loss=binary_crossentropy)
print("Model padding:", (input_shape[0] - model.output_shape[1]) / 2,
(input_shape[1] - model.output_shape[2]) / 2)
return model
def train(self):
self.model = Sequential()
self.model.add(Dense(100, input_dim=self.N, W_regularizer=l1l2(), b_regularizer=l1l2(), activity_regularizer=activity_l1l2()))
self.model.add(Activation('relu'))
self.model.add(Dropout(0.25))
self.model.add(Dense(100, W_regularizer=l1l2(), b_regularizer=l1l2(), activity_regularizer=activity_l1l2()))
self.model.add(Activation('relu'))
self.model.add(Dropout(0.25))
self.model.add(Dense(100, W_regularizer=l1l2(), b_regularizer=l1l2(), activity_regularizer=activity_l1l2()))
self.model.add(Activation('relu'))
self.model.add(Dropout(0.25))
self.model.add(Dense(100, W_regularizer=l1l2(), b_regularizer=l1l2(), activity_regularizer=activity_l1l2()))
self.model.add(Activation('relu'))
self.model.add(Dropout(0.25))
self.model.add(Dense(1))
self.model.compile(loss='mean_squared_error', optimizer='rmsprop')
self.model.fit(np.array(self.X), np.array(self.Y), show_accuracy=True, batch_size=16, nb_epoch=2000, verbose=2)
(X_train, y_train), (X_test, y_test) = cifar10.load_data()
print('X_train shape:', X_train.shape)
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')
# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)
model = Sequential()
model.add(Convolution2D(32, 3, 3, border_mode='same',
input_shape=(img_channels, img_rows, img_cols),
W_regularizer = l2(l = 0.),
b_regularizer = l2(l = 0.),
activity_regularizer = activity_l1l2(l1 = 0., l2 = 0.)))
model.add(Activation('relu'))
model.add(Convolution2D(32, 3, 3,
W_regularizer = l2(l = 0.),
b_regularizer = l2(l = 0.),
activity_regularizer = activity_l1l2(l1 = 0., l2 = 0.)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Convolution2D(64, 3, 3, border_mode='same',
W_regularizer = l2(l = 0.),
b_regularizer = l2(l = 0.),
activity_regularizer = activity_l1l2(l1 = 0., l2 = 0.)))
model.add(Activation('relu'))
model.add(Convolution2D(64, 3, 3,
def _get_model_Maxout_Dense(nb_features, nb_layers, number_maxout, activation_function, l1, l2, activity_l1, activity_l2, init, nb_classes, batch_normalization):
name = "MO"+str(number_maxout)+"_"+str(nb_classes*2**4)
if nb_layers==5 or nb_layers==4:
if nb_layers==5:
name+="_"+str(nb_classes*2**4)
name+="_"+str(nb_classes*2**4)+"_D_"+str(nb_classes*2**3)+"_"+str(nb_classes*2**2)+"_"+str(nb_classes*2)
elif nb_layers==3:
name+="_D_"+str(nb_classes*2**3)+"_"+str(int((nb_classes*2**2+nb_classes*2)*0.5))
name+="_"+str(nb_classes)
if batch_normalization:
name+="__BN_"
name+=activation_function
model = Sequential()
if activation_function in ["relu", "tanh"]:
if l1!=0. or l2!=0. or activity_l1!=0. or activity_l2!=0.:
name+="_l1"+str(int(l1*100))+"_l2"+str(int(l2*100))+"_al1"+str(int(activity_l1*100))+"_al2"+str(int(activity_l2*100))
model.add(MaxoutDense(output_dim=nb_classes*2**4, nb_feature=number_maxout, init=init, W_regularizer=l1l2(l1=l1, l2=l2), activity_regularizer=activity_l1l2(l1=activity_l1, l2=activity_l2), input_shape=(nb_features,)))
if batch_normalization:
model.add(BatchNormalization())
if nb_layers==5:
model.add(MaxoutDense(output_dim=nb_classes*2**4, nb_feature=number_maxout, init=init, W_regularizer=l1l2(l1=l1, l2=l2), activity_regularizer=activity_l1l2(l1=activity_l1, l2=activity_l2)))
if batch_normalization:
model.add(BatchNormalization())
model.add(Dense(nb_classes*2**3, activation=activation_function, init=init, W_regularizer=l1l2(l1=l1, l2=l2), activity_regularizer=activity_l1l2(l1=activity_l1, l2=activity_l2)))
if batch_normalization:
model.add(BatchNormalization())
if nb_layers==5 or nb_layers==4:
model.add(Dense(nb_classes*2**2, activation=activation_function, init=init, W_regularizer=l1l2(l1=l1, l2=l2), activity_regularizer=activity_l1l2(l1=activity_l1, l2=activity_l2)))
if batch_normalization:
model.add(BatchNormalization())
model.add(Dense(nb_classes*2, activation=activation_function, init=init, W_regularizer=l1l2(l1=l1, l2=l2), activity_regularizer=activity_l1l2(l1=activity_l1, l2=activity_l2)))
if batch_normalization:
model.add(BatchNormalization())
elif nb_layers==3:
model.add(Dense(int((nb_classes*2 + nb_classes)*0.5), activation=activation_function, init=init, W_regularizer=l1l2(l1=l1, l2=l2), activity_regularizer=activity_l1l2(l1=activity_l1, l2=activity_l2)))
if batch_normalization:
model.add(BatchNormalization())
model.add(Dense(nb_classes, activation="softmax", init=init))
else:
print number_maxout, nb_classes, init, nb_features
model.add(MaxoutDense(output_dim=nb_classes*2**4, nb_feature=number_maxout, init=init, input_shape=(nb_features,)))
if batch_normalization:
model.add(BatchNormalization())
if nb_layers==5:
model.add(MaxoutDense(output_dim=nb_classes*2**4, nb_feature=number_maxout, init=init))
if batch_normalization:
model.add(BatchNormalization())
model.add(Dense(nb_classes*2**3, activation=activation_function, init=init))
if batch_normalization:
model.add(BatchNormalization())
if nb_layers==5 or nb_layers==4:
model.add(Dense(nb_classes*2**2, activation=activation_function, init=init))
if batch_normalization:
model.add(BatchNormalization())
model.add(Dense(nb_classes*2, activation=activation_function, init=init))
if batch_normalization:
model.add(BatchNormalization())
elif nb_layers==3:
model.add(Dense(int((nb_classes*2 + nb_classes)*0.5), activation=activation_function, init=init))
if batch_normalization:
model.add(BatchNormalization())
model.add(Dense(nb_classes, activation="softmax", init=init))
elif activation_function=="SReLU":
if l1!=0. or l2!=0. or activity_l1!=0. or activity_l2!=0.:
name+="_l1"+str(int(l1*100))+"_l2"+str(int(l2*100))+"_al1"+str(int(activity_l1*100))+"_al2"+str(int(activity_l2*100))
model.add(MaxoutDense(output_dim=nb_classes*2**4, nb_feature=number_maxout, init=init, W_regularizer=l1l2(l1=l1, l2=l2), activity_regularizer=activity_l1l2(l1=activity_l1, l2=activity_l2), input_shape=(nb_features,)))
model.add(SReLU())
if batch_normalization:
model.add(BatchNormalization())
if nb_layers==5:
model.add(MaxoutDense(output_dim=nb_classes*2**4, nb_feature=number_maxout, init=init, W_regularizer=l1l2(l1=l1, l2=l2), activity_regularizer=activity_l1l2(l1=activity_l1, l2=activity_l2)))
model.add(SReLU())
if batch_normalization:
model.add(BatchNormalization())
model.add(Dense(nb_classes*2**3, init=init, W_regularizer=l1l2(l1=l1, l2=l2), activity_regularizer=activity_l1l2(l1=activity_l1, l2=activity_l2)))
model.add(SReLU())
if batch_normalization:
model.add(BatchNormalization())
if nb_layers==5 or nb_layers==4:
model.add(Dense(nb_classes*2**2, init=init, W_regularizer=l1l2(l1=l1, l2=l2), activity_regularizer=activity_l1l2(l1=activity_l1, l2=activity_l2)))
model.add(SReLU())
if batch_normalization:
model.add(BatchNormalization())
model.add(Dense(nb_classes*2, init=init, W_regularizer=l1l2(l1=l1, l2=l2), activity_regularizer=activity_l1l2(l1=activity_l1, l2=activity_l2)))
model.add(SReLU())
if batch_normalization:
model.add(BatchNormalization())
elif nb_layers==3:
model.add(Dense(int((nb_classes*2 + nb_classes)*0.5), init=init, W_regularizer=l1l2(l1=l1, l2=l2), activity_regularizer=activity_l1l2(l1=activity_l1, l2=activity_l2)))
model.add(SReLU())
if batch_normalization:
model.add(BatchNormalization())
model.add(Dense(nb_classes, activation="softmax", init=init))
else:
model.add(MaxoutDense(output_dim=nb_classes*2**4, nb_feature=number_maxout, init=init, input_shape=(nb_features,)))
model.add(SReLU())
if batch_normalization:
model.add(BatchNormalization())
if nb_layers==5:
model.add(MaxoutDense(output_dim=nb_classes*2**4, nb_feature=number_maxout, init=init))
model.add(SReLU())
if batch_normalization:
model.add(BatchNormalization())
model.add(Dense(nb_classes*2**3, init=init))
model.add(SReLU())
if batch_normalization:
model.add(BatchNormalization())
if nb_layers==5 or nb_layers==4:
model.add(Dense(nb_classes*2**2, init=init))
model.add(SReLU())
if batch_normalization:
model.add(BatchNormalization())
model.add(Dense(nb_classes*2, init=init))
model.add(SReLU())
if batch_normalization:
model.add(BatchNormalization())
elif nb_layers==3:
model.add(Dense(int((nb_classes*2 + nb_classes)*0.5), init=init))
model.add(SReLU())
if batch_normalization:
model.add(BatchNormalization())
model.add(Dense(nb_classes, activation="softmax", init=init))
elif activation_function=="ELU":
if l1!=0. or l2!=0. or activity_l1!=0. or activity_l2!=0.:
name+="_l1"+str(int(l1*100))+"_l2"+str(int(l2*100))+"_al1"+str(int(activity_l1*100))+"_al2"+str(int(activity_l2*100))
model.add(MaxoutDense(output_dim=nb_classes*2**4, nb_feature=number_maxout, init=init, W_regularizer=l1l2(l1=l1, l2=l2), activity_regularizer=activity_l1l2(l1=activity_l1, l2=activity_l2), input_shape=(nb_features,)))
model.add(ELU())
if batch_normalization:
model.add(BatchNormalization())
if nb_layers==5:
model.add(MaxoutDense(output_dim=nb_classes*2**4, nb_feature=number_maxout, init=init, W_regularizer=l1l2(l1=l1, l2=l2), activity_regularizer=activity_l1l2(l1=activity_l1, l2=activity_l2)))
model.add(ELU())
if batch_normalization:
model.add(BatchNormalization())
model.add(Dense(nb_classes*2**3, init=init, W_regularizer=l1l2(l1=l1, l2=l2), activity_regularizer=activity_l1l2(l1=activity_l1, l2=activity_l2)))
model.add(ELU())
if batch_normalization:
model.add(BatchNormalization())
if nb_layers==5 or nb_layers==4:
model.add(Dense(nb_classes*2**2, init=init, W_regularizer=l1l2(l1=l1, l2=l2), activity_regularizer=activity_l1l2(l1=activity_l1, l2=activity_l2)))
model.add(ELU())
if batch_normalization:
model.add(BatchNormalization())
model.add(Dense(nb_classes*2, init=init, W_regularizer=l1l2(l1=l1, l2=l2), activity_regularizer=activity_l1l2(l1=activity_l1, l2=activity_l2)))
model.add(ELU())
if batch_normalization:
model.add(BatchNormalization())
elif nb_layers==3:
model.add(Dense(int((nb_classes*2 + nb_classes)*0.5), init=init, W_regularizer=l1l2(l1=l1, l2=l2), activity_regularizer=activity_l1l2(l1=activity_l1, l2=activity_l2)))
model.add(ELU())
if batch_normalization:
model.add(BatchNormalization())
model.add(Dense(nb_classes, activation="softmax", init=init))
else:
model.add(MaxoutDense(output_dim=nb_classes*2**4, nb_feature=number_maxout, init=init, input_shape=(nb_features,)))
model.add(ELU())
if batch_normalization:
model.add(BatchNormalization())
if nb_layers==5:
model.add(MaxoutDense(output_dim=nb_classes*2**4, nb_feature=number_maxout, init=init))
model.add(ELU())
if batch_normalization:
model.add(BatchNormalization())
model.add(Dense(nb_classes*2**3, init=init))
model.add(ELU())
if batch_normalization:
model.add(BatchNormalization())
if nb_layers==5 or nb_layers==4:
model.add(Dense(nb_classes*2**2, init=init))
model.add(ELU())
if batch_normalization:
model.add(BatchNormalization())
model.add(Dense(nb_classes*2, init=init))
model.add(ELU())
if batch_normalization:
model.add(BatchNormalization())
elif nb_layers==3:
model.add(Dense(int((nb_classes*2 + nb_classes)*0.5), init=init))
model.add(ELU())
if batch_normalization:
model.add(BatchNormalization())
model.add(Dense(nb_classes, activation="softmax", init=init))
elif activation_function=="PReLU":
if l1!=0. or l2!=0. or activity_l1!=0. or activity_l2!=0.:
name+="_l1"+str(int(l1*100))+"_l2"+str(int(l2*100))+"_al1"+str(int(activity_l1*100))+"_al2"+str(int(activity_l2*100))
model.add(MaxoutDense(output_dim=nb_classes*2**4, nb_feature=number_maxout, init=init, W_regularizer=l1l2(l1=l1, l2=l2), activity_regularizer=activity_l1l2(l1=activity_l1, l2=activity_l2), input_shape=(nb_features,)))
model.add(PReLU())
if batch_normalization:
model.add(BatchNormalization())
if nb_layers==5:
model.add(MaxoutDense(output_dim=nb_classes*2**4, nb_feature=number_maxout, init=init, W_regularizer=l1l2(l1=l1, l2=l2), activity_regularizer=activity_l1l2(l1=activity_l1, l2=activity_l2)))
model.add(PReLU())
if batch_normalization:
model.add(BatchNormalization())
model.add(Dense(nb_classes*2**3, init=init, W_regularizer=l1l2(l1=l1, l2=l2), activity_regularizer=activity_l1l2(l1=activity_l1, l2=activity_l2)))
model.add(PReLU())
if batch_normalization:
model.add(BatchNormalization())
if nb_layers==5 or nb_layers==4:
model.add(Dense(nb_classes*2**2, init=init, W_regularizer=l1l2(l1=l1, l2=l2), activity_regularizer=activity_l1l2(l1=activity_l1, l2=activity_l2)))
model.add(PReLU())
if batch_normalization:
model.add(BatchNormalization())
model.add(Dense(nb_classes*2, init=init, W_regularizer=l1l2(l1=l1, l2=l2), activity_regularizer=activity_l1l2(l1=activity_l1, l2=activity_l2)))
model.add(PReLU())
if batch_normalization:
model.add(BatchNormalization())
elif nb_layers==3:
model.add(Dense(nb_classes*2, init=init, W_regularizer=l1l2(l1=l1, l2=l2), activity_regularizer=activity_l1l2(l1=activity_l1, l2=activity_l2)))
model.add(PReLU())
if batch_normalization:
model.add(BatchNormalization())
model.add(Dense(nb_classes, activation="softmax", init=init))
else:
model.add(MaxoutDense(output_dim=nb_classes*2**4, nb_feature=number_maxout, init=init, input_shape=(nb_features,)))
model.add(PReLU())
if batch_normalization:
model.add(BatchNormalization())
if nb_layers==5:
model.add(MaxoutDense(output_dim=nb_classes*2**4, nb_feature=number_maxout, init=init))
model.add(PReLU())
if batch_normalization:
model.add(BatchNormalization())
model.add(Dense(nb_classes*2**3, init=init))
model.add(PReLU())
if batch_normalization:
model.add(BatchNormalization())
if nb_layers==5 or nb_layers==4:
model.add(Dense(nb_classes*2**2, init=init))
model.add(PReLU())
if batch_normalization:
model.add(BatchNormalization())
model.add(Dense(nb_classes*2, init=init))
model.add(PReLU())
if batch_normalization:
model.add(BatchNormalization())
elif nb_layers==3:
model.add(Dense(int((nb_classes*2 + nb_classes)*0.5), init=init))
model.add(PReLU())
if batch_normalization:
model.add(BatchNormalization())
model.add(Dense(nb_classes, activation="softmax", init=init))
return model, name