def create_model(self, space: Optional[Dict[str, Any]] = None) -> Model:
if space:
print('Using hyperopt space:')
print(space)
for_optimization = True if space else False
if for_optimization:
if space['conv_kind'] == 'standard':
img_input = Input(shape=(640, 400, 3), dtype='float32')
x = layers.Conv2D(
32 if not for_optimization else space['Conv2D_0'],
5 if not for_optimization else space['kernel'],
activation='relu')(img_input)
x = layers.MaxPooling2D(2)(x)
x = layers.Conv2D(
64 if not for_optimization else space['Conv2D_1'],
5 if not for_optimization else space['kernel'],
activation='relu')(x)
x = layers.MaxPooling2D(2)(x)
x = layers.Conv2D(
128 if not for_optimization else space['Conv2D_2'],
5 if not for_optimization else space['kernel'],
activation='relu')(x)
x = layers.MaxPooling2D(2)(x)
x = layers.Conv2D(
128 if not for_optimization else space['Conv2D_3'],
5 if not for_optimization else space['kernel'],
activation='relu')(x)
x = layers.MaxPooling2D(2)(x)
x = layers.Conv2D(
128 if not for_optimization else space['Conv2D_4'],
5 if not for_optimization else space['kernel'],
activation='relu')(x)
x = layers.MaxPooling2D(2)(x)
x = layers.Flatten()(x)
x = layers.Dense(space['dense0'], activation='relu')(x)
rot_x_pred = layers.Dense(1, name='rot_x')(x)
else:
img_input = Input(shape=(640, 400, 3), dtype='float32')
x = layers.SeparableConv2D(
32 if not for_optimization else space['Conv2D_0'],
5 if not for_optimization else space['kernel'],
activation='relu')(img_input)
x = layers.MaxPooling2D(2)(x)
x = layers.SeparableConv2D(
64 if not for_optimization else space['Conv2D_1'],
5 if not for_optimization else space['kernel'],
activation='relu')(x)
x = layers.MaxPooling2D(2)(x)
x = layers.SeparableConv2D(
128 if not for_optimization else space['Conv2D_2'],
5 if not for_optimization else space['kernel'],
activation='relu')(x)
x = layers.MaxPooling2D(2)(x)
x = layers.SeparableConv2D(
128 if not for_optimization else space['Conv2D_3'],
5 if not for_optimization else space['kernel'],
activation='relu')(x)
x = layers.MaxPooling2D(2)(x)
x = layers.SeparableConv2D(
128 if not for_optimization else space['Conv2D_4'],
5 if not for_optimization else space['kernel'],
activation='relu')(x)
x = layers.MaxPooling2D(2)(x)
x = layers.Flatten()(x)
x = layers.Dense(space['dense0'], activation='relu')(x)
rot_x_pred = layers.Dense(1, name='rot_x')(x)
else:
img_input = Input(shape=(1384, 865, 3), dtype='float32')
x = layers.SeparableConv2D(256,
20,
strides=(10, 10),
activation='relu')(img_input)
x = layers.SeparableConv2D(512,
7,
strides=(2, 2),
activation='relu')(x)
x = layers.SeparableConv2D(1024,
7,
strides=(2, 2),
activation='relu')(x)
x = layers.SeparableConv2D(1024,
7,
strides=(2, 2),
activation='relu')(x)
x = layers.Flatten()(x)
x = layers.Dense(128, activation='relu')(x)
rot_x_pred = layers.Dense(1, name='rot_x')(x)
model = Model(img_input, rot_x_pred)
model.compile(optimizer='rmsprop', loss='mae')
return model
def kale(input_shape=(32, None, 1), num_classes=5991, max_string_len=10):
input = layers.Input(shape=input_shape)
# swl_output = tf.py_func(sliding_window_layer, [input], tf.float32)
swl_output = layers.Lambda(swl_lambda_func)(input)
xx = layers.Reshape((-1, 32, 32, 1))(swl_output)
print(xx.get_shape())
print(xx.shape)
xx._keras_shape = tuple(xx.get_shape().as_list())
print(K.int_shape(xx))
# swl_output = SlidingWindowLayer()(input)
x = TimeDistributed(
layers.Conv2D(50, (3, 3),
padding='same',
activation='relu',
name='conv1'))(xx)
x = TimeDistributed(layers.BatchNormalization(axis=-1))(x)
x = TimeDistributed(
layers.Conv2D(100, (3, 3),
padding='same',
activation='relu',
name='conv2'))(x)
x = TimeDistributed(layers.Dropout(0.1))(x)
x = TimeDistributed(
layers.Conv2D(100, (3, 3),
padding='same',
activation='relu',
name='conv3'))(x)
x = TimeDistributed(layers.Dropout(0.1))(x)
x = TimeDistributed(layers.BatchNormalization(axis=-1))(x)
x = TimeDistributed(layers.MaxPooling2D((2, 2)))(x)
x = TimeDistributed(
layers.Conv2D(150, (3, 3),
padding='same',
activation='relu',
name='conv4'))(x)
x = TimeDistributed(layers.BatchNormalization(axis=-1))(x)
x = TimeDistributed(
layers.Conv2D(200, (3, 3),
padding='same',
activation='relu',
name='conv5'))(x)
x = TimeDistributed(layers.Dropout(0.2))(x)
x = TimeDistributed(
layers.Conv2D(200, (3, 3),
padding='same',
activation='relu',
name='conv6'))(x)
x = TimeDistributed(layers.Dropout(0.2))(x)
x = TimeDistributed(layers.BatchNormalization(axis=-1))(x)
x = TimeDistributed(layers.MaxPooling2D((2, 2)))(x)
x = TimeDistributed(
layers.Conv2D(250, (3, 3),
padding='same',
activation='relu',
name='conv7'))(x)
x = TimeDistributed(layers.BatchNormalization(axis=-1))(x)
x = TimeDistributed(
layers.Conv2D(300, (3, 3),
padding='same',
activation='relu',
name='conv8'))(x)
x = TimeDistributed(layers.Dropout(0.3))(x)
x = TimeDistributed(
layers.Conv2D(300, (3, 3),
padding='same',
activation='relu',
name='conv9'))(x)
x = TimeDistributed(layers.Dropout(0.3))(x)
x = TimeDistributed(layers.BatchNormalization(axis=-1))(x)
x = TimeDistributed(layers.MaxPooling2D((2, 2)))(x)
x = TimeDistributed(
layers.Conv2D(350, (3, 3),
padding='same',
activation='relu',
name='conv10'))(x)
x = TimeDistributed(layers.BatchNormalization(axis=-1))(x)
x = TimeDistributed(
layers.Conv2D(400, (3, 3),
padding='same',
activation='relu',
name='conv11'))(x)
x = TimeDistributed(layers.Dropout(0.4))(x)
x = TimeDistributed(
layers.Conv2D(400, (3, 3),
padding='same',
activation='relu',
name='conv12'))(x)
x = TimeDistributed(layers.Dropout(0.4))(x)
x = TimeDistributed(layers.BatchNormalization(axis=-1))(x)
x = TimeDistributed(layers.MaxPooling2D((2, 2)))(x)
x = TimeDistributed(layers.Flatten())(x)
x = TimeDistributed(layers.Dense(900, activation='relu'))(x)
x = TimeDistributed(layers.Dropout(0.5))(x)
x = TimeDistributed(layers.Dense(200, activation='relu'))(x)
classifier_output = TimeDistributed(
layers.Dense(num_classes, activation='softmax'))(x)
# classifier = models.Model(inputs=input, outputs=classifier_output)
# classifier.summary()
label = layers.Input(name='label', shape=[max_string_len], dtype='int64')
# 序列的长度,此模型中为 (280 - 32) // 4 = 63
seq_length = layers.Input(name='seq_length', shape=[1], dtype='int64')
label_length = layers.Input(name='label_length', shape=[1], dtype='int64')
ctc_output = layers.Lambda(
ctc_lambda_func, output_shape=(1, ),
name='ctc')([label, classifier_output, seq_length, label_length])
model = models.Model(inputs=[input, label, seq_length, label_length],
outputs=[ctc_output])
model.summary()
return model
network = models.Sequential() #initialize the neural network
network.add(layers.Conv2D(Channel, kernel_size=Nf, strides=Stride, activation='relu', input_shape=input_shape))
#network.add(layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
#network.add(layers.Conv2D(Channel, kernel_size=Nf, strides=Stride, activation='relu', padding='valid'))
#dropout: spegne casualmente qualche neurone
indexlayer = 0
if dropout:
network.add(layers.Dropout(dropout_const))
indexlayer += 1
network.add(layers.Flatten())
network.add(layers.Dense(64, activation='relu'))
# network.add(layers.Dense(16, activation='relu'))
network.add(layers.Dense(2, activation='sigmoid'))
network.summary() # Shows information about layers and parameters of the entire network
print(len(network.layers))
def design_dnn(nb_features,
input_shape,
nb_levels,
conv_size,
nb_labels,
feat_mult=1,
pool_size=2,
padding='same',
activation='elu',
final_layer='dense-sigmoid',
conv_dropout=0,
conv_maxnorm=0,
nb_input_features=1,
batch_norm=False,
name=None,
prefix=None,
use_strided_convolution_maxpool=True,
nb_conv_per_level=2):
"""
"deep" cnn with dense or global max pooling layer @ end...
Could use sequential...
"""
model_name = name
if model_name is None:
model_name = 'model_1'
if prefix is None:
prefix = model_name
ndims = len(input_shape)
input_shape = tuple(input_shape)
convL = getattr(KL, 'Conv%dD' % ndims)
maxpool = KL.MaxPooling3D if len(input_shape) == 3 else KL.MaxPooling2D
if isinstance(pool_size, int):
pool_size = (pool_size, ) * ndims
# kwargs for the convolution layer
conv_kwargs = {'padding': padding, 'activation': activation}
if conv_maxnorm > 0:
conv_kwargs['kernel_constraint'] = maxnorm(conv_maxnorm)
# initialize a dictionary
enc_tensors = {}
# first layer: input
name = '%s_input' % prefix
enc_tensors[name] = KL.Input(shape=input_shape + (nb_input_features, ),
name=name)
last_tensor = enc_tensors[name]
# down arm:
# add nb_levels of conv + ReLu + conv + ReLu. Pool after each of first nb_levels - 1 layers
for level in range(nb_levels):
for conv in range(nb_conv_per_level):
if conv_dropout > 0:
name = '%s_dropout_%d_%d' % (prefix, level, conv)
enc_tensors[name] = KL.Dropout(conv_dropout)(last_tensor)
last_tensor = enc_tensors[name]
name = '%s_conv_%d_%d' % (prefix, level, conv)
nb_lvl_feats = np.round(nb_features * feat_mult**level).astype(int)
enc_tensors[name] = convL(nb_lvl_feats,
conv_size,
**conv_kwargs,
name=name)(last_tensor)
last_tensor = enc_tensors[name]
# max pool
if use_strided_convolution_maxpool:
name = '%s_strided_conv_%d' % (prefix, level)
enc_tensors[name] = convL(nb_lvl_feats,
pool_size,
**conv_kwargs,
name=name)(last_tensor)
last_tensor = enc_tensors[name]
else:
name = '%s_maxpool_%d' % (prefix, level)
enc_tensors[name] = maxpool(pool_size=pool_size,
name=name,
padding=padding)(last_tensor)
last_tensor = enc_tensors[name]
# dense layer
if final_layer == 'dense-sigmoid':
name = "%s_flatten" % prefix
enc_tensors[name] = KL.Flatten(name=name)(last_tensor)
last_tensor = enc_tensors[name]
name = '%s_dense' % prefix
enc_tensors[name] = KL.Dense(1, name=name,
activation="sigmoid")(last_tensor)
elif final_layer == 'dense-tanh':
name = "%s_flatten" % prefix
enc_tensors[name] = KL.Flatten(name=name)(last_tensor)
last_tensor = enc_tensors[name]
name = '%s_dense' % prefix
enc_tensors[name] = KL.Dense(1, name=name)(last_tensor)
last_tensor = enc_tensors[name]
# Omittting BatchNorm for now, it seems to have a cpu vs gpu problem
# https://github.com/tensorflow/tensorflow/pull/8906
# https://github.com/fchollet/keras/issues/5802
# name = '%s_%s_bn' % prefix
# enc_tensors[name] = KL.BatchNormalization(axis=batch_norm, name=name)(last_tensor)
# last_tensor = enc_tensors[name]
name = '%s_%s_tanh' % prefix
enc_tensors[name] = KL.Activation(activation="tanh",
name=name)(last_tensor)
elif final_layer == 'dense-softmax':
name = "%s_flatten" % prefix
enc_tensors[name] = KL.Flatten(name=name)(last_tensor)
last_tensor = enc_tensors[name]
name = '%s_dense' % prefix
enc_tensors[name] = KL.Dense(nb_labels,
name=name,
activation="softmax")(last_tensor)
# global max pooling layer
elif final_layer == 'myglobalmaxpooling':
name = '%s_batch_norm' % prefix
enc_tensors[name] = KL.BatchNormalization(axis=batch_norm,
name=name)(last_tensor)
last_tensor = enc_tensors[name]
name = '%s_global_max_pool' % prefix
enc_tensors[name] = KL.Lambda(_global_max_nd, name=name)(last_tensor)
last_tensor = enc_tensors[name]
name = '%s_global_max_pool_reshape' % prefix
enc_tensors[name] = KL.Reshape((1, 1), name=name)(last_tensor)
last_tensor = enc_tensors[name]
# cannot do activation in lambda layer. Could code inside, but will do extra lyaer
name = '%s_global_max_pool_sigmoid' % prefix
enc_tensors[name] = KL.Conv1D(1,
1,
name=name,
activation="sigmoid",
use_bias=True)(last_tensor)
elif final_layer == 'globalmaxpooling':
name = '%s_conv_to_featmaps' % prefix
enc_tensors[name] = KL.Conv3D(2, 1, name=name,
activation="relu")(last_tensor)
last_tensor = enc_tensors[name]
name = '%s_global_max_pool' % prefix
enc_tensors[name] = KL.GlobalMaxPooling3D(name=name)(last_tensor)
last_tensor = enc_tensors[name]
# cannot do activation in lambda layer. Could code inside, but will do extra lyaer
name = '%s_global_max_pool_softmax' % prefix
enc_tensors[name] = KL.Activation('softmax', name=name)(last_tensor)
last_tensor = enc_tensors[name]
# create the model
model = Model(inputs=[enc_tensors['%s_input' % prefix]],
outputs=[last_tensor],
name=model_name)
return model
def LBL_network_classification(conv_layers,
dense_layers,
train_generator,
validation_generator,
input_shape=(150, 150, 3),
nbr_outputs=1,
nbr_epochs=20,
save_results=False,
network_name="CD",
result_dir=""):
from keras import Input
input_tensor = Input(input_shape)
final_layers_list = []
#Create first layer model
conv_layer = layers.Conv2D(conv_layers[0], (3, 3),
activation='relu')(input_tensor)
pooling = layers.MaxPooling2D((2, 2))(conv_layer)
output = layers.Flatten()(pooling)
output = layers.Dense(nbr_outputs, activation="sigmoid")(output)
model = models.Model(input_tensor, output)
final_layers_list.append(conv_layer)
final_layers_list.append(pooling)
model.compile(loss='binary_crossentropy',
optimizer=optimizers.RMSprop(lr=1e-4),
metrics=['acc'])
model.summary()
history = model.fit_generator(train_generator,
steps_per_epoch=100,
epochs=nbr_epochs,
validation_data=validation_generator,
validation_steps=50)
final_layers_list[-1].trainable = False
### add subsequent layers
idx = 1 # Counts the layer indices
if (save_results):
model_name = network_name + "_conv_" + str(nbr_epochs) + "e_" + str(
idx) + "L"
file_name = model_name + "results.txt" #change so that this
print_results_to_file(model,
history,
file_name,
model_name,
path=result_dir)
model_name = result_dir + model_name + ".h5"
model.save(model_name)
for conv_size in conv_layers[1:]:
idx += 1
### Create new layer along with pooling etc
conv_layer = layers.Conv2D(conv_size, (3, 3),
activation='relu')(final_layers_list[-1])
pooling = layers.MaxPooling2D((2, 2))(conv_layer)
flatten = layers.Flatten()(pooling)
output = layers.Dense(nbr_outputs, activation="sigmoid")(flatten)
model = models.Model(input_tensor, output)
final_layers_list.append(conv_layer)
final_layers_list.append(pooling)
### Set layers in model untrainable
for i in range(len(model.layers) - 4):
model.layers[i].trainable = False
### compile and fit model
model.compile(loss='binary_crossentropy',
optimizer=optimizers.RMSprop(lr=1e-4),
metrics=['acc'])
model.summary()
history = model.fit_generator(train_generator,
steps_per_epoch=100,
epochs=nbr_epochs,
validation_data=validation_generator,
validation_steps=50)
# Save reuslts to a file
if (save_results):
model_name = network_name + "_conv_" + str(
nbr_epochs) + "e_" + str(idx) + "L"
file_name = model_name + "results.txt" #change so that this
print_results_to_file(model,
history,
file_name,
model_name,
path=result_dir)
model_name = result_dir + model_name + ".h5"
model.save(model_name)
## Add the flatten layer to bridge from conv to dense layers
final_layers_list.append(flatten)
### Add the dense layers
for dense_size in dense_layers:
idx += 1
dense_layer = layers.Dense(dense_size,
activation='relu')(final_layers_list[-1])
output = layers.Dense(nbr_outputs, activation="sigmoid")(dense_layer)
model = models.Model(input_tensor, output)
final_layers_list.append(dense_layer)
model = models.Model(input_tensor, output)
for i in range(len(model.layers) - 2):
model.layers[i].trainable = False
model.summary()
model.compile(loss='binary_crossentropy',
optimizer=optimizers.RMSprop(lr=1e-4),
metrics=['acc'])
model.summary()
history = model.fit_generator(train_generator,
steps_per_epoch=100,
epochs=nbr_epochs,
validation_data=validation_generator,
validation_steps=50)
if (save_results):
model_name = network_name + "_dense_" + str(
nbr_epochs) + "e_" + str(idx) + "L"
file_name = model_name + "results.txt" #change so that this
print_results_to_file(model,
history,
file_name,
model_name,
path=result_dir)
model_name = result_dir + model_name + ".h5"
model.save(model_name)
return model, final_layers_list
#x_train, x_validation, y_train, y_validation = cross_validation.train_test_split(x_train, y_train, test_size=0.1, random_state=121)
#Advanced topic: use tensorflow directly inside keras we won't discuss now
def roc_auc(y_true, y_pred):
auc = tf.metrics.auc(y_true, y_pred)[1]
keras.backend.get_session().run(tf.local_variables_initializer())
return auc
print("data ready")
#Model definition
net_input= layers.Input( shape= x_train.shape[1:])
net_out= layers.Conv2D(filters=32, kernel_size= 5, strides=1,padding='same',
activation='relu' )(net_input)
net_out= layers.MaxPooling2D(pool_size=4, strides=2 )(net_out)
net_out= layers.Flatten()(net_out)
net_out= layers.Dense(N_HIDDEN, activation='relu')(net_out)
net_out= layers.Dense(y_train.shape[-1], activation='softmax')(net_out)
model = keras.models.Model( inputs=net_input, outputs= net_out )
sgd= keras.optimizers.SGD(lr=LEARNING_RATE, decay=0., momentum=0., nesterov=False)
model.compile(loss='categorical_crossentropy',optimizer=sgd, metrics=['accuracy', roc_auc])
print("network created")
print("training begin")
model.fit(x_train, y_train,epochs=N_EPOCHS, batch_size=BATCH_SIZE, validation_split=0.1)
score = tuple(model.evaluate(x_test, y_test, batch_size=BATCH_SIZE))
print("\nTesting evaluation loss %f acc %f roc_auc %f"%score)
train_labels = to_categorical(ori_train_labels.copy())
test_images = ori_test_images.copy()
test_images = test_images.reshape((10000, 28, 28, 1))
test_images = test_images.astype('float32') / 255
test_labels = to_categorical(ori_test_labels.copy())
#step 3: Design Model
net = models.Sequential()
net.add(layers.Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=(28, 28, 1))) # need 4D tensor so we need (batch, 28, 28, 1)
net.add(layers.MaxPooling2D((2, 2)))
net.add(layers.Conv2D(64, kernel_size=(3, 3), activation='relu'))
net.add(layers.MaxPooling2D(pool_size=(2, 2)))
net.add(layers.Conv2D(64, kernel_size=(3, 3), activation='relu'))
net.add(layers.Flatten())
net.add(layers.Dense(64, activation='relu'))
net.add(layers.Dense(10, activation='softmax'))
net.summary()
# Step 4: Training Data
net.compile(optimizer=optimizers.Adam(),
loss=losses.categorical_crossentropy,
metrics=[metrics.categorical_accuracy])
history = net.fit(train_images,
train_labels,
epochs=10,
batch_size=64,
validation_data=(test_images, test_labels))
# Stept 5: In ra bieu do
def Conv1DRegressorIn1(flag):
K.clear_session()
current_neighbor = space['neighbor']
current_idx_idx = space['idx_idx']
current_batch_size = space['batch_size']
current_conv1D_filter_num1 = space['conv1D_filter_num1']
current_conv1D_filter_num2 = space['conv1D_filter_num2']
current_conv1D_filter_num3 = space['conv1D_filter_num3']
current_dropout_rate_dense = space['dropout_rate_dense']
summary = True
verbose = 0
#
# setHyperParams
#
## hypers for data
neighbor = {{choice([50, 60, 70, 80, 90, 100, 110, 120, 130, 140])}}
idx_idx = {{choice([0, 1, 2, 3, 4, 5, 6, 7, 8])}}
idx_lst = [
[x for x in range(158) if x not in [24, 26]], # 去除无用特征
[
x for x in range(158) if x not in [24, 26] +
[x for x in range(1, 6)] + [x for x in range(16, 22)] + [40, 42]
], # 去除无用特征+冗余特征
[
x for x in range(158)
if x not in [24, 26] + [x for x in range(0, 22)]
], # 去除无用特征+方位特征
[x for x in range(158)
if x not in [24, 26] + [22, 23, 26, 37, 38]], # 去除无用特征+深度特征
[
x for x in range(158) if x not in [24, 26] +
[x for x in range(27, 37)] + [x for x in range(40, 46)]
], # 去除无用特征+二级结构信息
# [x for x in range(158) if x not in [24, 26] + [x for x in range(27, 34)] + [x for x in range(40, 46)]],# 去除无用特征+二级结构信息1
# [x for x in range(158) if x not in [24, 26] + [x for x in range(34, 37)] + [x for x in range(40, 46)]],# 去除无用特征+二级结构信息2
[x for x in range(158) if x not in [24, 26] + [46, 47]], # 去除无用特征+实验条件
[
x for x in range(158) if x not in [24, 26] + [39] +
[x for x in range(57, 61)] + [x for x in range(48, 57)] +
[x for x in range(61, 81)] + [x for x in range(140, 155)]
], # 去除无用特征+所有原子编码
# [x for x in range(158) if x not in [24, 26] + [39] + [x for x in range(57, 61)] + [x for x in range(48, 57)] + [x for x in range(140, 145)]],# 去除无用特征+原子编码1
# [x for x in range(158) if x not in [24, 26] + [39] + [x for x in range(57, 61)] + [x for x in range(61, 77)] + [x for x in range(145, 153)]],# 去除无用特征+原子编码2
# [x for x in range(158) if x not in [24, 26] + [39] + [x for x in range(57, 61)] + [x for x in range(77, 81)] + [x for x in range(153, 155)]],# 去除无用特征+原子编码3
[
x for x in range(158)
if x not in [24, 26] + [x for x in range(81, 98)]
], # 去除无用特征+rosetta_energy
[
x for x in range(158) if x not in [24, 26] +
[x for x in range(98, 140)] + [x for x in range(155, 158)]
] # 去除无用特征+msa
]
idx = idx_lst[idx_idx]
## hypers for net
lr = 1e-4 # 0.0001
batch_size = {{choice([1, 32, 64, 128])}}
epochs = 200
conv1D_filter_num1 = {{choice([16, 32])}}
conv1D_filter_num2 = {{choice([16, 32, 64])}}
conv1D_filter_num3 = {{choice([32, 64])}}
dropout_rate_dense = {{choice([0.1, 0.2, 0.3, 0.4, 0.5])}}
metrics = ('mae', pearson_r, rmse)
def _data(fold_num, neighbor, idx):
train_data_pth = '/dl/sry/mCNN/dataset/deepddg/npz/wild/cross_valid/cro_fold%s_train_center_CA_PCA_False_neighbor_140.npz' % fold_num
val_data_pth = '/dl/sry/mCNN/dataset/deepddg/npz/wild/cross_valid/cro_fold%s_valid_center_CA_PCA_False_neighbor_140.npz' % fold_num
## train data
train_data = np.load(train_data_pth)
x_train = train_data['x']
y_train = train_data['y']
ddg_train = train_data['ddg'].reshape(-1)
## select kneighbor atoms
x_train_kneighbor_lst = []
for sample in x_train:
dist_arr = sample[:, 0]
indices = sorted(dist_arr.argsort()[:neighbor])
x_train_kneighbor_lst.append(sample[indices, :])
x_train = np.array(x_train_kneighbor_lst)
## idx
x_train = x_train[:, :, idx]
## val data
val_data = np.load(val_data_pth)
x_val = val_data['x']
y_val = val_data['y']
ddg_val = val_data['ddg'].reshape(-1)
## select kneighbor atoms
x_val_kneighbor_lst = []
for sample in x_val:
dist_arr = sample[:, 0]
indices = sorted(dist_arr.argsort()[:neighbor])
x_val_kneighbor_lst.append(sample[indices, :])
x_val = np.array(x_val_kneighbor_lst)
## idx
x_val = x_val[:, :, idx]
# sort row default is chain, pass
# reshape and one-hot
y_train = to_categorical(y_train)
y_val = to_categorical(y_val)
# normalization
train_shape = x_train.shape
val_shape = x_val.shape
col_train = train_shape[-1]
col_val = val_shape[-1]
x_train = x_train.reshape((-1, col_train))
x_val = x_val.reshape((-1, col_val))
mean = x_train.mean(axis=0)
std = x_train.std(axis=0)
std[np.argwhere(std == 0)] = 0.01
x_train -= mean
x_train /= std
x_val -= mean
x_val /= std
x_train = x_train.reshape(train_shape)
x_val = x_val.reshape(val_shape)
print('x_train: %s'
'\ny_train: %s'
'\nddg_train: %s'
'\nx_val: %s'
'\ny_val: %s'
'\nddg_val: %s' % (x_train.shape, y_train.shape, ddg_train.shape,
x_val.shape, y_val.shape, ddg_val.shape))
return x_train, y_train, ddg_train, x_val, y_val, ddg_val
#
# cross_valid
#
hyper_param_tag = '%s_%s_%s_%s_%s_%s_%s' % (
current_neighbor, current_idx_idx, current_batch_size,
current_conv1D_filter_num1, current_conv1D_filter_num2,
current_conv1D_filter_num3, current_dropout_rate_dense)
modeldir = '/dl/sry/projects/from_hp/mCNN/src/Network/deepddg/opt_all_simpleNet_v4/model/%s-%s' % (
hyper_param_tag, time.strftime("%Y.%m.%d.%H.%M.%S", time.localtime()))
os.makedirs(modeldir, exist_ok=True)
opt_lst = []
for k_count in range(1, 11):
print('\n** fold %s is processing **\n' % k_count)
filepth = '%s/fold_%s_weights-best.h5' % (modeldir, k_count)
my_callbacks = [
callbacks.ReduceLROnPlateau(
monitor='val_loss',
factor=0.33,
patience=5,
verbose=verbose,
mode='min',
min_lr=1e-8,
),
callbacks.EarlyStopping(monitor='val_loss',
patience=10,
verbose=verbose),
callbacks.ModelCheckpoint(filepath=filepth,
monitor='val_mean_absolute_error',
verbose=verbose,
save_best_only=True,
mode='min',
save_weights_only=True)
]
x_train, y_train, ddg_train, x_val, y_val, ddg_val = _data(
k_count, neighbor, idx)
row_num, col_num = x_train.shape[1:3]
#
# build net
#
network = models.Sequential()
network.add(
layers.SeparableConv1D(filters=conv1D_filter_num1,
kernel_size=5,
activation='relu',
input_shape=(row_num, col_num)))
network.add(layers.MaxPooling1D(pool_size=2))
network.add(
layers.SeparableConv1D(filters=conv1D_filter_num2,
kernel_size=5,
activation='relu'))
network.add(layers.MaxPooling1D(pool_size=2))
network.add(
layers.SeparableConv1D(filters=conv1D_filter_num3,
kernel_size=3,
activation='relu'))
network.add(layers.MaxPooling1D(pool_size=2))
network.add(layers.Flatten())
network.add(layers.Dense(128, activation='relu'))
network.add(layers.Dropout(dropout_rate_dense))
network.add(layers.Dense(16, activation='relu'))
network.add(layers.Dropout(0.3))
network.add(layers.Dense(1))
if summary:
trainable_count = int(
np.sum([
K.count_params(p) for p in set(network.trainable_weights)
]))
non_trainable_count = int(
np.sum([
K.count_params(p)
for p in set(network.non_trainable_weights)
]))
print('Total params: {:,}'.format(trainable_count +
non_trainable_count))
print('Trainable params: {:,}'.format(trainable_count))
print('Non-trainable params: {:,}'.format(non_trainable_count))
# print(network.summary())
# rmsp = optimizers.RMSprop(lr=0.0001, decay=0.1)
rmsp = optimizers.RMSprop(lr=lr)
network.compile(
optimizer=rmsp, # 'rmsprop', # SGD,adam,rmsprop
loss='mae',
metrics=list(metrics)) # mae平均绝对误差（mean absolute error） accuracy
result = network.fit(
x=x_train,
y=ddg_train,
batch_size=batch_size,
epochs=epochs,
verbose=verbose,
callbacks=my_callbacks,
validation_data=(x_val, ddg_val),
shuffle=True,
)
# print('\n----------History:\n%s'%result.history)
#
# save
#
save_train_cv(network, modeldir, result.history, k_count)
opt_lst.append(np.mean(
result.history['val_mean_absolute_error'][-10:]))
opt_loss = np.mean(opt_lst)
#
# print hyper combination group and current loss value
#
print('\n@current_hyper_tag: %s'
'\n@current optmized_loss: %s' % (hyper_param_tag, opt_loss))
# return {'loss': validation_loss, 'status': STATUS_OK, 'model':model}
return {'loss': opt_loss, 'status': STATUS_OK}
X_train = X_train_val[tt_index[0]]
Y_train = Y_train_val[tt_index[0]]
X_val = X_train_val[tt_index[1]]
Y_val = Y_train_val[tt_index[1]]
#define model
input = layers.Input(shape=(183, 183, 6))
x = layers.Conv2D(128, 5, strides=2, kernel_regularizer=regularizers.l2(0.01))(
input) #outputs 90x90x128
x = layers.ReLU()(x)
x = layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x)
x = layers.Conv2D(256, 5, strides=2, kernel_regularizer=regularizers.l2(0.01))(
x) #outputs 14x14x128
x = layers.ReLU()(x)
x = layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x)
x = layers.Flatten()(x) #outputs 512
x = layers.Dropout(0.4)(x)
x = layers.Dense(512,
activation='relu',
kernel_regularizer=regularizers.l2(0.01))(x)
x = layers.Dense(6, activation='softmax')(x)
clf = keras.models.Model(input, x)
clf_optimzer = keras.optimizers.Adam(lr=0.001)
clf.compile(optimizer=clf_optimzer,
loss='categorical_crossentropy',
weighted_metrics=['categorical_accuracy'])
#train model________________________________________________________________________________________________________________________
def __init__(self, shape):
"""
将dropout输出的结果直接影像最终结果
:param shape:
"""
self.re_rate = 0.6
self.inputs = layers.Input(shape=shape)
self.f_block = layers.Conv3D(4, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.inputs)
self.bn = layers.BatchNormalization()(self.f_block)
self.mp1 = layers.MaxPooling3D((2, 2, 2))(self.bn)
self.f_block1 = layers.Conv3D(8, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.mp1)
self.bn = layers.BatchNormalization()(self.f_block1)
self.mp2 = layers.MaxPooling3D((2, 2, 2))(self.bn)
self.f_block2 = layers.Conv3D(16, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.mp2)
self.f_block2 = layers.BatchNormalization()(self.f_block2)
self.b_back2 = layers.Conv3D(32, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.f_block2)
self.b_back2 = layers.BatchNormalization()(self.b_back2)
self.b_back2 = layers.Conv3D(
64, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(self.re_rate),
padding='same')(layers.UpSampling3D((2, 2, 2))(self.f_block2))
self.b_back2 = layers.BatchNormalization()(self.b_back2)
self.cat2 = layers.concatenate([self.f_block1, self.b_back2])
self.bn = layers.BatchNormalization()(self.cat2)
self.b_back1 = layers.Conv3D(
32, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(self.re_rate),
padding='same')(layers.UpSampling3D((2, 2, 2))(self.bn))
self.b_back1 = layers.BatchNormalization()(self.b_back1)
self.gb = layers.GlobalAveragePooling3D()(self.b_back1)
self.gb_drop = layers.Dropout(rate=0.9)(self.gb)
self.pure_dense = layers.Dense(1, activation='sigmoid')(self.gb_drop)
# add mmse
mmse_input = layers.Input(shape=(1, ), dtype='int32')
embedded_layer = layers.Embedding(shape[-1], 1)(mmse_input)
embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer)
embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer)
emCon = layers.Flatten()(embedded_layer)
self.mmse = emCon
# add sex
sex_input = layers.Input(shape=(1, ), dtype='int32')
embedded_layer = layers.Embedding(shape[-1], 1)(sex_input)
embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer)
embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer)
emCon = layers.Flatten()(embedded_layer)
self.sex = emCon
# add age
age_input = layers.Input(shape=(1, ), dtype='int32')
embedded_layer = layers.Embedding(shape[-1], 1)(age_input)
embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer)
embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer)
emCon = layers.Flatten()(embedded_layer)
self.age = emCon
# add marriage
marriage_input = layers.Input(shape=(1, ), dtype='int32')
embedded_layer = layers.Embedding(shape[-1], 1)(marriage_input)
embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer)
embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer)
emCon = layers.Flatten()(embedded_layer)
self.marriage = emCon
# add apoe4
apoe4_input = layers.Input(shape=(1, ), dtype='int32')
embedded_layer = layers.Embedding(shape[-1], 1)(apoe4_input)
embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer)
embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer)
emCon = layers.Flatten()(embedded_layer)
self.apoe4 = emCon
# add education
edu_input = layers.Input(shape=(1, ), dtype='int32')
embedded_layer = layers.Embedding(shape[-1], 1)(edu_input)
embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer)
embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer)
emCon = layers.Flatten()(embedded_layer)
self.edu = emCon
self.cat = layers.concatenate([
self.mmse, self.sex, self.age, self.marriage, self.apoe4, self.edu
])
self.cli_dense = layers.Dense(6, activation='relu')(self.cat)
self.cli_judge = layers.Dense(1, activation='sigmoid')(self.cli_dense)
self.cat = layers.concatenate([self.gb_drop, self.cli_dense])
self.dense = layers.Dense(1, activation='sigmoid')(self.cat)
self.model = keras.Model(
input=[
self.inputs, mmse_input, sex_input, age_input, marriage_input,
apoe4_input, edu_input
],
output=[self.pure_dense, self.cli_judge, self.dense])
def getModel(self, modelNo, learningRate=0.001):
model_input = layers.Input(shape=(28, 28, 1))
if modelNo == 1:
#model 1
model = models.Sequential()
model.add(
layers.Conv2D(16, (3, 3),
activation=activations.relu,
input_shape=(28, 28, 1)))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(32, (3, 3), activation=activations.relu))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(32, (3, 3), activation=activations.relu))
model.add(layers.Flatten())
model.add(layers.Dense(32, activation=activations.relu))
model.add(
layers.Dense(self.outputShape, activation=activations.softmax))
model.summary()
model.compile(optimizer=optimizers.rmsprop(lr=learningRate),
loss=losses.categorical_crossentropy,
metrics=[metrics.categorical_accuracy])
model.name = "basic CNN, 32"
return model
if modelNo == 2:
#model 1
model = models.Sequential()
model.add(
layers.Conv2D(32, (3, 3),
activation=activations.relu,
input_shape=(28, 28, 1)))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation=activations.relu))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation=activations.relu))
model.add(layers.Flatten())
model.add(layers.Dense(64, activation=activations.relu))
model.add(
layers.Dense(self.outputShape, activation=activations.softmax))
model.summary()
model.compile(optimizer=optimizers.rmsprop(lr=learningRate),
loss=losses.categorical_crossentropy,
metrics=[metrics.categorical_accuracy])
model.name = "basic CNN, 64"
return model
if modelNo == 3:
return self.getConvPoolCNNCModel(model_input, learningRate)
if modelNo == 4:
return self.getAllCNNC(model_input, learningRate)
if modelNo == 5:
return self.NINCNN(model_input, learningRate)
if modelNo == 6:
#model 1
model = models.Sequential()
model.add(
layers.Conv2D(32, (3, 3),
activation=activations.relu,
input_shape=(28, 28, 1)))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Dropout(0.2))
model.add(layers.Conv2D(64, (3, 3), activation=activations.relu))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation=activations.relu))
model.add(layers.Flatten())
model.add(layers.Dense(64, activation=activations.relu))
model.add(
layers.Dense(self.outputShape, activation=activations.softmax))
model.summary()
model.compile(optimizer=optimizers.rmsprop(lr=learningRate),
loss=losses.categorical_crossentropy,
metrics=[metrics.categorical_accuracy])
model.name = "basic CNN, 64, Dropout 0.2"
return model
if modelNo == 7:
#model 1
model = models.Sequential()
model.add(
layers.Conv2D(32, (3, 3),
kernel_regularizer=regularizers.l2(0.01),
activation=activations.relu,
input_shape=(28, 28, 1)))
model.add(layers.MaxPooling2D((2, 2)))
model.add(
layers.Conv2D(64, (3, 3),
kernel_regularizer=regularizers.l2(0.01),
activation=activations.relu))
model.add(layers.MaxPooling2D((2, 2)))
model.add(
layers.Conv2D(64, (3, 3),
kernel_regularizer=regularizers.l2(0.01),
activation=activations.relu))
model.add(layers.Flatten())
model.add(layers.Dense(64, activation=activations.relu))
model.add(
layers.Dense(self.outputShape, activation=activations.softmax))
model.summary()
model.compile(optimizer=optimizers.rmsprop(lr=learningRate),
loss=losses.categorical_crossentropy,
metrics=[metrics.categorical_accuracy])
model.name = "basic CNN, 64, L2 .01"
return model
if modelNo == 8:
#model 1
model = models.Sequential()
model.add(
layers.Conv2D(32, (3, 3),
kernel_regularizer=regularizers.l2(0.01),
activation=activations.relu,
input_shape=(28, 28, 1)))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Dropout(0.2))
model.add(
layers.Conv2D(64, (3, 3),
kernel_regularizer=regularizers.l2(0.01),
activation=activations.relu))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Dropout(0.2))
model.add(
layers.Conv2D(64, (3, 3),
kernel_regularizer=regularizers.l2(0.01),
activation=activations.relu))
model.add(layers.Flatten())
model.add(layers.Dense(64, activation=activations.relu))
model.add(
layers.Dense(self.outputShape, activation=activations.softmax))
model.summary()
model.compile(optimizer=optimizers.rmsprop(lr=learningRate),
loss=losses.categorical_crossentropy,
metrics=[metrics.categorical_accuracy])
model.name = "basic CNN, 64, Dropout 0.2, L2 .01"
return model
if modelNo == 9:
return self.wideNet(model_input, learningRate)
if modelNo == 10:
model = ResnetBuilder.build_resnet_18((1, 28, 28),
self.outputShape)
model.summary()
model.compile(optimizer=optimizers.rmsprop(lr=learningRate),
loss=losses.categorical_crossentropy,
metrics=[metrics.categorical_accuracy])
model.name = "build_resnet_18"
return model
if modelNo == 11:
model = ResnetBuilder.build_resnet_50((1, 28, 28),
self.outputShape)
model.summary()
model.compile(optimizer=optimizers.rmsprop(lr=learningRate),
loss=losses.categorical_crossentropy,
metrics=[metrics.categorical_accuracy])
model.name = "build_resnet_50"
return model
mnist_conv_base = models.load_model('mnist-base.h5')
"""
Base mode for transfer learning is configured below.
1. Layers are shaved off.
2. Layers are set to be none trainable
"""
print('TOTAL LAYERS: ', len(mnist_conv_base.layers))
mnist_conv_base.trainable = False
mnist_conv_base.summary()
n = 6
for i in range(n):
mnist_conv_base.layers.pop()
mnist_conv_base.add(layers.Flatten())
mnist_conv_base.summary()
train_data_gen = ImageDataGenerator(rescale=1. / 255, rotation_range=180)
test_data_gen = ImageDataGenerator(rescale=1. / 255)
train_generator = train_data_gen.flow_from_directory(train_dir,
target_size=(28, 28),
batch_size=20,
color_mode="grayscale")
test_generator = test_data_gen.flow_from_directory(test_dir,
target_size=(28, 28),
batch_size=20,
color_mode="grayscale")
def independent_avg_modular_deeptrack_new_layer(
layer_size, # Total number of nodes in layer
train_generator,
conv_list,
output_list,
input_tensor,
layer_no=2,
nbr_nodes_added=1,
sample_sizes=(8, 32, 128, 512, 1024),
iteration_numbers=(401, 301, 201, 101, 51),
verbose=0.01,
mp_training = False, # use multiprocessing training?
translation_distance=5, # parameters for multiprocessing training
SN_limits=[10,100], # parameters for multiprocessing training
save_networks=False,
model_path="",
layer_type='conv'):
"""
Adds a new layer to the modular averaging architecture where each output is
trained independently
"""
import deeptrack
from keras import models,layers
from keras.models import Model
from feature_by_feature import freeze_all_layers
new_layer_node_list = [] # Convolutions in the new layer
new_layer_flattened_list = [] # flattened output from new layer
# Create input tenosr for new node
if len(conv_list)>1:
new_layer_input = layers.Concatenate()(conv_list)
else:
new_layer_input = conv_list[0]
# If next layer is dense then we (probably) need to flatten previous input
if layer_type=='dense':
import keras.backend as K
# Check dimension of previous output to see if Flatten layer is neede.
prev_out_size = K.shape(new_layer_input).shape
if(prev_out_size[0]>2):
new_layer_input = layers.Flatten()(new_layer_input)
# Add all the new nodes and train network in between
for i in range(round(layer_size/nbr_nodes_added)):
if layer_type=='dense':
next_node = layers.Dense(nbr_nodes_added,activation='relu')(new_layer_input)
next_flattened = next_node
else:
next_node = layers.Conv2D(nbr_nodes_added,(3,3),activation='relu')(new_layer_input)
next_node = layers.MaxPooling2D((2,2))(next_node)
next_flattened = layers.Flatten()(next_node)
new_layer_flattened_list.append(next_flattened)
if(i==0):
# i = 0 special case. No concatenation needed
next_output = layers.Dense(3)(next_flattened) # Different for i==0
else:
next_output = layers.Concatenate(axis=-1)(new_layer_flattened_list)
next_output = layers.Dense(3)(next_output)
# Construct and compile network
network = models.Model(input_tensor,next_output)
network.compile(optimizer='rmsprop', loss='mse', metrics=['mse', 'mae'])
network.summary()
output_list.append(next_output)
# Train and freeze layers in network
if mp_training:
deeptrack.train_deep_learning_network_mp(
network,
sample_sizes = sample_sizes,
iteration_numbers = iteration_numbers,
verbose=verbose,
SN_limits=SN_limits,
translation_distance=translation_distance,
)
else:
deeptrack.train_deep_learning_network(
network,
train_generator,
sample_sizes = sample_sizes,
iteration_numbers = iteration_numbers,
verbose=verbose)
freeze_all_layers(network)
new_layer_node_list.append(next_node)
if(save_networks):
network.save(model_path+"L"+str(layer_no)+"_"+str((i+1)*nbr_nodes_added)+"F.h5")
avg_out = layers.average(output_list)
network = models.Model(input_tensor,avg_out)
network.compile(optimizer='rmsprop', loss='mse', metrics=['mse', 'mae'])
print('final network architecture')
network.summary()
if(save_networks):
network.save(model_path+"final_L"+str(layer_no)+"_F.h5")
# IF dense statement needed
return network,new_layer_node_list,output_list,new_layer_flattened_list
def independent_avg_modular_deeptrack_L1(# bad name
layer_size,
train_generator,
input_shape=(51,51,1),
output_shape=3,
nbr_nodes_added=1,
sample_sizes=(8, 32, 128, 512, 1024),
iteration_numbers=(401, 301, 201, 101, 51),
verbose=0.01,
save_networks=False,
mp_training=False, # use multiprocessing training
translation_distance=5, # parameters for multiprocessing training
SN_limits=[10,100], # parameters for multiprocessing training
model_path="", # determines the type of layer used for combining the
# predictions. Addition and average only options at the moment
):
"""
First layer in the modular averaging architecture where each layer is
trained independetly of previous ones.
Inputs:
layer_size - size of first layer in model
train_generator - generator for images
output_shape - number of outputs from network, 3 for normal deeptrack
nbr_nodes_added - number of nodes to add at a time
sample_size - same as deeptrack
iteration_numbers - same as deeptrack
save_networks - if networks are to be saved automatically once training is finished
mp_training - use multiprocessing to speed up training. Not available for custom
image generators as supplied by train_generator, uses FBFs own image generator.
translation_distance - parameter for mp_training image generator, determines
the area the particle is allowed to appear in
SN_limits -
Outputs:
TODO - Implement weight decay in later layers and lowering learning rate
"""
import deeptrack
import keras
from keras import Input,models,layers
from keras.models import Model
from feature_by_feature import freeze_all_layers
#import deeptrackelli_mod_mproc as multiprocess_training # Needed for fast parallelized image generator
input_tensor = Input(input_shape)
conv_list = [] # List of convolutional neruons in L1, needed for subsequent layers
flattened_list = [] # List of flattened layers
output_list = [] # List of the output layers
# Loop thorugh the neoruns and add them one by one
for i in range(round(layer_size/nbr_nodes_added)):
next_node = layers.Conv2D(nbr_nodes_added,(3,3),activation='relu')(input_tensor)
next_node = layers.MaxPooling2D((2,2))(next_node)
if(i==0):
# i = 0 special case. No addition needed
next_flattened = layers.Flatten()(next_node)
next_output = layers.Dense(3)(next_flattened)
final_output = next_output
output_list.append(next_output)
flattened_list.append(next_flattened)
else:
# Construct the next output node
next_flattened = layers.Flatten()(next_node)
flattened_list.append(next_flattened)
# Can't concatenate a single layer
if(len(flattened_list)>1):
next_output = layers.Concatenate(axis=-1)(flattened_list)
next_output = layers.Dense(3)(next_output)
output_list.append(next_output)
# Construct and compile network
network = models.Model(input_tensor,next_output)
network.compile(optimizer='rmsprop', loss='mse', metrics=['mse', 'mae'])
network.summary()
# Train and freeze layers in network
if mp_training:
deeptrack.train_deep_learning_network_mp(
network,
sample_sizes = sample_sizes,
iteration_numbers = iteration_numbers,
verbose=verbose,
SN_limits=SN_limits,
translation_distance=translation_distance,
)
else:
deeptrack.train_deep_learning_network(
network,
train_generator,
sample_sizes = sample_sizes,
iteration_numbers = iteration_numbers,
verbose=verbose)
freeze_all_layers(network)
conv_list.append(next_node)
if(save_networks):
network.save(model_path+"L1_"+str((i+1)*nbr_nodes_added)+"F.h5")
# Create final output using all the output layers and averaging them
if(len(output_list)>1):
avg_out = layers.average(output_list)
else:
avg_out = output_list[0]
network = models.Model(input_tensor,avg_out)
network.compile(optimizer='rmsprop', loss='mse', metrics=['mse', 'mae'])
print('final network architecture')
network.summary()
if(save_networks):
network.save(model_path+"final_L"+str(1)+"_F.h5")
return network,conv_list,output_list,flattened_list,input_tensor
def multi_net(img_rows, img_cols, color_type, num_classes=None):
small = L.Input(shape=(img_rows, img_cols, color_type), name='small')
x = conv_block(small, 5)
x = L.Conv2D(36, (5, 5), activation='relu')(small)
x = L.Conv2D(48, (3, 3), activation='relu')(x)
x = L.Flatten()(x)
x1 = L.Dense(256)(x)
x = L.Dense(512, activation='relu')(x)
x = L.Dropout(0.4)(x)
x = L.Dense(512, activation='relu')(x)
x = L.Dropout(0.4)(x)
logits1 = L.Dense(num_classes, activation='softmax')(x)
medium = L.Input(shape=(img_rows * 2, img_cols * 2, color_type),
name='medium')
x = conv_block(medium, 5)
x = L.Conv2D(16, (3, 3), activation='relu')(x)
x = L.Conv2D(16, (3, 3), activation='relu')(x)
x = conv_block(x, 3)
x = L.Conv2D(16, (3, 3), activation='relu')(x)
x = L.Flatten()(x)
x2 = L.Dense(256)(x)
x = L.Dense(512, activation='relu')(x)
x = L.Dropout(0.4)(x)
x = L.Dense(512, activation='relu')(x)
x = L.Dropout(0.4)(x)
logits2 = L.Dense(num_classes, activation='softmax')(x)
large = L.Input(shape=(img_rows * 3, img_cols * 3, color_type),
name='large')
x = conv_block(large, 5)
x = L.Conv2D(16, (3, 3), activation='relu')(x)
x = L.Conv2D(16, (3, 3), activation='relu')(x)
x = conv_block(x, 3)
x = L.Conv2D(16, (3, 3), activation='relu')(x)
x = L.Conv2D(16, (3, 3), activation='relu')(x)
x = L.Conv2D(16, (3, 3), activation='relu')(x)
x = L.Conv2D(16, (3, 3), activation='relu')(x)
x = L.Conv2D(16, (3, 3), activation='relu')(x)
x = L.Flatten()(x)
x3 = L.Dense(256)(x)
x = L.Dense(512, activation='relu')(x)
x = L.Dropout(0.4)(x)
x = L.Dense(512, activation='relu')(x)
x = L.Dropout(0.4)(x)
logits3 = L.Dense(num_classes, activation='softmax')(x)
# # combine all patch
# merge0=L.concatenate([x1, x2, x3],axis=-1)
# merge1=L.Dense(1024)(merge0)
# merge2=L.Activation('relu')(merge1)
# merge2 = L.Dropout(0.4)(merge2)
# merge3=L.Dense(512)(merge2)
# merge3=L.Activation('relu')(merge3)
# merge3 = L.Dropout(0.4)(merge3)
# logits=L.Dense(num_classes,activation='softmax')(merge0)
logits = L.average([logits1, logits2, logits3])
new_model = K.models.Model([small, medium, large], logits)
sgd = K.optimizers.SGD(lr=0.001, momentum=0.99, decay=1e-4)
new_model.compile(optimizer=sgd,
loss='categorical_crossentropy',
metrics=['acc'])
return new_model
def VGG16():
# Determine proper input shape
input_shape = (128, 128, 200)
img_input = layers.Input(shape=input_shape)
# Block 1
x = layers.Conv2D(64, (3, 3),
activation='relu',
padding='same',
name='block1_conv1')(img_input)
x = layers.Conv2D(64, (3, 3),
activation='relu',
padding='same',
name='block1_conv2')(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)
# Block 2
x = layers.Conv2D(128, (3, 3),
activation='relu',
padding='same',
name='block2_conv1')(x)
x = layers.Conv2D(128, (3, 3),
activation='relu',
padding='same',
name='block2_conv2')(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)
# Block 3
x = layers.Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv1')(x)
x = layers.Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv2')(x)
x = layers.Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv3')(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)
# Block 4
x = layers.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv1')(x)
x = layers.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv2')(x)
x = layers.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv3')(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)
# Block 5
x = layers.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv1')(x)
x = layers.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv2')(x)
x = layers.Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv3')(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)
# Classification block
x = layers.Flatten(name='flatten')(x)
x = layers.Dropout(0.3)(x)
x = layers.Dense(4096, activation='relu', name='fc1')(x)
x = layers.Dropout(0.3)(x)
x = layers.Dense(4096, activation='relu', name='fc2')(x)
#x = layers.Dense(2, name='predictions')(x)
inputs = img_input
# Create model.
#model = models.Model(inputs, x, name='vgg16')
y1 = layers.Dense(11, activation='softmax')(x)
y2 = layers.Dense(11, activation='softmax')(x)
model = Model(inputs=inputs, outputs=[y1, y2])
model.load_weights("weight/model_2_v1_tf.hdf5")
adam = Adam(lr=0.0001)
model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=['accuracy'])
return model
import keras.layers as layers
classifier = keras.Sequential()
classifier.add(
layers.Conv2D(filters=6,
kernel_size=(3, 3),
activation='relu',
input_shape=(64, 64, 3)))
classifier.add(layers.AveragePooling2D())
classifier.add(layers.Conv2D(filters=16, kernel_size=(3, 3),
activation='relu'))
classifier.add(layers.AveragePooling2D())
classifier.add(layers.Flatten())
classifier.add(layers.Dense(units=120, activation='relu'))
classifier.add(layers.Dense(units=84, activation='relu'))
classifier.add(layers.Dense(units=1, activation='sigmoid'))
classifier.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
tf.keras.utils.plot_model(classifier,
to_file='model.png',
show_shapes=False,
show_layer_names=True,
textNet.summary()
# In[301]:
SVG(
model_to_dot(textNet, show_layer_names=True,
show_shapes=True).create(prog='dot', format='svg'))
# # Define Complete Joint Network
# In[302]:
image_input = Input(shape=(256, 256, 3), name='image_input')
image_features = imgNet(image_input)
image_features = kl.Dropout(0.1)(image_features)
image_features = kl.Flatten()(image_features)
image_features = kl.Dense(512, activation='relu')(image_features)
words_input = kl.Input(shape=(max_wlen, ), name='words_input')
text_features = textNet(words_input)
text_features = kl.Dropout(0.1)(text_features)
text_features = kl.Dense(512, activation='relu')(text_features)
g = kl.Concatenate()([image_features, text_features])
g = kl.Dense(512, activation='relu')(g)
target = Dense(vocab_size + 1, activation='softmax', name='target_word')(g)
model = Model([image_input, words_input], target)
# In[303]:
print('New model {0}. loading word2vec embeddings'.format(model_name))
def LF_experiment(train_x,
train_y,
test_x,
test_y,
ntrees,
shift,
slot,
model,
num_points_per_task,
acorn=None):
df = pd.DataFrame()
single_task_accuracies = np.zeros(10, dtype=float)
shifts = []
tasks = []
base_tasks = []
accuracies_across_tasks = []
train_times_across_tasks = []
single_task_inference_times_across_tasks = []
multitask_inference_times_across_tasks = []
model_size = []
if model == "dnn":
default_transformer_class = NeuralClassificationTransformer
network = keras.Sequential()
network.add(
layers.Conv2D(filters=16,
kernel_size=(3, 3),
activation='relu',
input_shape=np.shape(train_x)[1:]))
network.add(layers.BatchNormalization())
network.add(
layers.Conv2D(filters=32,
kernel_size=(3, 3),
strides=2,
padding="same",
activation='relu'))
network.add(layers.BatchNormalization())
network.add(
layers.Conv2D(filters=64,
kernel_size=(3, 3),
strides=2,
padding="same",
activation='relu'))
network.add(layers.BatchNormalization())
network.add(
layers.Conv2D(filters=128,
kernel_size=(3, 3),
strides=2,
padding="same",
activation='relu'))
network.add(layers.BatchNormalization())
network.add(
layers.Conv2D(filters=254,
kernel_size=(3, 3),
strides=2,
padding="same",
activation='relu'))
network.add(layers.Flatten())
network.add(layers.BatchNormalization())
network.add(layers.Dense(2000, activation='relu'))
network.add(layers.BatchNormalization())
network.add(layers.Dense(2000, activation='relu'))
network.add(layers.BatchNormalization())
network.add(layers.Dense(units=10, activation='softmax'))
default_transformer_kwargs = {
"network": network,
"euclidean_layer_idx": -2,
"num_classes": 10,
"optimizer": keras.optimizers.Adam(3e-4)
}
default_voter_class = KNNClassificationVoter
default_voter_kwargs = {"k": int(np.log2(num_points_per_task))}
default_decider_class = SimpleArgmaxAverage
elif model == "uf":
default_transformer_class = TreeClassificationTransformer
default_transformer_kwargs = {
"kwargs": {
"max_depth": None,
"max_features": "auto"
}
}
default_voter_class = TreeClassificationVoter
default_voter_kwargs = {}
default_decider_class = SimpleArgmaxAverage
progressive_learner = ProgressiveLearner(
default_transformer_class=default_transformer_class,
default_transformer_kwargs=default_transformer_kwargs,
default_voter_class=default_voter_class,
default_voter_kwargs=default_voter_kwargs,
default_decider_class=default_decider_class)
for task_ii in range(10):
print("Starting Task {} For Fold {}".format(task_ii, shift))
if acorn is not None:
np.random.seed(acorn)
reduced_sample_no = int(num_points_per_task * (1.29**task_ii))
print(reduced_sample_no)
train_start_time = time.time()
progressive_learner.add_task(
X=train_x[task_ii * 5000 +
slot * reduced_sample_no:task_ii * 5000 +
(slot + 1) * reduced_sample_no],
y=train_y[task_ii * 5000 +
slot * reduced_sample_no:task_ii * 5000 +
(slot + 1) * reduced_sample_no],
num_transformers=1 if model == "dnn" else ntrees,
transformer_voter_decider_split=[0.67, 0.33, 0],
decider_kwargs={
"classes":
np.unique(train_y[task_ii * 5000 +
slot * reduced_sample_no:task_ii * 5000 +
(slot + 1) * reduced_sample_no])
})
train_end_time = time.time()
train_times_across_tasks.append(train_end_time - train_start_time)
model_size.append(getsize(progressive_learner))
print(model_size)
single_task_inference_start_time = time.time()
llf_task = progressive_learner.predict(
X=test_x[task_ii * 1000:(task_ii + 1) * 1000, :],
transformer_ids=[task_ii],
task_id=task_ii)
single_task_inference_end_time = time.time()
single_task_accuracies[task_ii] = np.mean(
llf_task == test_y[task_ii * 1000:(task_ii + 1) * 1000])
single_task_inference_times_across_tasks.append(
single_task_inference_end_time - single_task_inference_start_time)
for task_jj in range(task_ii + 1):
multitask_inference_start_time = time.time()
llf_task = progressive_learner.predict(
X=test_x[task_jj * 1000:(task_jj + 1) * 1000, :],
task_id=task_jj)
multitask_inference_end_time = time.time()
shifts.append(shift)
tasks.append(task_jj + 1)
base_tasks.append(task_ii + 1)
accuracies_across_tasks.append(
np.mean(llf_task == test_y[task_jj * 1000:(task_jj + 1) *
1000]))
multitask_inference_times_across_tasks.append(
multitask_inference_end_time - multitask_inference_start_time)
df['data_fold'] = shifts
df['task'] = tasks
df['base_task'] = base_tasks
df['accuracy'] = accuracies_across_tasks
df['multitask_inference_times'] = multitask_inference_times_across_tasks
df_single_task = pd.DataFrame()
df_single_task['task'] = range(1, 11)
df_single_task['data_fold'] = shift
df_single_task['accuracy'] = single_task_accuracies
df_single_task[
'single_task_inference_times'] = single_task_inference_times_across_tasks
df_single_task['train_times'] = train_times_across_tasks
df_single_task['model_size'] = model_size
summary = (df, df_single_task)
file_to_save = 'result/result/increased_sample_' + model + str(
ntrees) + '_' + str(shift) + '_' + str(slot) + '.pickle'
with open(file_to_save, 'wb') as f:
pickle.dump(summary, f)
lenc = MultiLabelBinarizer()
Y = lenc.fit_transform(Y)
feature_maps_shape = (16, 224, 224, 3)
feature_maps_tf = tf.placeholder(tf.float32, shape=feature_maps_shape)
#feature_maps_np = np.ones(feature_maps_tf.shape, dtype='float32')
#feature_maps_np[0, img_height-1, img_width-3, 0] = 50
roiss_tf = tf.placeholder(tf.float32, shape=(16, 2, 4))
#roiss_np = np.asarray([[[0.5,0.2,0.7,0.4], [0.0,0.0,1.0,1.0]]], dtype='float32')
roi_layer = ROI_pooling.ROIPoolingLayer(7, 7)
pooled_features = roi_layer([feature_maps_tf, roiss_tf])
additional_model = models.Sequential()
additional_model.add(pre_trained_vgg)
additional_model.add(layers.Flatten())
#additional_model.add(pooled_features)
additional_model.add(layers.Dense(4096, activation='relu', name='fc1'))
additional_model.add(layers.Dense(4096, activation='relu', name='fc2'))
additional_model.add(layers.Dense(20, activation='softmax', name='classifier'))
additional_model.summary()
additional_model.compile(optimizer='adam',
loss=keras.losses.categorical_crossentropy,
metrics=["accuracy"])
# 일단은 돌아가게는 만들었으나 정확도는 거의 없다시피하다 Y에는 이름만 들어가있는 상태이며, 현재는 돌아가긴 하는 상황이니 ROI_Pooling 계층에 대해서 알아봐야겠다
hist = additional_model.fit_generator(X,
steps_per_epoch=10,
epochs=5,
def single_ae(
enc_size,
input_shape,
name='single_ae',
prefix=None,
ae_type='dense', # 'dense', or 'conv'
conv_size=None,
input_model=None,
enc_lambda_layers=None,
batch_norm=True,
padding='same',
activation=None,
include_mu_shift_layer=False,
do_vae=False):
"""single-layer Autoencoder (i.e. input - encoding - output"""
# naming
model_name = name
if prefix is None:
prefix = model_name
if enc_lambda_layers is None:
enc_lambda_layers = []
# prepare input
input_name = '%s_input' % prefix
if input_model is None:
assert input_shape is not None, 'input_shape of input_model is necessary'
input_tensor = KL.Input(shape=input_shape, name=input_name)
last_tensor = input_tensor
else:
input_tensor = input_model.input
last_tensor = input_model.output
input_shape = last_tensor.shape.as_list()[1:]
input_nb_feats = last_tensor.shape.as_list()[-1]
# prepare conv type based on input
if ae_type == 'conv':
ndims = len(input_shape) - 1
convL = getattr(KL, 'Conv%dD' % ndims)
assert conv_size is not None, 'with conv ae, need conv_size'
conv_kwargs = {'padding': padding, 'activation': activation}
# if want to go through a dense layer in the middle of the U, need to:
# - flatten last layer if not flat
# - do dense encoding and decoding
# - unflatten (rehsape spatially) at end
if ae_type == 'dense' and len(input_shape) > 1:
name = '%s_ae_%s_down_flat' % (prefix, ae_type)
last_tensor = KL.Flatten(name=name)(last_tensor)
# recall this layer
pre_enc_layer = last_tensor
# encoding layer
if ae_type == 'dense':
assert len(
enc_size) == 1, "enc_size should be of length 1 for dense layer"
enc_size_str = ''.join(['%d_' % d for d in enc_size])[:-1]
name = '%s_ae_mu_enc_dense_%s' % (prefix, enc_size_str)
last_tensor = KL.Dense(enc_size[0], name=name)(pre_enc_layer)
else: # convolution
# convolve then resize. enc_size should be [nb_dim1, nb_dim2, ..., nb_feats]
assert len(enc_size) == len(input_shape), \
"encoding size does not match input shape %d %d" % (len(enc_size), len(input_shape))
if list(enc_size)[:-1] != list(input_shape)[:-1] and \
all([f is not None for f in input_shape[:-1]]) and \
all([f is not None for f in enc_size[:-1]]):
# assert len(enc_size) - 1 == 2, "Sorry, I have not yet implemented non-2D resizing -- need to check out interpn!"
name = '%s_ae_mu_enc_conv' % (prefix)
last_tensor = convL(enc_size[-1],
conv_size,
name=name,
**conv_kwargs)(pre_enc_layer)
name = '%s_ae_mu_enc' % (prefix)
zf = [
enc_size[:-1][f] / last_tensor.shape.as_list()[1:-1][f]
for f in range(len(enc_size) - 1)
]
last_tensor = layers.Resize(zoom_factor=zf, name=name)(last_tensor)
# resize_fn = lambda x: tf.image.resize_bilinear(x, enc_size[:-1])
# last_tensor = KL.Lambda(resize_fn, name=name)(last_tensor)
elif enc_size[
-1] is None: # convolutional, but won't tell us bottleneck
name = '%s_ae_mu_enc' % (prefix)
last_tensor = KL.Lambda(lambda x: x, name=name)(pre_enc_layer)
else:
name = '%s_ae_mu_enc' % (prefix)
last_tensor = convL(enc_size[-1],
conv_size,
name=name,
**conv_kwargs)(pre_enc_layer)
if include_mu_shift_layer:
# shift
name = '%s_ae_mu_shift' % (prefix)
last_tensor = layers.LocalBias(name=name)(last_tensor)
# encoding clean-up layers
for layer_fcn in enc_lambda_layers:
lambda_name = layer_fcn.__name__
name = '%s_ae_mu_%s' % (prefix, lambda_name)
last_tensor = KL.Lambda(layer_fcn, name=name)(last_tensor)
if batch_norm is not None:
name = '%s_ae_mu_bn' % (prefix)
last_tensor = KL.BatchNormalization(axis=batch_norm,
name=name)(last_tensor)
# have a simple layer that does nothing to have a clear name before sampling
name = '%s_ae_mu' % (prefix)
last_tensor = KL.Lambda(lambda x: x, name=name)(last_tensor)
# if doing variational AE, will need the sigma layer as well.
if do_vae:
mu_tensor = last_tensor
# encoding layer
if ae_type == 'dense':
name = '%s_ae_sigma_enc_dense_%s' % (prefix, enc_size_str)
last_tensor = KL.Dense(
enc_size[0],
name=name,
# kernel_initializer=keras.initializers.RandomNormal(mean=0.0, stddev=1e-5),
# bias_initializer=keras.initializers.RandomNormal(mean=-5.0, stddev=1e-5)
)(pre_enc_layer)
else:
if list(enc_size)[:-1] != list(input_shape)[:-1] and \
all([f is not None for f in input_shape[:-1]]) and \
all([f is not None for f in enc_size[:-1]]):
assert len(
enc_size
) - 1 == 2, "Sorry, I have not yet implemented non-2D resizing..."
name = '%s_ae_sigma_enc_conv' % (prefix)
last_tensor = convL(enc_size[-1],
conv_size,
name=name,
**conv_kwargs)(pre_enc_layer)
name = '%s_ae_sigma_enc' % (prefix)
resize_fn = lambda x: tf.image.resize_bilinear(
x, enc_size[:-1])
last_tensor = KL.Lambda(resize_fn, name=name)(last_tensor)
elif enc_size[
-1] is None: # convolutional, but won't tell us bottleneck
name = '%s_ae_sigma_enc' % (prefix)
last_tensor = convL(pre_enc_layer.shape.as_list()[-1],
conv_size,
name=name,
**conv_kwargs)(pre_enc_layer)
# cannot use lambda, then mu and sigma will be same layer.
# last_tensor = KL.Lambda(lambda x: x, name=name)(pre_enc_layer)
else:
name = '%s_ae_sigma_enc' % (prefix)
last_tensor = convL(enc_size[-1],
conv_size,
name=name,
**conv_kwargs)(pre_enc_layer)
# encoding clean-up layers
for layer_fcn in enc_lambda_layers:
lambda_name = layer_fcn.__name__
name = '%s_ae_sigma_%s' % (prefix, lambda_name)
last_tensor = KL.Lambda(layer_fcn, name=name)(last_tensor)
if batch_norm is not None:
name = '%s_ae_sigma_bn' % (prefix)
last_tensor = KL.BatchNormalization(axis=batch_norm,
name=name)(last_tensor)
# have a simple layer that does nothing to have a clear name before sampling
name = '%s_ae_sigma' % (prefix)
last_tensor = KL.Lambda(lambda x: x, name=name)(last_tensor)
logvar_tensor = last_tensor
# VAE sampling
sampler = _VAESample().sample_z
name = '%s_ae_sample' % (prefix)
last_tensor = KL.Lambda(sampler, name=name)([mu_tensor, logvar_tensor])
if include_mu_shift_layer:
# shift
name = '%s_ae_sample_shift' % (prefix)
last_tensor = layers.LocalBias(name=name)(last_tensor)
# decoding layer
if ae_type == 'dense':
name = '%s_ae_%s_dec_flat_%s' % (prefix, ae_type, enc_size_str)
last_tensor = KL.Dense(np.prod(input_shape), name=name)(last_tensor)
# unflatten if dense method
if len(input_shape) > 1:
name = '%s_ae_%s_dec' % (prefix, ae_type)
last_tensor = KL.Reshape(input_shape, name=name)(last_tensor)
else:
if list(enc_size)[:-1] != list(input_shape)[:-1] and \
all([f is not None for f in input_shape[:-1]]) and \
all([f is not None for f in enc_size[:-1]]):
name = '%s_ae_mu_dec' % (prefix)
zf = [
last_tensor.shape.as_list()[1:-1][f] / enc_size[:-1][f]
for f in range(len(enc_size) - 1)
]
last_tensor = layers.Resize(zoom_factor=zf, name=name)(last_tensor)
# resize_fn = lambda x: tf.image.resize_bilinear(x, input_shape[:-1])
# last_tensor = KL.Lambda(resize_fn, name=name)(last_tensor)
name = '%s_ae_%s_dec' % (prefix, ae_type)
last_tensor = convL(input_nb_feats,
conv_size,
name=name,
**conv_kwargs)(last_tensor)
if batch_norm is not None:
name = '%s_bn_ae_%s_dec' % (prefix, ae_type)
last_tensor = KL.BatchNormalization(axis=batch_norm,
name=name)(last_tensor)
# create the model and retun
model = Model(inputs=input_tensor, outputs=[last_tensor], name=model_name)
return model
evaluate_naive_method()
def visualize(history):
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)
plt.figure()
plt.plot(epochs, loss, 'bo')
plt.plot(epochs, val_loss, 'b')
plt.show()
## simple model ##
model = models.Sequential()
model.add(layers.Flatten(input_shape=(lookback // step, float_data.shape[-1])))
model.add(layers.Dense(32, activation='relu'))
model.add(layers.Dense(1))
model.compile(optimizer=optimizers.adam(), loss='mae')
history = model.fit_generator(train_gen,
steps_per_epoch=500,
epochs=20,
validation_data=val_gen,
validation_steps=val_steps)
## rnn model ##
model = models.Sequential()
model.add(layers.GRU(units=32, input_shape=(None, float_data.shape[-1])))
model.add(layers.Dense(1))
model.compile(optimizer='adam', loss='mae')
history = model.fit_generator(train_gen,
": ")))
fully_connected_size = int(input("Fully connected layer size: "))
epochs_phase_1 = int(input("Phase 1 epochs: "))
epochs_phase_2 = int(input("Phase 2 epochs: "))
batch_size = int(input("Batch size: "))
dropout_rate = float(input("Dropout rate: "))
# Clear previous layer session to match continiously constructed layer names in weights file
K.clear_session()
# Construct the classifier NN(input -> encoder -> Flatten -> FC -> output with 10 classes(0 - 9))
encoded = encoder(input_img, convolutional_layers,
convolutional_filter_size,
convolutional_filters_per_layer, dropout_rate)
flatten = layers.Flatten()(encoded)
fc = layers.Dense(fully_connected_size, activation='relu')(flatten)
dropout = layers.Dropout(rate=dropout_rate)(fc)
output_layer = layers.Dense(training_labels.num_classes(),
activation='softmax')(dropout)
classifier = Model(input_img, output_layer)
classifier.compile(loss='categorical_crossentropy',
optimizer=optimizers.Adam())
# Print it's summary
classifier.summary()
# Load encoder weights
classifier.load_weights(autoencoder_weights_file, by_name=True)
def Conv2DMultiTaskIn1(x_train, y_train, ddg_train, x_test, y_test, ddg_test,
x_val, y_val, ddg_val, class_weights_dict, modeldir):
K.clear_session()
summary = False
verbose = 0
# setHyperParams------------------------------------------------------------------------------------------------
batch_size = 64
epochs = 200
lr = 0.0049
optimizer = 'sgd'
activator = 'elu'
basic_conv2D_layers = 1
basic_conv2D_filter_num = 16
loop_dilation2D_layers = 2
loop_dilation2D_filter_num = 64
loop_dilation2D_dropout_rate = 0.2008
dilation_lower = 2
dilation_upper = 16
reduce_layers = 3 # conv 3 times: 120 => 60 => 30 => 15
# print(reduce_layers)
reduce_conv2D_filter_num = 16
reduce_conv2D_dropout_rate = 0.1783
residual_stride = 2
dense1_num = 128
dense2_num = 32
drop_num = 0.2605
kernel_size = (3, 3)
pool_size = (2, 2)
initializer = 'random_uniform'
padding_style = 'same'
loss_type = 'mse'
metrics = ('mae', pearson_r)
my_callbacks = [
callbacks.ReduceLROnPlateau(
monitor='val_loss',
factor=0.5,
patience=5,
)
]
if lr > 0:
if optimizer == 'adam':
chosed_optimizer = optimizers.Adam(lr=lr)
elif optimizer == 'sgd':
chosed_optimizer = optimizers.SGD(lr=lr)
elif optimizer == 'rmsprop':
chosed_optimizer = optimizers.RMSprop(lr=lr)
# build --------------------------------------------------------------------------------------------------------
## basic Conv2D
input_layer = Input(shape=x_train.shape[1:])
y = layers.Conv2D(basic_conv2D_filter_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(input_layer)
y = layers.BatchNormalization(axis=-1)(y)
if basic_conv2D_layers == 2:
y = layers.Conv2D(basic_conv2D_filter_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(y)
y = layers.BatchNormalization(axis=-1)(y)
## loop with Conv2D with dilation (padding='same')
for _ in range(loop_dilation2D_layers):
y = layers.Conv2D(loop_dilation2D_filter_num,
kernel_size,
padding=padding_style,
dilation_rate=dilation_lower,
kernel_initializer=initializer,
activation=activator)(y)
y = layers.BatchNormalization(axis=-1)(y)
y = layers.Dropout(loop_dilation2D_dropout_rate)(y)
dilation_lower *= 2
if dilation_lower > dilation_upper:
dilation_lower = 2
## Conv2D with dilation (padding='valaid') and residual block to reduce dimention.
for _ in range(reduce_layers):
y = layers.Conv2D(reduce_conv2D_filter_num,
kernel_size,
padding=padding_style,
kernel_initializer=initializer,
activation=activator)(y)
y = layers.BatchNormalization(axis=-1)(y)
y = layers.Dropout(reduce_conv2D_dropout_rate)(y)
y = layers.MaxPooling2D(pool_size, padding=padding_style)(y)
residual = layers.Conv2D(reduce_conv2D_filter_num,
1,
strides=residual_stride,
padding='same')(input_layer)
y = layers.add([y, residual])
residual_stride *= 2
## flat & dense
y = layers.Flatten()(y)
y = layers.Dense(dense1_num, activation=activator)(y)
y = layers.BatchNormalization(axis=-1)(y)
y = layers.Dropout(drop_num)(y)
y = layers.Dense(dense2_num, activation=activator)(y)
y = layers.BatchNormalization(axis=-1)(y)
y = layers.Dropout(drop_num)(y)
output_layer = layers.Dense(1)(y)
model = models.Model(inputs=input_layer, outputs=output_layer)
if summary:
model.summary()
model.compile(
optimizer=chosed_optimizer,
loss=loss_type,
metrics=list(metrics) # accuracy
)
K.set_session(tf.Session(graph=model.output.graph))
init = K.tf.global_variables_initializer()
K.get_session().run(init)
result = model.fit(
x=x_train,
y=ddg_train,
batch_size=batch_size,
epochs=epochs,
verbose=verbose,
callbacks=my_callbacks,
validation_data=(x_test, ddg_test),
shuffle=True,
)
# print('\n----------History:\n%s'%result.history)
model_json = model.to_json()
with open(modeldir + '/model.json', 'w') as json_file:
json_file.write(model_json)
return model, result.history
#
# Then, we develop a `discriminator` model, that takes as input a candidate image (real or synthetic) and classifies it into one of two
# classes, either "generated image" or "real image that comes from the training set".
# In[2]:
discriminator_input = layers.Input(shape=(height, width, channels))
x = layers.Conv2D(128, 3)(discriminator_input)
x = layers.LeakyReLU()(x)
x = layers.Conv2D(128, 4, strides=2)(x)
x = layers.LeakyReLU()(x)
x = layers.Conv2D(128, 4, strides=2)(x)
x = layers.LeakyReLU()(x)
x = layers.Conv2D(128, 4, strides=2)(x)
x = layers.LeakyReLU()(x)
x = layers.Flatten()(x)
# One dropout layer - important trick!
x = layers.Dropout(0.4)(x)
# Classification layer
x = layers.Dense(1, activation='sigmoid')(x)
discriminator = keras.models.Model(discriminator_input, x)
discriminator.summary()
# To stabilize training, we use learning rate decay
# and gradient clipping (by value) in the optimizer.
discriminator_optimizer = keras.optimizers.RMSprop(lr=0.0008,
clipvalue=1.0,
decay=1e-8)
def build_model(**params):
# TODO: get all these from **params
CNN = 'resnet'
INCLUDE_TOP = False
LEARNABLE_CNN_LAYERS = params['learnable_cnn_layers']
RNN_TYPE = 'LSTM'
RNN_SIZE = 1024
WORDVEC_SIZE = params['wordvec_size']
ACTIVATION = 'relu'
USE_CGRU = params['use_cgru']
CGRU_SIZE = params['cgru_size']
REDUCE_MEAN = params['reduce_visual']
max_words = params['max_words']
if CNN == 'vgg16':
cnn = applications.vgg16.VGG16(include_top=INCLUDE_TOP)
elif CNN == 'resnet':
cnn = applications.resnet50.ResNet50(include_top=INCLUDE_TOP)
# Pop the mean pooling layer
cnn = models.Model(inputs=cnn.inputs, outputs=cnn.layers[-2].output)
for layer in cnn.layers[:-LEARNABLE_CNN_LAYERS]:
layer.trainable = False
# Context Vector input
# normalized to [0,1] the values:
# left, top, right, bottom, (box area / image area)
input_ctx = layers.Input(shape=(5, ))
ctx = layers.BatchNormalization()(input_ctx)
repeat_ctx = layers.RepeatVector(max_words)(ctx)
# Global Image featuers (convnet output for the whole image)
input_img_global = layers.Input(shape=(IMG_HEIGHT, IMG_WIDTH,
IMG_CHANNELS))
image_global = cnn(input_img_global)
# Add a residual CGRU layer
if USE_CGRU:
image_global = layers.Conv2D(CGRU_SIZE, (1, 1),
padding='same',
activation='relu')(image_global)
res_cgru = SpatialCGRU(image_global, CGRU_SIZE)
image_global = layers.add([image_global, res_cgru])
if REDUCE_MEAN:
image_global = layers.Lambda(lambda x: tf.reduce_mean(x, axis=1))(
image_global)
image_global = layers.Lambda(lambda x: tf.reduce_mean(x, axis=1))(
image_global)
else:
image_global = layers.Conv2D(WORDVEC_SIZE / 4, (3, 3),
activation='relu')(image_global)
image_global = layers.Conv2D(WORDVEC_SIZE / 2, (3, 3),
activation='relu')(image_global)
image_global = layers.Flatten()(image_global)
image_global = layers.Concatenate()([image_global, ctx])
image_global = layers.Dense(1024, activation='relu')(image_global)
image_global = layers.BatchNormalization()(image_global)
image_global = layers.Dense(WORDVEC_SIZE / 2,
activation=ACTIVATION)(image_global)
image_global = layers.BatchNormalization()(image_global)
image_global = layers.RepeatVector(max_words)(image_global)
# Local Image featuers (convnet output for just the bounding box)
input_img_local = layers.Input(shape=(IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS))
image_local = cnn(input_img_local)
if USE_CGRU:
image_local = layers.Conv2D(CGRU_SIZE, (1, 1),
padding='same',
activation='relu')(image_local)
res_cgru = SpatialCGRU(image_local, CGRU_SIZE)
image_local = layers.add([image_local, res_cgru])
if REDUCE_MEAN:
image_local = layers.Lambda(lambda x: tf.reduce_mean(x, axis=1))(
image_local)
image_local = layers.Lambda(lambda x: tf.reduce_mean(x, axis=1))(
image_local)
else:
image_local = layers.Conv2D(WORDVEC_SIZE / 4, (3, 3),
activation='relu')(image_local)
image_local = layers.Conv2D(WORDVEC_SIZE / 2, (3, 3),
activation='relu')(image_local)
image_local = layers.Flatten()(image_local)
image_local = layers.Concatenate()([image_local, ctx])
image_local = layers.Dense(1024, activation='relu')(image_local)
image_local = layers.BatchNormalization()(image_local)
image_local = layers.Dense(WORDVEC_SIZE / 2,
activation=ACTIVATION)(image_local)
image_local = layers.BatchNormalization()(image_local)
image_local = layers.RepeatVector(max_words)(image_local)
language_model = models.Sequential()
input_words = layers.Input(shape=(max_words, ), dtype='int32')
language = layers.Embedding(words.VOCABULARY_SIZE,
WORDVEC_SIZE,
input_length=max_words)(input_words)
x = layers.concatenate([image_global, image_local, repeat_ctx, language])
if RNN_TYPE == 'LSTM':
x = layers.LSTM(RNN_SIZE)(x)
else:
x = layers.GRU(RNN_SIZE)(x)
x = layers.BatchNormalization()(x)
x = layers.Dense(words.VOCABULARY_SIZE, activation='softmax')(x)
return models.Model(
inputs=[input_img_global, input_img_local, input_words, input_ctx],
outputs=x)
def Conv2DClassifierIn1(x_train,y_train,x_test,y_test):
summary = True
verbose = 1
# setHyperParams------------------------------------------------------------------------------------------------
batch_size = {{choice([32,64,128,256,512])}}
epoch = {{choice([25,50,75,100,125,150,175,200])}}
conv_block={{choice(['two', 'three', 'four'])}}
conv1_num={{choice([8, 16, 32, 64])}}
conv2_num={{choice([16,32,64,128])}}
conv3_num={{choice([32,64,128])}}
conv4_num={{choice([32, 64, 128, 256])}}
dense1_num={{choice([128, 256, 512])}}
dense2_num={{choice([64, 128, 256])}}
l1_regular_rate = {{uniform(0.00001, 1)}}
l2_regular_rate = {{uniform(0.000001, 1)}}
drop1_num={{uniform(0.1, 1)}}
drop2_num={{uniform(0.0001, 1)}}
activator={{choice(['elu','relu','tanh'])}}
optimizer={{choice(['adam','rmsprop','SGD'])}}
#---------------------------------------------------------------------------------------------------------------
kernel_size = (3, 3)
pool_size = (2, 2)
initializer = 'random_uniform'
padding_style = 'same'
loss_type='binary_crossentropy'
metrics=['accuracy']
my_callback = None
# early_stopping = EarlyStopping(monitor='val_loss', patience=4)
# checkpointer = ModelCheckpoint(filepath='keras_weights.hdf5',
# verbose=1,
# save_best_only=True)
# my_callback = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2,
# patience=5, min_lr=0.0001)
# build --------------------------------------------------------------------------------------------------------
input_layer = Input(shape=x_train.shape[1:])
conv = layers.Conv2D(conv1_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(input_layer)
conv = layers.Conv2D(conv1_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(conv)
pool = layers.MaxPooling2D(pool_size,padding=padding_style)(conv)
if conv_block == 'two':
conv = layers.Conv2D(conv2_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(pool)
conv = layers.Conv2D(conv2_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(conv)
BatchNorm = layers.BatchNormalization(axis=-1)(conv)
pool = layers.MaxPooling2D(pool_size,padding=padding_style)(BatchNorm)
elif conv_block == 'three':
conv = layers.Conv2D(conv2_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(pool)
conv = layers.Conv2D(conv2_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(conv)
BatchNorm = layers.BatchNormalization(axis=-1)(conv)
pool = layers.MaxPooling2D(pool_size,padding=padding_style)(BatchNorm)
conv = layers.Conv2D(conv3_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(pool)
conv = layers.Conv2D(conv3_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(conv)
BatchNorm = layers.BatchNormalization(axis=-1)(conv)
pool = layers.MaxPooling2D(pool_size,padding=padding_style)(BatchNorm)
elif conv_block == 'four':
conv = layers.Conv2D(conv2_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(pool)
conv = layers.Conv2D(conv2_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(conv)
BatchNorm = layers.BatchNormalization(axis=-1)(conv)
pool = layers.MaxPooling2D(pool_size,padding=padding_style)(BatchNorm)
conv = layers.Conv2D(conv3_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(pool)
conv = layers.Conv2D(conv3_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(conv)
BatchNorm = layers.BatchNormalization(axis=-1)(conv)
pool = layers.MaxPooling2D(pool_size,padding=padding_style)(BatchNorm)
conv = layers.Conv2D(conv4_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(pool)
conv = layers.Conv2D(conv4_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(conv)
BatchNorm = layers.BatchNormalization(axis=-1)(conv)
pool = layers.MaxPooling2D(pool_size,padding=padding_style)(BatchNorm)
flat = layers.Flatten()(pool)
drop = layers.Dropout(drop1_num)(flat)
dense = layers.Dense(dense1_num, activation=activator, kernel_regularizer=regularizers.l1_l2(l1=l1_regular_rate,l2=l2_regular_rate))(drop)
BatchNorm = layers.BatchNormalization(axis=-1)(dense)
drop = layers.Dropout(drop2_num)(BatchNorm)
dense = layers.Dense(dense2_num, activation=activator, kernel_regularizer=regularizers.l1_l2(l1=l1_regular_rate,l2=l2_regular_rate))(drop)
output_layer = layers.Dense(len(np.unique(y_train)),activation='softmax')(dense)
model = models.Model(inputs=input_layer, outputs=output_layer)
if summary:
model.summary()
# train(self):
class_weights = class_weight.compute_class_weight('balanced', np.unique(y_train), y_train.reshape(-1))
class_weights_dict = dict(enumerate(class_weights))
model.compile(optimizer=optimizer,
loss=loss_type,
metrics=metrics # accuracy
)
result = model.fit(x=x_train,
y=y_train,
batch_size=batch_size,
epochs=epoch,
verbose=verbose,
callbacks=my_callback,
validation_data=(x_test, y_test),
shuffle=True,
class_weight=class_weights_dict
)
validation_acc = np.amax(result.history['val_acc'])
print('Best validation acc of epoch:', validation_acc)
return {'loss': -validation_acc, 'status': STATUS_OK, 'model': model}
from keras import layers
from keras import models
from keras.datasets import cifar10
from keras.utils import to_categorical
from sklearn.model_selection import train_test_split
import numpy as np
model1 = models.Sequential([
layers.Conv2D(32, (3, 3), activation='relu', input_shape=(32, 32, 3)),
layers.MaxPooling2D((2, 2)),
layers.Flatten(),
layers.Dense(32, activation='relu'),
layers.Dense(3, activation='softmax')
])
model1.summary()
print("")
model2 = models.Sequential([
layers.Conv2D(32, (3, 3), activation='relu', input_shape=(32, 32, 3)),
layers.MaxPooling2D((2, 2)),
layers.Conv2D(64, (3, 3), activation='relu'),
layers.MaxPooling2D((2, 2)),
layers.Flatten(),
layers.Dense(32, activation='relu'),
layers.Dense(3, activation='softmax')
])
model2.summary()
print("")
model3 = models.Sequential([
layers.Conv2D(32, (3, 3), activation='relu', input_shape=(32, 32, 3)),
# Simple ConvNet
# ConvNet uses (image_height, image_width, image_channels) size input tensor
# in this example, input shape is (28, 28, 1) for MNIST image format
model = models.Sequential()
model.add(layers.Conv2D(32, (3, 3), activation='relu',
input_shape=(28, 28, 1)))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
model.summary()
# Add more layers
model.add(layers.Flatten())
model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dense(10, activation='softmax'))
model.summary()
# ConvNet Training
(train_images, train_labels), (test_images, test_labels) = mnist.load_data()
train_images = train_images.reshape((60000, 28, 28, 1))
train_images = train_images.astype('float32') / 255
test_images = test_images.reshape((10000, 28, 28, 1))
test_images = test_images.astype('float32') / 255
train_labels = to_categorical(train_labels)
kernel_size=8,
strides=2,
activation='relu')(encoded_summary)
# Size: 29
encoded_summary = layers.MaxPool1D(pool_size=2)(encoded_summary)
encoded_summary = layers.Activation('relu')(encoded_summary)
# Size: 14
encoded_summary = layers.Conv1D(filters=60,
kernel_size=4,
strides=1,
activation='relu')(encoded_summary)
# Size: 11
encoded_summary = layers.MaxPool1D(pool_size=2)(encoded_summary)
encoded_summary = layers.Activation('relu')(encoded_summary)
# Size: 5
encoded_summary = layers.Flatten()(encoded_summary)
encoded_summary = layers.RepeatVector(MAX_DOCUMENT_LENGTH)(encoded_summary)
document = layers.Input(shape=(MAX_DOCUMENT_LENGTH, ))
encoded_document = layers.Embedding(np.shape(embedding_matrix)[0],
EMBEDDINGS_SIZE,
weights=[embedding_matrix],
input_length=MAX_DOCUMENT_LENGTH,
trainable=False)(document)
merged = layers.add([encoded_summary, encoded_document])
merged = layers.Bidirectional(
layers.LSTM((int)(EMBEDDINGS_SIZE / 2), return_sequences=True))(merged)
merged = layers.Dropout(0.3)(merged)
merged = layers.Bidirectional(
layers.LSTM((int)(EMBEDDINGS_SIZE / 4), return_sequences=True))(merged)
