def create_model(self, space: Optional[Dict[str, Any]] = None) -> Model: if space: print('Using hyperopt space:') print(space) for_optimization = True if space else False if for_optimization: if space['conv_kind'] == 'standard': img_input = Input(shape=(640, 400, 3), dtype='float32') x = layers.Conv2D( 32 if not for_optimization else space['Conv2D_0'], 5 if not for_optimization else space['kernel'], activation='relu')(img_input) x = layers.MaxPooling2D(2)(x) x = layers.Conv2D( 64 if not for_optimization else space['Conv2D_1'], 5 if not for_optimization else space['kernel'], activation='relu')(x) x = layers.MaxPooling2D(2)(x) x = layers.Conv2D( 128 if not for_optimization else space['Conv2D_2'], 5 if not for_optimization else space['kernel'], activation='relu')(x) x = layers.MaxPooling2D(2)(x) x = layers.Conv2D( 128 if not for_optimization else space['Conv2D_3'], 5 if not for_optimization else space['kernel'], activation='relu')(x) x = layers.MaxPooling2D(2)(x) x = layers.Conv2D( 128 if not for_optimization else space['Conv2D_4'], 5 if not for_optimization else space['kernel'], activation='relu')(x) x = layers.MaxPooling2D(2)(x) x = layers.Flatten()(x) x = layers.Dense(space['dense0'], activation='relu')(x) rot_x_pred = layers.Dense(1, name='rot_x')(x) else: img_input = Input(shape=(640, 400, 3), dtype='float32') x = layers.SeparableConv2D( 32 if not for_optimization else space['Conv2D_0'], 5 if not for_optimization else space['kernel'], activation='relu')(img_input) x = layers.MaxPooling2D(2)(x) x = layers.SeparableConv2D( 64 if not for_optimization else space['Conv2D_1'], 5 if not for_optimization else space['kernel'], activation='relu')(x) x = layers.MaxPooling2D(2)(x) x = layers.SeparableConv2D( 128 if not for_optimization else space['Conv2D_2'], 5 if not for_optimization else space['kernel'], activation='relu')(x) x = layers.MaxPooling2D(2)(x) x = layers.SeparableConv2D( 128 if not for_optimization else space['Conv2D_3'], 5 if not for_optimization else space['kernel'], activation='relu')(x) x = layers.MaxPooling2D(2)(x) x = layers.SeparableConv2D( 128 if not for_optimization else space['Conv2D_4'], 5 if not for_optimization else space['kernel'], activation='relu')(x) x = layers.MaxPooling2D(2)(x) x = layers.Flatten()(x) x = layers.Dense(space['dense0'], activation='relu')(x) rot_x_pred = layers.Dense(1, name='rot_x')(x) else: img_input = Input(shape=(1384, 865, 3), dtype='float32') x = layers.SeparableConv2D(256, 20, strides=(10, 10), activation='relu')(img_input) x = layers.SeparableConv2D(512, 7, strides=(2, 2), activation='relu')(x) x = layers.SeparableConv2D(1024, 7, strides=(2, 2), activation='relu')(x) x = layers.SeparableConv2D(1024, 7, strides=(2, 2), activation='relu')(x) x = layers.Flatten()(x) x = layers.Dense(128, activation='relu')(x) rot_x_pred = layers.Dense(1, name='rot_x')(x) model = Model(img_input, rot_x_pred) model.compile(optimizer='rmsprop', loss='mae') return model
def kale(input_shape=(32, None, 1), num_classes=5991, max_string_len=10): input = layers.Input(shape=input_shape) # swl_output = tf.py_func(sliding_window_layer, [input], tf.float32) swl_output = layers.Lambda(swl_lambda_func)(input) xx = layers.Reshape((-1, 32, 32, 1))(swl_output) print(xx.get_shape()) print(xx.shape) xx._keras_shape = tuple(xx.get_shape().as_list()) print(K.int_shape(xx)) # swl_output = SlidingWindowLayer()(input) x = TimeDistributed( layers.Conv2D(50, (3, 3), padding='same', activation='relu', name='conv1'))(xx) x = TimeDistributed(layers.BatchNormalization(axis=-1))(x) x = TimeDistributed( layers.Conv2D(100, (3, 3), padding='same', activation='relu', name='conv2'))(x) x = TimeDistributed(layers.Dropout(0.1))(x) x = TimeDistributed( layers.Conv2D(100, (3, 3), padding='same', activation='relu', name='conv3'))(x) x = TimeDistributed(layers.Dropout(0.1))(x) x = TimeDistributed(layers.BatchNormalization(axis=-1))(x) x = TimeDistributed(layers.MaxPooling2D((2, 2)))(x) x = TimeDistributed( layers.Conv2D(150, (3, 3), padding='same', activation='relu', name='conv4'))(x) x = TimeDistributed(layers.BatchNormalization(axis=-1))(x) x = TimeDistributed( layers.Conv2D(200, (3, 3), padding='same', activation='relu', name='conv5'))(x) x = TimeDistributed(layers.Dropout(0.2))(x) x = TimeDistributed( layers.Conv2D(200, (3, 3), padding='same', activation='relu', name='conv6'))(x) x = TimeDistributed(layers.Dropout(0.2))(x) x = TimeDistributed(layers.BatchNormalization(axis=-1))(x) x = TimeDistributed(layers.MaxPooling2D((2, 2)))(x) x = TimeDistributed( layers.Conv2D(250, (3, 3), padding='same', activation='relu', name='conv7'))(x) x = TimeDistributed(layers.BatchNormalization(axis=-1))(x) x = TimeDistributed( layers.Conv2D(300, (3, 3), padding='same', activation='relu', name='conv8'))(x) x = TimeDistributed(layers.Dropout(0.3))(x) x = TimeDistributed( layers.Conv2D(300, (3, 3), padding='same', activation='relu', name='conv9'))(x) x = TimeDistributed(layers.Dropout(0.3))(x) x = TimeDistributed(layers.BatchNormalization(axis=-1))(x) x = TimeDistributed(layers.MaxPooling2D((2, 2)))(x) x = TimeDistributed( layers.Conv2D(350, (3, 3), padding='same', activation='relu', name='conv10'))(x) x = TimeDistributed(layers.BatchNormalization(axis=-1))(x) x = TimeDistributed( layers.Conv2D(400, (3, 3), padding='same', activation='relu', name='conv11'))(x) x = TimeDistributed(layers.Dropout(0.4))(x) x = TimeDistributed( layers.Conv2D(400, (3, 3), padding='same', activation='relu', name='conv12'))(x) x = TimeDistributed(layers.Dropout(0.4))(x) x = TimeDistributed(layers.BatchNormalization(axis=-1))(x) x = TimeDistributed(layers.MaxPooling2D((2, 2)))(x) x = TimeDistributed(layers.Flatten())(x) x = TimeDistributed(layers.Dense(900, activation='relu'))(x) x = TimeDistributed(layers.Dropout(0.5))(x) x = TimeDistributed(layers.Dense(200, activation='relu'))(x) classifier_output = TimeDistributed( layers.Dense(num_classes, activation='softmax'))(x) # classifier = models.Model(inputs=input, outputs=classifier_output) # classifier.summary() label = layers.Input(name='label', shape=[max_string_len], dtype='int64') # 序列的长度,此模型中为 (280 - 32) // 4 = 63 seq_length = layers.Input(name='seq_length', shape=[1], dtype='int64') label_length = layers.Input(name='label_length', shape=[1], dtype='int64') ctc_output = layers.Lambda( ctc_lambda_func, output_shape=(1, ), name='ctc')([label, classifier_output, seq_length, label_length]) model = models.Model(inputs=[input, label, seq_length, label_length], outputs=[ctc_output]) model.summary() return model
network = models.Sequential() #initialize the neural network network.add(layers.Conv2D(Channel, kernel_size=Nf, strides=Stride, activation='relu', input_shape=input_shape)) #network.add(layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')) #network.add(layers.Conv2D(Channel, kernel_size=Nf, strides=Stride, activation='relu', padding='valid')) #dropout: spegne casualmente qualche neurone indexlayer = 0 if dropout: network.add(layers.Dropout(dropout_const)) indexlayer += 1 network.add(layers.Flatten()) network.add(layers.Dense(64, activation='relu')) # network.add(layers.Dense(16, activation='relu')) network.add(layers.Dense(2, activation='sigmoid')) network.summary() # Shows information about layers and parameters of the entire network print(len(network.layers))
def design_dnn(nb_features, input_shape, nb_levels, conv_size, nb_labels, feat_mult=1, pool_size=2, padding='same', activation='elu', final_layer='dense-sigmoid', conv_dropout=0, conv_maxnorm=0, nb_input_features=1, batch_norm=False, name=None, prefix=None, use_strided_convolution_maxpool=True, nb_conv_per_level=2): """ "deep" cnn with dense or global max pooling layer @ end... Could use sequential... """ model_name = name if model_name is None: model_name = 'model_1' if prefix is None: prefix = model_name ndims = len(input_shape) input_shape = tuple(input_shape) convL = getattr(KL, 'Conv%dD' % ndims) maxpool = KL.MaxPooling3D if len(input_shape) == 3 else KL.MaxPooling2D if isinstance(pool_size, int): pool_size = (pool_size, ) * ndims # kwargs for the convolution layer conv_kwargs = {'padding': padding, 'activation': activation} if conv_maxnorm > 0: conv_kwargs['kernel_constraint'] = maxnorm(conv_maxnorm) # initialize a dictionary enc_tensors = {} # first layer: input name = '%s_input' % prefix enc_tensors[name] = KL.Input(shape=input_shape + (nb_input_features, ), name=name) last_tensor = enc_tensors[name] # down arm: # add nb_levels of conv + ReLu + conv + ReLu. Pool after each of first nb_levels - 1 layers for level in range(nb_levels): for conv in range(nb_conv_per_level): if conv_dropout > 0: name = '%s_dropout_%d_%d' % (prefix, level, conv) enc_tensors[name] = KL.Dropout(conv_dropout)(last_tensor) last_tensor = enc_tensors[name] name = '%s_conv_%d_%d' % (prefix, level, conv) nb_lvl_feats = np.round(nb_features * feat_mult**level).astype(int) enc_tensors[name] = convL(nb_lvl_feats, conv_size, **conv_kwargs, name=name)(last_tensor) last_tensor = enc_tensors[name] # max pool if use_strided_convolution_maxpool: name = '%s_strided_conv_%d' % (prefix, level) enc_tensors[name] = convL(nb_lvl_feats, pool_size, **conv_kwargs, name=name)(last_tensor) last_tensor = enc_tensors[name] else: name = '%s_maxpool_%d' % (prefix, level) enc_tensors[name] = maxpool(pool_size=pool_size, name=name, padding=padding)(last_tensor) last_tensor = enc_tensors[name] # dense layer if final_layer == 'dense-sigmoid': name = "%s_flatten" % prefix enc_tensors[name] = KL.Flatten(name=name)(last_tensor) last_tensor = enc_tensors[name] name = '%s_dense' % prefix enc_tensors[name] = KL.Dense(1, name=name, activation="sigmoid")(last_tensor) elif final_layer == 'dense-tanh': name = "%s_flatten" % prefix enc_tensors[name] = KL.Flatten(name=name)(last_tensor) last_tensor = enc_tensors[name] name = '%s_dense' % prefix enc_tensors[name] = KL.Dense(1, name=name)(last_tensor) last_tensor = enc_tensors[name] # Omittting BatchNorm for now, it seems to have a cpu vs gpu problem # https://github.com/tensorflow/tensorflow/pull/8906 # https://github.com/fchollet/keras/issues/5802 # name = '%s_%s_bn' % prefix # enc_tensors[name] = KL.BatchNormalization(axis=batch_norm, name=name)(last_tensor) # last_tensor = enc_tensors[name] name = '%s_%s_tanh' % prefix enc_tensors[name] = KL.Activation(activation="tanh", name=name)(last_tensor) elif final_layer == 'dense-softmax': name = "%s_flatten" % prefix enc_tensors[name] = KL.Flatten(name=name)(last_tensor) last_tensor = enc_tensors[name] name = '%s_dense' % prefix enc_tensors[name] = KL.Dense(nb_labels, name=name, activation="softmax")(last_tensor) # global max pooling layer elif final_layer == 'myglobalmaxpooling': name = '%s_batch_norm' % prefix enc_tensors[name] = KL.BatchNormalization(axis=batch_norm, name=name)(last_tensor) last_tensor = enc_tensors[name] name = '%s_global_max_pool' % prefix enc_tensors[name] = KL.Lambda(_global_max_nd, name=name)(last_tensor) last_tensor = enc_tensors[name] name = '%s_global_max_pool_reshape' % prefix enc_tensors[name] = KL.Reshape((1, 1), name=name)(last_tensor) last_tensor = enc_tensors[name] # cannot do activation in lambda layer. Could code inside, but will do extra lyaer name = '%s_global_max_pool_sigmoid' % prefix enc_tensors[name] = KL.Conv1D(1, 1, name=name, activation="sigmoid", use_bias=True)(last_tensor) elif final_layer == 'globalmaxpooling': name = '%s_conv_to_featmaps' % prefix enc_tensors[name] = KL.Conv3D(2, 1, name=name, activation="relu")(last_tensor) last_tensor = enc_tensors[name] name = '%s_global_max_pool' % prefix enc_tensors[name] = KL.GlobalMaxPooling3D(name=name)(last_tensor) last_tensor = enc_tensors[name] # cannot do activation in lambda layer. Could code inside, but will do extra lyaer name = '%s_global_max_pool_softmax' % prefix enc_tensors[name] = KL.Activation('softmax', name=name)(last_tensor) last_tensor = enc_tensors[name] # create the model model = Model(inputs=[enc_tensors['%s_input' % prefix]], outputs=[last_tensor], name=model_name) return model
def LBL_network_classification(conv_layers, dense_layers, train_generator, validation_generator, input_shape=(150, 150, 3), nbr_outputs=1, nbr_epochs=20, save_results=False, network_name="CD", result_dir=""): from keras import Input input_tensor = Input(input_shape) final_layers_list = [] #Create first layer model conv_layer = layers.Conv2D(conv_layers[0], (3, 3), activation='relu')(input_tensor) pooling = layers.MaxPooling2D((2, 2))(conv_layer) output = layers.Flatten()(pooling) output = layers.Dense(nbr_outputs, activation="sigmoid")(output) model = models.Model(input_tensor, output) final_layers_list.append(conv_layer) final_layers_list.append(pooling) model.compile(loss='binary_crossentropy', optimizer=optimizers.RMSprop(lr=1e-4), metrics=['acc']) model.summary() history = model.fit_generator(train_generator, steps_per_epoch=100, epochs=nbr_epochs, validation_data=validation_generator, validation_steps=50) final_layers_list[-1].trainable = False ### add subsequent layers idx = 1 # Counts the layer indices if (save_results): model_name = network_name + "_conv_" + str(nbr_epochs) + "e_" + str( idx) + "L" file_name = model_name + "results.txt" #change so that this print_results_to_file(model, history, file_name, model_name, path=result_dir) model_name = result_dir + model_name + ".h5" model.save(model_name) for conv_size in conv_layers[1:]: idx += 1 ### Create new layer along with pooling etc conv_layer = layers.Conv2D(conv_size, (3, 3), activation='relu')(final_layers_list[-1]) pooling = layers.MaxPooling2D((2, 2))(conv_layer) flatten = layers.Flatten()(pooling) output = layers.Dense(nbr_outputs, activation="sigmoid")(flatten) model = models.Model(input_tensor, output) final_layers_list.append(conv_layer) final_layers_list.append(pooling) ### Set layers in model untrainable for i in range(len(model.layers) - 4): model.layers[i].trainable = False ### compile and fit model model.compile(loss='binary_crossentropy', optimizer=optimizers.RMSprop(lr=1e-4), metrics=['acc']) model.summary() history = model.fit_generator(train_generator, steps_per_epoch=100, epochs=nbr_epochs, validation_data=validation_generator, validation_steps=50) # Save reuslts to a file if (save_results): model_name = network_name + "_conv_" + str( nbr_epochs) + "e_" + str(idx) + "L" file_name = model_name + "results.txt" #change so that this print_results_to_file(model, history, file_name, model_name, path=result_dir) model_name = result_dir + model_name + ".h5" model.save(model_name) ## Add the flatten layer to bridge from conv to dense layers final_layers_list.append(flatten) ### Add the dense layers for dense_size in dense_layers: idx += 1 dense_layer = layers.Dense(dense_size, activation='relu')(final_layers_list[-1]) output = layers.Dense(nbr_outputs, activation="sigmoid")(dense_layer) model = models.Model(input_tensor, output) final_layers_list.append(dense_layer) model = models.Model(input_tensor, output) for i in range(len(model.layers) - 2): model.layers[i].trainable = False model.summary() model.compile(loss='binary_crossentropy', optimizer=optimizers.RMSprop(lr=1e-4), metrics=['acc']) model.summary() history = model.fit_generator(train_generator, steps_per_epoch=100, epochs=nbr_epochs, validation_data=validation_generator, validation_steps=50) if (save_results): model_name = network_name + "_dense_" + str( nbr_epochs) + "e_" + str(idx) + "L" file_name = model_name + "results.txt" #change so that this print_results_to_file(model, history, file_name, model_name, path=result_dir) model_name = result_dir + model_name + ".h5" model.save(model_name) return model, final_layers_list
#x_train, x_validation, y_train, y_validation = cross_validation.train_test_split(x_train, y_train, test_size=0.1, random_state=121) #Advanced topic: use tensorflow directly inside keras we won't discuss now def roc_auc(y_true, y_pred): auc = tf.metrics.auc(y_true, y_pred)[1] keras.backend.get_session().run(tf.local_variables_initializer()) return auc print("data ready") #Model definition net_input= layers.Input( shape= x_train.shape[1:]) net_out= layers.Conv2D(filters=32, kernel_size= 5, strides=1,padding='same', activation='relu' )(net_input) net_out= layers.MaxPooling2D(pool_size=4, strides=2 )(net_out) net_out= layers.Flatten()(net_out) net_out= layers.Dense(N_HIDDEN, activation='relu')(net_out) net_out= layers.Dense(y_train.shape[-1], activation='softmax')(net_out) model = keras.models.Model( inputs=net_input, outputs= net_out ) sgd= keras.optimizers.SGD(lr=LEARNING_RATE, decay=0., momentum=0., nesterov=False) model.compile(loss='categorical_crossentropy',optimizer=sgd, metrics=['accuracy', roc_auc]) print("network created") print("training begin") model.fit(x_train, y_train,epochs=N_EPOCHS, batch_size=BATCH_SIZE, validation_split=0.1) score = tuple(model.evaluate(x_test, y_test, batch_size=BATCH_SIZE)) print("\nTesting evaluation loss %f acc %f roc_auc %f"%score)
train_labels = to_categorical(ori_train_labels.copy()) test_images = ori_test_images.copy() test_images = test_images.reshape((10000, 28, 28, 1)) test_images = test_images.astype('float32') / 255 test_labels = to_categorical(ori_test_labels.copy()) #step 3: Design Model net = models.Sequential() net.add(layers.Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=(28, 28, 1))) # need 4D tensor so we need (batch, 28, 28, 1) net.add(layers.MaxPooling2D((2, 2))) net.add(layers.Conv2D(64, kernel_size=(3, 3), activation='relu')) net.add(layers.MaxPooling2D(pool_size=(2, 2))) net.add(layers.Conv2D(64, kernel_size=(3, 3), activation='relu')) net.add(layers.Flatten()) net.add(layers.Dense(64, activation='relu')) net.add(layers.Dense(10, activation='softmax')) net.summary() # Step 4: Training Data net.compile(optimizer=optimizers.Adam(), loss=losses.categorical_crossentropy, metrics=[metrics.categorical_accuracy]) history = net.fit(train_images, train_labels, epochs=10, batch_size=64, validation_data=(test_images, test_labels)) # Stept 5: In ra bieu do
def Conv1DRegressorIn1(flag): K.clear_session() current_neighbor = space['neighbor'] current_idx_idx = space['idx_idx'] current_batch_size = space['batch_size'] current_conv1D_filter_num1 = space['conv1D_filter_num1'] current_conv1D_filter_num2 = space['conv1D_filter_num2'] current_conv1D_filter_num3 = space['conv1D_filter_num3'] current_dropout_rate_dense = space['dropout_rate_dense'] summary = True verbose = 0 # # setHyperParams # ## hypers for data neighbor = {{choice([50, 60, 70, 80, 90, 100, 110, 120, 130, 140])}} idx_idx = {{choice([0, 1, 2, 3, 4, 5, 6, 7, 8])}} idx_lst = [ [x for x in range(158) if x not in [24, 26]], # 去除无用特征 [ x for x in range(158) if x not in [24, 26] + [x for x in range(1, 6)] + [x for x in range(16, 22)] + [40, 42] ], # 去除无用特征+冗余特征 [ x for x in range(158) if x not in [24, 26] + [x for x in range(0, 22)] ], # 去除无用特征+方位特征 [x for x in range(158) if x not in [24, 26] + [22, 23, 26, 37, 38]], # 去除无用特征+深度特征 [ x for x in range(158) if x not in [24, 26] + [x for x in range(27, 37)] + [x for x in range(40, 46)] ], # 去除无用特征+二级结构信息 # [x for x in range(158) if x not in [24, 26] + [x for x in range(27, 34)] + [x for x in range(40, 46)]],# 去除无用特征+二级结构信息1 # [x for x in range(158) if x not in [24, 26] + [x for x in range(34, 37)] + [x for x in range(40, 46)]],# 去除无用特征+二级结构信息2 [x for x in range(158) if x not in [24, 26] + [46, 47]], # 去除无用特征+实验条件 [ x for x in range(158) if x not in [24, 26] + [39] + [x for x in range(57, 61)] + [x for x in range(48, 57)] + [x for x in range(61, 81)] + [x for x in range(140, 155)] ], # 去除无用特征+所有原子编码 # [x for x in range(158) if x not in [24, 26] + [39] + [x for x in range(57, 61)] + [x for x in range(48, 57)] + [x for x in range(140, 145)]],# 去除无用特征+原子编码1 # [x for x in range(158) if x not in [24, 26] + [39] + [x for x in range(57, 61)] + [x for x in range(61, 77)] + [x for x in range(145, 153)]],# 去除无用特征+原子编码2 # [x for x in range(158) if x not in [24, 26] + [39] + [x for x in range(57, 61)] + [x for x in range(77, 81)] + [x for x in range(153, 155)]],# 去除无用特征+原子编码3 [ x for x in range(158) if x not in [24, 26] + [x for x in range(81, 98)] ], # 去除无用特征+rosetta_energy [ x for x in range(158) if x not in [24, 26] + [x for x in range(98, 140)] + [x for x in range(155, 158)] ] # 去除无用特征+msa ] idx = idx_lst[idx_idx] ## hypers for net lr = 1e-4 # 0.0001 batch_size = {{choice([1, 32, 64, 128])}} epochs = 200 conv1D_filter_num1 = {{choice([16, 32])}} conv1D_filter_num2 = {{choice([16, 32, 64])}} conv1D_filter_num3 = {{choice([32, 64])}} dropout_rate_dense = {{choice([0.1, 0.2, 0.3, 0.4, 0.5])}} metrics = ('mae', pearson_r, rmse) def _data(fold_num, neighbor, idx): train_data_pth = '/dl/sry/mCNN/dataset/deepddg/npz/wild/cross_valid/cro_fold%s_train_center_CA_PCA_False_neighbor_140.npz' % fold_num val_data_pth = '/dl/sry/mCNN/dataset/deepddg/npz/wild/cross_valid/cro_fold%s_valid_center_CA_PCA_False_neighbor_140.npz' % fold_num ## train data train_data = np.load(train_data_pth) x_train = train_data['x'] y_train = train_data['y'] ddg_train = train_data['ddg'].reshape(-1) ## select kneighbor atoms x_train_kneighbor_lst = [] for sample in x_train: dist_arr = sample[:, 0] indices = sorted(dist_arr.argsort()[:neighbor]) x_train_kneighbor_lst.append(sample[indices, :]) x_train = np.array(x_train_kneighbor_lst) ## idx x_train = x_train[:, :, idx] ## val data val_data = np.load(val_data_pth) x_val = val_data['x'] y_val = val_data['y'] ddg_val = val_data['ddg'].reshape(-1) ## select kneighbor atoms x_val_kneighbor_lst = [] for sample in x_val: dist_arr = sample[:, 0] indices = sorted(dist_arr.argsort()[:neighbor]) x_val_kneighbor_lst.append(sample[indices, :]) x_val = np.array(x_val_kneighbor_lst) ## idx x_val = x_val[:, :, idx] # sort row default is chain, pass # reshape and one-hot y_train = to_categorical(y_train) y_val = to_categorical(y_val) # normalization train_shape = x_train.shape val_shape = x_val.shape col_train = train_shape[-1] col_val = val_shape[-1] x_train = x_train.reshape((-1, col_train)) x_val = x_val.reshape((-1, col_val)) mean = x_train.mean(axis=0) std = x_train.std(axis=0) std[np.argwhere(std == 0)] = 0.01 x_train -= mean x_train /= std x_val -= mean x_val /= std x_train = x_train.reshape(train_shape) x_val = x_val.reshape(val_shape) print('x_train: %s' '\ny_train: %s' '\nddg_train: %s' '\nx_val: %s' '\ny_val: %s' '\nddg_val: %s' % (x_train.shape, y_train.shape, ddg_train.shape, x_val.shape, y_val.shape, ddg_val.shape)) return x_train, y_train, ddg_train, x_val, y_val, ddg_val # # cross_valid # hyper_param_tag = '%s_%s_%s_%s_%s_%s_%s' % ( current_neighbor, current_idx_idx, current_batch_size, current_conv1D_filter_num1, current_conv1D_filter_num2, current_conv1D_filter_num3, current_dropout_rate_dense) modeldir = '/dl/sry/projects/from_hp/mCNN/src/Network/deepddg/opt_all_simpleNet_v4/model/%s-%s' % ( hyper_param_tag, time.strftime("%Y.%m.%d.%H.%M.%S", time.localtime())) os.makedirs(modeldir, exist_ok=True) opt_lst = [] for k_count in range(1, 11): print('\n** fold %s is processing **\n' % k_count) filepth = '%s/fold_%s_weights-best.h5' % (modeldir, k_count) my_callbacks = [ callbacks.ReduceLROnPlateau( monitor='val_loss', factor=0.33, patience=5, verbose=verbose, mode='min', min_lr=1e-8, ), callbacks.EarlyStopping(monitor='val_loss', patience=10, verbose=verbose), callbacks.ModelCheckpoint(filepath=filepth, monitor='val_mean_absolute_error', verbose=verbose, save_best_only=True, mode='min', save_weights_only=True) ] x_train, y_train, ddg_train, x_val, y_val, ddg_val = _data( k_count, neighbor, idx) row_num, col_num = x_train.shape[1:3] # # build net # network = models.Sequential() network.add( layers.SeparableConv1D(filters=conv1D_filter_num1, kernel_size=5, activation='relu', input_shape=(row_num, col_num))) network.add(layers.MaxPooling1D(pool_size=2)) network.add( layers.SeparableConv1D(filters=conv1D_filter_num2, kernel_size=5, activation='relu')) network.add(layers.MaxPooling1D(pool_size=2)) network.add( layers.SeparableConv1D(filters=conv1D_filter_num3, kernel_size=3, activation='relu')) network.add(layers.MaxPooling1D(pool_size=2)) network.add(layers.Flatten()) network.add(layers.Dense(128, activation='relu')) network.add(layers.Dropout(dropout_rate_dense)) network.add(layers.Dense(16, activation='relu')) network.add(layers.Dropout(0.3)) network.add(layers.Dense(1)) if summary: trainable_count = int( np.sum([ K.count_params(p) for p in set(network.trainable_weights) ])) non_trainable_count = int( np.sum([ K.count_params(p) for p in set(network.non_trainable_weights) ])) print('Total params: {:,}'.format(trainable_count + non_trainable_count)) print('Trainable params: {:,}'.format(trainable_count)) print('Non-trainable params: {:,}'.format(non_trainable_count)) # print(network.summary()) # rmsp = optimizers.RMSprop(lr=0.0001, decay=0.1) rmsp = optimizers.RMSprop(lr=lr) network.compile( optimizer=rmsp, # 'rmsprop', # SGD,adam,rmsprop loss='mae', metrics=list(metrics)) # mae平均绝对误差(mean absolute error) accuracy result = network.fit( x=x_train, y=ddg_train, batch_size=batch_size, epochs=epochs, verbose=verbose, callbacks=my_callbacks, validation_data=(x_val, ddg_val), shuffle=True, ) # print('\n----------History:\n%s'%result.history) # # save # save_train_cv(network, modeldir, result.history, k_count) opt_lst.append(np.mean( result.history['val_mean_absolute_error'][-10:])) opt_loss = np.mean(opt_lst) # # print hyper combination group and current loss value # print('\n@current_hyper_tag: %s' '\n@current optmized_loss: %s' % (hyper_param_tag, opt_loss)) # return {'loss': validation_loss, 'status': STATUS_OK, 'model':model} return {'loss': opt_loss, 'status': STATUS_OK}
X_train = X_train_val[tt_index[0]] Y_train = Y_train_val[tt_index[0]] X_val = X_train_val[tt_index[1]] Y_val = Y_train_val[tt_index[1]] #define model input = layers.Input(shape=(183, 183, 6)) x = layers.Conv2D(128, 5, strides=2, kernel_regularizer=regularizers.l2(0.01))( input) #outputs 90x90x128 x = layers.ReLU()(x) x = layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x) x = layers.Conv2D(256, 5, strides=2, kernel_regularizer=regularizers.l2(0.01))( x) #outputs 14x14x128 x = layers.ReLU()(x) x = layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x) x = layers.Flatten()(x) #outputs 512 x = layers.Dropout(0.4)(x) x = layers.Dense(512, activation='relu', kernel_regularizer=regularizers.l2(0.01))(x) x = layers.Dense(6, activation='softmax')(x) clf = keras.models.Model(input, x) clf_optimzer = keras.optimizers.Adam(lr=0.001) clf.compile(optimizer=clf_optimzer, loss='categorical_crossentropy', weighted_metrics=['categorical_accuracy']) #train model________________________________________________________________________________________________________________________
def __init__(self, shape): """ 将dropout输出的结果直接影像最终结果 :param shape: """ self.re_rate = 0.6 self.inputs = layers.Input(shape=shape) self.f_block = layers.Conv3D(4, (3, 3, 3), activation='relu', kernel_regularizer=regularizers.l2( self.re_rate), padding='same')(self.inputs) self.bn = layers.BatchNormalization()(self.f_block) self.mp1 = layers.MaxPooling3D((2, 2, 2))(self.bn) self.f_block1 = layers.Conv3D(8, (3, 3, 3), activation='relu', kernel_regularizer=regularizers.l2( self.re_rate), padding='same')(self.mp1) self.bn = layers.BatchNormalization()(self.f_block1) self.mp2 = layers.MaxPooling3D((2, 2, 2))(self.bn) self.f_block2 = layers.Conv3D(16, (3, 3, 3), activation='relu', kernel_regularizer=regularizers.l2( self.re_rate), padding='same')(self.mp2) self.f_block2 = layers.BatchNormalization()(self.f_block2) self.b_back2 = layers.Conv3D(32, (3, 3, 3), activation='relu', kernel_regularizer=regularizers.l2( self.re_rate), padding='same')(self.f_block2) self.b_back2 = layers.BatchNormalization()(self.b_back2) self.b_back2 = layers.Conv3D( 64, (3, 3, 3), activation='relu', kernel_regularizer=regularizers.l2(self.re_rate), padding='same')(layers.UpSampling3D((2, 2, 2))(self.f_block2)) self.b_back2 = layers.BatchNormalization()(self.b_back2) self.cat2 = layers.concatenate([self.f_block1, self.b_back2]) self.bn = layers.BatchNormalization()(self.cat2) self.b_back1 = layers.Conv3D( 32, (3, 3, 3), activation='relu', kernel_regularizer=regularizers.l2(self.re_rate), padding='same')(layers.UpSampling3D((2, 2, 2))(self.bn)) self.b_back1 = layers.BatchNormalization()(self.b_back1) self.gb = layers.GlobalAveragePooling3D()(self.b_back1) self.gb_drop = layers.Dropout(rate=0.9)(self.gb) self.pure_dense = layers.Dense(1, activation='sigmoid')(self.gb_drop) # add mmse mmse_input = layers.Input(shape=(1, ), dtype='int32') embedded_layer = layers.Embedding(shape[-1], 1)(mmse_input) embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer) embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer) emCon = layers.Flatten()(embedded_layer) self.mmse = emCon # add sex sex_input = layers.Input(shape=(1, ), dtype='int32') embedded_layer = layers.Embedding(shape[-1], 1)(sex_input) embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer) embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer) emCon = layers.Flatten()(embedded_layer) self.sex = emCon # add age age_input = layers.Input(shape=(1, ), dtype='int32') embedded_layer = layers.Embedding(shape[-1], 1)(age_input) embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer) embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer) emCon = layers.Flatten()(embedded_layer) self.age = emCon # add marriage marriage_input = layers.Input(shape=(1, ), dtype='int32') embedded_layer = layers.Embedding(shape[-1], 1)(marriage_input) embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer) embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer) emCon = layers.Flatten()(embedded_layer) self.marriage = emCon # add apoe4 apoe4_input = layers.Input(shape=(1, ), dtype='int32') embedded_layer = layers.Embedding(shape[-1], 1)(apoe4_input) embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer) embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer) emCon = layers.Flatten()(embedded_layer) self.apoe4 = emCon # add education edu_input = layers.Input(shape=(1, ), dtype='int32') embedded_layer = layers.Embedding(shape[-1], 1)(edu_input) embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer) embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer) emCon = layers.Flatten()(embedded_layer) self.edu = emCon self.cat = layers.concatenate([ self.mmse, self.sex, self.age, self.marriage, self.apoe4, self.edu ]) self.cli_dense = layers.Dense(6, activation='relu')(self.cat) self.cli_judge = layers.Dense(1, activation='sigmoid')(self.cli_dense) self.cat = layers.concatenate([self.gb_drop, self.cli_dense]) self.dense = layers.Dense(1, activation='sigmoid')(self.cat) self.model = keras.Model( input=[ self.inputs, mmse_input, sex_input, age_input, marriage_input, apoe4_input, edu_input ], output=[self.pure_dense, self.cli_judge, self.dense])
def getModel(self, modelNo, learningRate=0.001): model_input = layers.Input(shape=(28, 28, 1)) if modelNo == 1: #model 1 model = models.Sequential() model.add( layers.Conv2D(16, (3, 3), activation=activations.relu, input_shape=(28, 28, 1))) model.add(layers.MaxPooling2D((2, 2))) model.add(layers.Conv2D(32, (3, 3), activation=activations.relu)) model.add(layers.MaxPooling2D((2, 2))) model.add(layers.Conv2D(32, (3, 3), activation=activations.relu)) model.add(layers.Flatten()) model.add(layers.Dense(32, activation=activations.relu)) model.add( layers.Dense(self.outputShape, activation=activations.softmax)) model.summary() model.compile(optimizer=optimizers.rmsprop(lr=learningRate), loss=losses.categorical_crossentropy, metrics=[metrics.categorical_accuracy]) model.name = "basic CNN, 32" return model if modelNo == 2: #model 1 model = models.Sequential() model.add( layers.Conv2D(32, (3, 3), activation=activations.relu, input_shape=(28, 28, 1))) model.add(layers.MaxPooling2D((2, 2))) model.add(layers.Conv2D(64, (3, 3), activation=activations.relu)) model.add(layers.MaxPooling2D((2, 2))) model.add(layers.Conv2D(64, (3, 3), activation=activations.relu)) model.add(layers.Flatten()) model.add(layers.Dense(64, activation=activations.relu)) model.add( layers.Dense(self.outputShape, activation=activations.softmax)) model.summary() model.compile(optimizer=optimizers.rmsprop(lr=learningRate), loss=losses.categorical_crossentropy, metrics=[metrics.categorical_accuracy]) model.name = "basic CNN, 64" return model if modelNo == 3: return self.getConvPoolCNNCModel(model_input, learningRate) if modelNo == 4: return self.getAllCNNC(model_input, learningRate) if modelNo == 5: return self.NINCNN(model_input, learningRate) if modelNo == 6: #model 1 model = models.Sequential() model.add( layers.Conv2D(32, (3, 3), activation=activations.relu, input_shape=(28, 28, 1))) model.add(layers.MaxPooling2D((2, 2))) model.add(layers.Dropout(0.2)) model.add(layers.Conv2D(64, (3, 3), activation=activations.relu)) model.add(layers.MaxPooling2D((2, 2))) model.add(layers.Conv2D(64, (3, 3), activation=activations.relu)) model.add(layers.Flatten()) model.add(layers.Dense(64, activation=activations.relu)) model.add( layers.Dense(self.outputShape, activation=activations.softmax)) model.summary() model.compile(optimizer=optimizers.rmsprop(lr=learningRate), loss=losses.categorical_crossentropy, metrics=[metrics.categorical_accuracy]) model.name = "basic CNN, 64, Dropout 0.2" return model if modelNo == 7: #model 1 model = models.Sequential() model.add( layers.Conv2D(32, (3, 3), kernel_regularizer=regularizers.l2(0.01), activation=activations.relu, input_shape=(28, 28, 1))) model.add(layers.MaxPooling2D((2, 2))) model.add( layers.Conv2D(64, (3, 3), kernel_regularizer=regularizers.l2(0.01), activation=activations.relu)) model.add(layers.MaxPooling2D((2, 2))) model.add( layers.Conv2D(64, (3, 3), kernel_regularizer=regularizers.l2(0.01), activation=activations.relu)) model.add(layers.Flatten()) model.add(layers.Dense(64, activation=activations.relu)) model.add( layers.Dense(self.outputShape, activation=activations.softmax)) model.summary() model.compile(optimizer=optimizers.rmsprop(lr=learningRate), loss=losses.categorical_crossentropy, metrics=[metrics.categorical_accuracy]) model.name = "basic CNN, 64, L2 .01" return model if modelNo == 8: #model 1 model = models.Sequential() model.add( layers.Conv2D(32, (3, 3), kernel_regularizer=regularizers.l2(0.01), activation=activations.relu, input_shape=(28, 28, 1))) model.add(layers.MaxPooling2D((2, 2))) model.add(layers.Dropout(0.2)) model.add( layers.Conv2D(64, (3, 3), kernel_regularizer=regularizers.l2(0.01), activation=activations.relu)) model.add(layers.MaxPooling2D((2, 2))) model.add(layers.Dropout(0.2)) model.add( layers.Conv2D(64, (3, 3), kernel_regularizer=regularizers.l2(0.01), activation=activations.relu)) model.add(layers.Flatten()) model.add(layers.Dense(64, activation=activations.relu)) model.add( layers.Dense(self.outputShape, activation=activations.softmax)) model.summary() model.compile(optimizer=optimizers.rmsprop(lr=learningRate), loss=losses.categorical_crossentropy, metrics=[metrics.categorical_accuracy]) model.name = "basic CNN, 64, Dropout 0.2, L2 .01" return model if modelNo == 9: return self.wideNet(model_input, learningRate) if modelNo == 10: model = ResnetBuilder.build_resnet_18((1, 28, 28), self.outputShape) model.summary() model.compile(optimizer=optimizers.rmsprop(lr=learningRate), loss=losses.categorical_crossentropy, metrics=[metrics.categorical_accuracy]) model.name = "build_resnet_18" return model if modelNo == 11: model = ResnetBuilder.build_resnet_50((1, 28, 28), self.outputShape) model.summary() model.compile(optimizer=optimizers.rmsprop(lr=learningRate), loss=losses.categorical_crossentropy, metrics=[metrics.categorical_accuracy]) model.name = "build_resnet_50" return model
mnist_conv_base = models.load_model('mnist-base.h5') """ Base mode for transfer learning is configured below. 1. Layers are shaved off. 2. Layers are set to be none trainable """ print('TOTAL LAYERS: ', len(mnist_conv_base.layers)) mnist_conv_base.trainable = False mnist_conv_base.summary() n = 6 for i in range(n): mnist_conv_base.layers.pop() mnist_conv_base.add(layers.Flatten()) mnist_conv_base.summary() train_data_gen = ImageDataGenerator(rescale=1. / 255, rotation_range=180) test_data_gen = ImageDataGenerator(rescale=1. / 255) train_generator = train_data_gen.flow_from_directory(train_dir, target_size=(28, 28), batch_size=20, color_mode="grayscale") test_generator = test_data_gen.flow_from_directory(test_dir, target_size=(28, 28), batch_size=20, color_mode="grayscale")
def independent_avg_modular_deeptrack_new_layer( layer_size, # Total number of nodes in layer train_generator, conv_list, output_list, input_tensor, layer_no=2, nbr_nodes_added=1, sample_sizes=(8, 32, 128, 512, 1024), iteration_numbers=(401, 301, 201, 101, 51), verbose=0.01, mp_training = False, # use multiprocessing training? translation_distance=5, # parameters for multiprocessing training SN_limits=[10,100], # parameters for multiprocessing training save_networks=False, model_path="", layer_type='conv'): """ Adds a new layer to the modular averaging architecture where each output is trained independently """ import deeptrack from keras import models,layers from keras.models import Model from feature_by_feature import freeze_all_layers new_layer_node_list = [] # Convolutions in the new layer new_layer_flattened_list = [] # flattened output from new layer # Create input tenosr for new node if len(conv_list)>1: new_layer_input = layers.Concatenate()(conv_list) else: new_layer_input = conv_list[0] # If next layer is dense then we (probably) need to flatten previous input if layer_type=='dense': import keras.backend as K # Check dimension of previous output to see if Flatten layer is neede. prev_out_size = K.shape(new_layer_input).shape if(prev_out_size[0]>2): new_layer_input = layers.Flatten()(new_layer_input) # Add all the new nodes and train network in between for i in range(round(layer_size/nbr_nodes_added)): if layer_type=='dense': next_node = layers.Dense(nbr_nodes_added,activation='relu')(new_layer_input) next_flattened = next_node else: next_node = layers.Conv2D(nbr_nodes_added,(3,3),activation='relu')(new_layer_input) next_node = layers.MaxPooling2D((2,2))(next_node) next_flattened = layers.Flatten()(next_node) new_layer_flattened_list.append(next_flattened) if(i==0): # i = 0 special case. No concatenation needed next_output = layers.Dense(3)(next_flattened) # Different for i==0 else: next_output = layers.Concatenate(axis=-1)(new_layer_flattened_list) next_output = layers.Dense(3)(next_output) # Construct and compile network network = models.Model(input_tensor,next_output) network.compile(optimizer='rmsprop', loss='mse', metrics=['mse', 'mae']) network.summary() output_list.append(next_output) # Train and freeze layers in network if mp_training: deeptrack.train_deep_learning_network_mp( network, sample_sizes = sample_sizes, iteration_numbers = iteration_numbers, verbose=verbose, SN_limits=SN_limits, translation_distance=translation_distance, ) else: deeptrack.train_deep_learning_network( network, train_generator, sample_sizes = sample_sizes, iteration_numbers = iteration_numbers, verbose=verbose) freeze_all_layers(network) new_layer_node_list.append(next_node) if(save_networks): network.save(model_path+"L"+str(layer_no)+"_"+str((i+1)*nbr_nodes_added)+"F.h5") avg_out = layers.average(output_list) network = models.Model(input_tensor,avg_out) network.compile(optimizer='rmsprop', loss='mse', metrics=['mse', 'mae']) print('final network architecture') network.summary() if(save_networks): network.save(model_path+"final_L"+str(layer_no)+"_F.h5") # IF dense statement needed return network,new_layer_node_list,output_list,new_layer_flattened_list
def independent_avg_modular_deeptrack_L1(# bad name layer_size, train_generator, input_shape=(51,51,1), output_shape=3, nbr_nodes_added=1, sample_sizes=(8, 32, 128, 512, 1024), iteration_numbers=(401, 301, 201, 101, 51), verbose=0.01, save_networks=False, mp_training=False, # use multiprocessing training translation_distance=5, # parameters for multiprocessing training SN_limits=[10,100], # parameters for multiprocessing training model_path="", # determines the type of layer used for combining the # predictions. Addition and average only options at the moment ): """ First layer in the modular averaging architecture where each layer is trained independetly of previous ones. Inputs: layer_size - size of first layer in model train_generator - generator for images output_shape - number of outputs from network, 3 for normal deeptrack nbr_nodes_added - number of nodes to add at a time sample_size - same as deeptrack iteration_numbers - same as deeptrack save_networks - if networks are to be saved automatically once training is finished mp_training - use multiprocessing to speed up training. Not available for custom image generators as supplied by train_generator, uses FBFs own image generator. translation_distance - parameter for mp_training image generator, determines the area the particle is allowed to appear in SN_limits - Outputs: TODO - Implement weight decay in later layers and lowering learning rate """ import deeptrack import keras from keras import Input,models,layers from keras.models import Model from feature_by_feature import freeze_all_layers #import deeptrackelli_mod_mproc as multiprocess_training # Needed for fast parallelized image generator input_tensor = Input(input_shape) conv_list = [] # List of convolutional neruons in L1, needed for subsequent layers flattened_list = [] # List of flattened layers output_list = [] # List of the output layers # Loop thorugh the neoruns and add them one by one for i in range(round(layer_size/nbr_nodes_added)): next_node = layers.Conv2D(nbr_nodes_added,(3,3),activation='relu')(input_tensor) next_node = layers.MaxPooling2D((2,2))(next_node) if(i==0): # i = 0 special case. No addition needed next_flattened = layers.Flatten()(next_node) next_output = layers.Dense(3)(next_flattened) final_output = next_output output_list.append(next_output) flattened_list.append(next_flattened) else: # Construct the next output node next_flattened = layers.Flatten()(next_node) flattened_list.append(next_flattened) # Can't concatenate a single layer if(len(flattened_list)>1): next_output = layers.Concatenate(axis=-1)(flattened_list) next_output = layers.Dense(3)(next_output) output_list.append(next_output) # Construct and compile network network = models.Model(input_tensor,next_output) network.compile(optimizer='rmsprop', loss='mse', metrics=['mse', 'mae']) network.summary() # Train and freeze layers in network if mp_training: deeptrack.train_deep_learning_network_mp( network, sample_sizes = sample_sizes, iteration_numbers = iteration_numbers, verbose=verbose, SN_limits=SN_limits, translation_distance=translation_distance, ) else: deeptrack.train_deep_learning_network( network, train_generator, sample_sizes = sample_sizes, iteration_numbers = iteration_numbers, verbose=verbose) freeze_all_layers(network) conv_list.append(next_node) if(save_networks): network.save(model_path+"L1_"+str((i+1)*nbr_nodes_added)+"F.h5") # Create final output using all the output layers and averaging them if(len(output_list)>1): avg_out = layers.average(output_list) else: avg_out = output_list[0] network = models.Model(input_tensor,avg_out) network.compile(optimizer='rmsprop', loss='mse', metrics=['mse', 'mae']) print('final network architecture') network.summary() if(save_networks): network.save(model_path+"final_L"+str(1)+"_F.h5") return network,conv_list,output_list,flattened_list,input_tensor
def multi_net(img_rows, img_cols, color_type, num_classes=None): small = L.Input(shape=(img_rows, img_cols, color_type), name='small') x = conv_block(small, 5) x = L.Conv2D(36, (5, 5), activation='relu')(small) x = L.Conv2D(48, (3, 3), activation='relu')(x) x = L.Flatten()(x) x1 = L.Dense(256)(x) x = L.Dense(512, activation='relu')(x) x = L.Dropout(0.4)(x) x = L.Dense(512, activation='relu')(x) x = L.Dropout(0.4)(x) logits1 = L.Dense(num_classes, activation='softmax')(x) medium = L.Input(shape=(img_rows * 2, img_cols * 2, color_type), name='medium') x = conv_block(medium, 5) x = L.Conv2D(16, (3, 3), activation='relu')(x) x = L.Conv2D(16, (3, 3), activation='relu')(x) x = conv_block(x, 3) x = L.Conv2D(16, (3, 3), activation='relu')(x) x = L.Flatten()(x) x2 = L.Dense(256)(x) x = L.Dense(512, activation='relu')(x) x = L.Dropout(0.4)(x) x = L.Dense(512, activation='relu')(x) x = L.Dropout(0.4)(x) logits2 = L.Dense(num_classes, activation='softmax')(x) large = L.Input(shape=(img_rows * 3, img_cols * 3, color_type), name='large') x = conv_block(large, 5) x = L.Conv2D(16, (3, 3), activation='relu')(x) x = L.Conv2D(16, (3, 3), activation='relu')(x) x = conv_block(x, 3) x = L.Conv2D(16, (3, 3), activation='relu')(x) x = L.Conv2D(16, (3, 3), activation='relu')(x) x = L.Conv2D(16, (3, 3), activation='relu')(x) x = L.Conv2D(16, (3, 3), activation='relu')(x) x = L.Conv2D(16, (3, 3), activation='relu')(x) x = L.Flatten()(x) x3 = L.Dense(256)(x) x = L.Dense(512, activation='relu')(x) x = L.Dropout(0.4)(x) x = L.Dense(512, activation='relu')(x) x = L.Dropout(0.4)(x) logits3 = L.Dense(num_classes, activation='softmax')(x) # # combine all patch # merge0=L.concatenate([x1, x2, x3],axis=-1) # merge1=L.Dense(1024)(merge0) # merge2=L.Activation('relu')(merge1) # merge2 = L.Dropout(0.4)(merge2) # merge3=L.Dense(512)(merge2) # merge3=L.Activation('relu')(merge3) # merge3 = L.Dropout(0.4)(merge3) # logits=L.Dense(num_classes,activation='softmax')(merge0) logits = L.average([logits1, logits2, logits3]) new_model = K.models.Model([small, medium, large], logits) sgd = K.optimizers.SGD(lr=0.001, momentum=0.99, decay=1e-4) new_model.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['acc']) return new_model
def VGG16(): # Determine proper input shape input_shape = (128, 128, 200) img_input = layers.Input(shape=input_shape) # Block 1 x = layers.Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv1')(img_input) x = layers.Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv2')(x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x) # Block 2 x = layers.Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv1')(x) x = layers.Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv2')(x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x) # Block 3 x = layers.Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv1')(x) x = layers.Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv2')(x) x = layers.Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv3')(x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x) # Block 4 x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv1')(x) x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv2')(x) x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv3')(x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x) # Block 5 x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv1')(x) x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv2')(x) x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv3')(x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x) # Classification block x = layers.Flatten(name='flatten')(x) x = layers.Dropout(0.3)(x) x = layers.Dense(4096, activation='relu', name='fc1')(x) x = layers.Dropout(0.3)(x) x = layers.Dense(4096, activation='relu', name='fc2')(x) #x = layers.Dense(2, name='predictions')(x) inputs = img_input # Create model. #model = models.Model(inputs, x, name='vgg16') y1 = layers.Dense(11, activation='softmax')(x) y2 = layers.Dense(11, activation='softmax')(x) model = Model(inputs=inputs, outputs=[y1, y2]) model.load_weights("weight/model_2_v1_tf.hdf5") adam = Adam(lr=0.0001) model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy']) return model
import keras.layers as layers classifier = keras.Sequential() classifier.add( layers.Conv2D(filters=6, kernel_size=(3, 3), activation='relu', input_shape=(64, 64, 3))) classifier.add(layers.AveragePooling2D()) classifier.add(layers.Conv2D(filters=16, kernel_size=(3, 3), activation='relu')) classifier.add(layers.AveragePooling2D()) classifier.add(layers.Flatten()) classifier.add(layers.Dense(units=120, activation='relu')) classifier.add(layers.Dense(units=84, activation='relu')) classifier.add(layers.Dense(units=1, activation='sigmoid')) classifier.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy']) tf.keras.utils.plot_model(classifier, to_file='model.png', show_shapes=False, show_layer_names=True,
textNet.summary() # In[301]: SVG( model_to_dot(textNet, show_layer_names=True, show_shapes=True).create(prog='dot', format='svg')) # # Define Complete Joint Network # In[302]: image_input = Input(shape=(256, 256, 3), name='image_input') image_features = imgNet(image_input) image_features = kl.Dropout(0.1)(image_features) image_features = kl.Flatten()(image_features) image_features = kl.Dense(512, activation='relu')(image_features) words_input = kl.Input(shape=(max_wlen, ), name='words_input') text_features = textNet(words_input) text_features = kl.Dropout(0.1)(text_features) text_features = kl.Dense(512, activation='relu')(text_features) g = kl.Concatenate()([image_features, text_features]) g = kl.Dense(512, activation='relu')(g) target = Dense(vocab_size + 1, activation='softmax', name='target_word')(g) model = Model([image_input, words_input], target) # In[303]: print('New model {0}. loading word2vec embeddings'.format(model_name))
def LF_experiment(train_x, train_y, test_x, test_y, ntrees, shift, slot, model, num_points_per_task, acorn=None): df = pd.DataFrame() single_task_accuracies = np.zeros(10, dtype=float) shifts = [] tasks = [] base_tasks = [] accuracies_across_tasks = [] train_times_across_tasks = [] single_task_inference_times_across_tasks = [] multitask_inference_times_across_tasks = [] model_size = [] if model == "dnn": default_transformer_class = NeuralClassificationTransformer network = keras.Sequential() network.add( layers.Conv2D(filters=16, kernel_size=(3, 3), activation='relu', input_shape=np.shape(train_x)[1:])) network.add(layers.BatchNormalization()) network.add( layers.Conv2D(filters=32, kernel_size=(3, 3), strides=2, padding="same", activation='relu')) network.add(layers.BatchNormalization()) network.add( layers.Conv2D(filters=64, kernel_size=(3, 3), strides=2, padding="same", activation='relu')) network.add(layers.BatchNormalization()) network.add( layers.Conv2D(filters=128, kernel_size=(3, 3), strides=2, padding="same", activation='relu')) network.add(layers.BatchNormalization()) network.add( layers.Conv2D(filters=254, kernel_size=(3, 3), strides=2, padding="same", activation='relu')) network.add(layers.Flatten()) network.add(layers.BatchNormalization()) network.add(layers.Dense(2000, activation='relu')) network.add(layers.BatchNormalization()) network.add(layers.Dense(2000, activation='relu')) network.add(layers.BatchNormalization()) network.add(layers.Dense(units=10, activation='softmax')) default_transformer_kwargs = { "network": network, "euclidean_layer_idx": -2, "num_classes": 10, "optimizer": keras.optimizers.Adam(3e-4) } default_voter_class = KNNClassificationVoter default_voter_kwargs = {"k": int(np.log2(num_points_per_task))} default_decider_class = SimpleArgmaxAverage elif model == "uf": default_transformer_class = TreeClassificationTransformer default_transformer_kwargs = { "kwargs": { "max_depth": None, "max_features": "auto" } } default_voter_class = TreeClassificationVoter default_voter_kwargs = {} default_decider_class = SimpleArgmaxAverage progressive_learner = ProgressiveLearner( default_transformer_class=default_transformer_class, default_transformer_kwargs=default_transformer_kwargs, default_voter_class=default_voter_class, default_voter_kwargs=default_voter_kwargs, default_decider_class=default_decider_class) for task_ii in range(10): print("Starting Task {} For Fold {}".format(task_ii, shift)) if acorn is not None: np.random.seed(acorn) reduced_sample_no = int(num_points_per_task * (1.29**task_ii)) print(reduced_sample_no) train_start_time = time.time() progressive_learner.add_task( X=train_x[task_ii * 5000 + slot * reduced_sample_no:task_ii * 5000 + (slot + 1) * reduced_sample_no], y=train_y[task_ii * 5000 + slot * reduced_sample_no:task_ii * 5000 + (slot + 1) * reduced_sample_no], num_transformers=1 if model == "dnn" else ntrees, transformer_voter_decider_split=[0.67, 0.33, 0], decider_kwargs={ "classes": np.unique(train_y[task_ii * 5000 + slot * reduced_sample_no:task_ii * 5000 + (slot + 1) * reduced_sample_no]) }) train_end_time = time.time() train_times_across_tasks.append(train_end_time - train_start_time) model_size.append(getsize(progressive_learner)) print(model_size) single_task_inference_start_time = time.time() llf_task = progressive_learner.predict( X=test_x[task_ii * 1000:(task_ii + 1) * 1000, :], transformer_ids=[task_ii], task_id=task_ii) single_task_inference_end_time = time.time() single_task_accuracies[task_ii] = np.mean( llf_task == test_y[task_ii * 1000:(task_ii + 1) * 1000]) single_task_inference_times_across_tasks.append( single_task_inference_end_time - single_task_inference_start_time) for task_jj in range(task_ii + 1): multitask_inference_start_time = time.time() llf_task = progressive_learner.predict( X=test_x[task_jj * 1000:(task_jj + 1) * 1000, :], task_id=task_jj) multitask_inference_end_time = time.time() shifts.append(shift) tasks.append(task_jj + 1) base_tasks.append(task_ii + 1) accuracies_across_tasks.append( np.mean(llf_task == test_y[task_jj * 1000:(task_jj + 1) * 1000])) multitask_inference_times_across_tasks.append( multitask_inference_end_time - multitask_inference_start_time) df['data_fold'] = shifts df['task'] = tasks df['base_task'] = base_tasks df['accuracy'] = accuracies_across_tasks df['multitask_inference_times'] = multitask_inference_times_across_tasks df_single_task = pd.DataFrame() df_single_task['task'] = range(1, 11) df_single_task['data_fold'] = shift df_single_task['accuracy'] = single_task_accuracies df_single_task[ 'single_task_inference_times'] = single_task_inference_times_across_tasks df_single_task['train_times'] = train_times_across_tasks df_single_task['model_size'] = model_size summary = (df, df_single_task) file_to_save = 'result/result/increased_sample_' + model + str( ntrees) + '_' + str(shift) + '_' + str(slot) + '.pickle' with open(file_to_save, 'wb') as f: pickle.dump(summary, f)
lenc = MultiLabelBinarizer() Y = lenc.fit_transform(Y) feature_maps_shape = (16, 224, 224, 3) feature_maps_tf = tf.placeholder(tf.float32, shape=feature_maps_shape) #feature_maps_np = np.ones(feature_maps_tf.shape, dtype='float32') #feature_maps_np[0, img_height-1, img_width-3, 0] = 50 roiss_tf = tf.placeholder(tf.float32, shape=(16, 2, 4)) #roiss_np = np.asarray([[[0.5,0.2,0.7,0.4], [0.0,0.0,1.0,1.0]]], dtype='float32') roi_layer = ROI_pooling.ROIPoolingLayer(7, 7) pooled_features = roi_layer([feature_maps_tf, roiss_tf]) additional_model = models.Sequential() additional_model.add(pre_trained_vgg) additional_model.add(layers.Flatten()) #additional_model.add(pooled_features) additional_model.add(layers.Dense(4096, activation='relu', name='fc1')) additional_model.add(layers.Dense(4096, activation='relu', name='fc2')) additional_model.add(layers.Dense(20, activation='softmax', name='classifier')) additional_model.summary() additional_model.compile(optimizer='adam', loss=keras.losses.categorical_crossentropy, metrics=["accuracy"]) # 일단은 돌아가게는 만들었으나 정확도는 거의 없다시피하다 Y에는 이름만 들어가있는 상태이며, 현재는 돌아가긴 하는 상황이니 ROI_Pooling 계층에 대해서 알아봐야겠다 hist = additional_model.fit_generator(X, steps_per_epoch=10, epochs=5,
def single_ae( enc_size, input_shape, name='single_ae', prefix=None, ae_type='dense', # 'dense', or 'conv' conv_size=None, input_model=None, enc_lambda_layers=None, batch_norm=True, padding='same', activation=None, include_mu_shift_layer=False, do_vae=False): """single-layer Autoencoder (i.e. input - encoding - output""" # naming model_name = name if prefix is None: prefix = model_name if enc_lambda_layers is None: enc_lambda_layers = [] # prepare input input_name = '%s_input' % prefix if input_model is None: assert input_shape is not None, 'input_shape of input_model is necessary' input_tensor = KL.Input(shape=input_shape, name=input_name) last_tensor = input_tensor else: input_tensor = input_model.input last_tensor = input_model.output input_shape = last_tensor.shape.as_list()[1:] input_nb_feats = last_tensor.shape.as_list()[-1] # prepare conv type based on input if ae_type == 'conv': ndims = len(input_shape) - 1 convL = getattr(KL, 'Conv%dD' % ndims) assert conv_size is not None, 'with conv ae, need conv_size' conv_kwargs = {'padding': padding, 'activation': activation} # if want to go through a dense layer in the middle of the U, need to: # - flatten last layer if not flat # - do dense encoding and decoding # - unflatten (rehsape spatially) at end if ae_type == 'dense' and len(input_shape) > 1: name = '%s_ae_%s_down_flat' % (prefix, ae_type) last_tensor = KL.Flatten(name=name)(last_tensor) # recall this layer pre_enc_layer = last_tensor # encoding layer if ae_type == 'dense': assert len( enc_size) == 1, "enc_size should be of length 1 for dense layer" enc_size_str = ''.join(['%d_' % d for d in enc_size])[:-1] name = '%s_ae_mu_enc_dense_%s' % (prefix, enc_size_str) last_tensor = KL.Dense(enc_size[0], name=name)(pre_enc_layer) else: # convolution # convolve then resize. enc_size should be [nb_dim1, nb_dim2, ..., nb_feats] assert len(enc_size) == len(input_shape), \ "encoding size does not match input shape %d %d" % (len(enc_size), len(input_shape)) if list(enc_size)[:-1] != list(input_shape)[:-1] and \ all([f is not None for f in input_shape[:-1]]) and \ all([f is not None for f in enc_size[:-1]]): # assert len(enc_size) - 1 == 2, "Sorry, I have not yet implemented non-2D resizing -- need to check out interpn!" name = '%s_ae_mu_enc_conv' % (prefix) last_tensor = convL(enc_size[-1], conv_size, name=name, **conv_kwargs)(pre_enc_layer) name = '%s_ae_mu_enc' % (prefix) zf = [ enc_size[:-1][f] / last_tensor.shape.as_list()[1:-1][f] for f in range(len(enc_size) - 1) ] last_tensor = layers.Resize(zoom_factor=zf, name=name)(last_tensor) # resize_fn = lambda x: tf.image.resize_bilinear(x, enc_size[:-1]) # last_tensor = KL.Lambda(resize_fn, name=name)(last_tensor) elif enc_size[ -1] is None: # convolutional, but won't tell us bottleneck name = '%s_ae_mu_enc' % (prefix) last_tensor = KL.Lambda(lambda x: x, name=name)(pre_enc_layer) else: name = '%s_ae_mu_enc' % (prefix) last_tensor = convL(enc_size[-1], conv_size, name=name, **conv_kwargs)(pre_enc_layer) if include_mu_shift_layer: # shift name = '%s_ae_mu_shift' % (prefix) last_tensor = layers.LocalBias(name=name)(last_tensor) # encoding clean-up layers for layer_fcn in enc_lambda_layers: lambda_name = layer_fcn.__name__ name = '%s_ae_mu_%s' % (prefix, lambda_name) last_tensor = KL.Lambda(layer_fcn, name=name)(last_tensor) if batch_norm is not None: name = '%s_ae_mu_bn' % (prefix) last_tensor = KL.BatchNormalization(axis=batch_norm, name=name)(last_tensor) # have a simple layer that does nothing to have a clear name before sampling name = '%s_ae_mu' % (prefix) last_tensor = KL.Lambda(lambda x: x, name=name)(last_tensor) # if doing variational AE, will need the sigma layer as well. if do_vae: mu_tensor = last_tensor # encoding layer if ae_type == 'dense': name = '%s_ae_sigma_enc_dense_%s' % (prefix, enc_size_str) last_tensor = KL.Dense( enc_size[0], name=name, # kernel_initializer=keras.initializers.RandomNormal(mean=0.0, stddev=1e-5), # bias_initializer=keras.initializers.RandomNormal(mean=-5.0, stddev=1e-5) )(pre_enc_layer) else: if list(enc_size)[:-1] != list(input_shape)[:-1] and \ all([f is not None for f in input_shape[:-1]]) and \ all([f is not None for f in enc_size[:-1]]): assert len( enc_size ) - 1 == 2, "Sorry, I have not yet implemented non-2D resizing..." name = '%s_ae_sigma_enc_conv' % (prefix) last_tensor = convL(enc_size[-1], conv_size, name=name, **conv_kwargs)(pre_enc_layer) name = '%s_ae_sigma_enc' % (prefix) resize_fn = lambda x: tf.image.resize_bilinear( x, enc_size[:-1]) last_tensor = KL.Lambda(resize_fn, name=name)(last_tensor) elif enc_size[ -1] is None: # convolutional, but won't tell us bottleneck name = '%s_ae_sigma_enc' % (prefix) last_tensor = convL(pre_enc_layer.shape.as_list()[-1], conv_size, name=name, **conv_kwargs)(pre_enc_layer) # cannot use lambda, then mu and sigma will be same layer. # last_tensor = KL.Lambda(lambda x: x, name=name)(pre_enc_layer) else: name = '%s_ae_sigma_enc' % (prefix) last_tensor = convL(enc_size[-1], conv_size, name=name, **conv_kwargs)(pre_enc_layer) # encoding clean-up layers for layer_fcn in enc_lambda_layers: lambda_name = layer_fcn.__name__ name = '%s_ae_sigma_%s' % (prefix, lambda_name) last_tensor = KL.Lambda(layer_fcn, name=name)(last_tensor) if batch_norm is not None: name = '%s_ae_sigma_bn' % (prefix) last_tensor = KL.BatchNormalization(axis=batch_norm, name=name)(last_tensor) # have a simple layer that does nothing to have a clear name before sampling name = '%s_ae_sigma' % (prefix) last_tensor = KL.Lambda(lambda x: x, name=name)(last_tensor) logvar_tensor = last_tensor # VAE sampling sampler = _VAESample().sample_z name = '%s_ae_sample' % (prefix) last_tensor = KL.Lambda(sampler, name=name)([mu_tensor, logvar_tensor]) if include_mu_shift_layer: # shift name = '%s_ae_sample_shift' % (prefix) last_tensor = layers.LocalBias(name=name)(last_tensor) # decoding layer if ae_type == 'dense': name = '%s_ae_%s_dec_flat_%s' % (prefix, ae_type, enc_size_str) last_tensor = KL.Dense(np.prod(input_shape), name=name)(last_tensor) # unflatten if dense method if len(input_shape) > 1: name = '%s_ae_%s_dec' % (prefix, ae_type) last_tensor = KL.Reshape(input_shape, name=name)(last_tensor) else: if list(enc_size)[:-1] != list(input_shape)[:-1] and \ all([f is not None for f in input_shape[:-1]]) and \ all([f is not None for f in enc_size[:-1]]): name = '%s_ae_mu_dec' % (prefix) zf = [ last_tensor.shape.as_list()[1:-1][f] / enc_size[:-1][f] for f in range(len(enc_size) - 1) ] last_tensor = layers.Resize(zoom_factor=zf, name=name)(last_tensor) # resize_fn = lambda x: tf.image.resize_bilinear(x, input_shape[:-1]) # last_tensor = KL.Lambda(resize_fn, name=name)(last_tensor) name = '%s_ae_%s_dec' % (prefix, ae_type) last_tensor = convL(input_nb_feats, conv_size, name=name, **conv_kwargs)(last_tensor) if batch_norm is not None: name = '%s_bn_ae_%s_dec' % (prefix, ae_type) last_tensor = KL.BatchNormalization(axis=batch_norm, name=name)(last_tensor) # create the model and retun model = Model(inputs=input_tensor, outputs=[last_tensor], name=model_name) return model
evaluate_naive_method() def visualize(history): loss = history.history['loss'] val_loss = history.history['val_loss'] epochs = range(1, len(loss) + 1) plt.figure() plt.plot(epochs, loss, 'bo') plt.plot(epochs, val_loss, 'b') plt.show() ## simple model ## model = models.Sequential() model.add(layers.Flatten(input_shape=(lookback // step, float_data.shape[-1]))) model.add(layers.Dense(32, activation='relu')) model.add(layers.Dense(1)) model.compile(optimizer=optimizers.adam(), loss='mae') history = model.fit_generator(train_gen, steps_per_epoch=500, epochs=20, validation_data=val_gen, validation_steps=val_steps) ## rnn model ## model = models.Sequential() model.add(layers.GRU(units=32, input_shape=(None, float_data.shape[-1]))) model.add(layers.Dense(1)) model.compile(optimizer='adam', loss='mae') history = model.fit_generator(train_gen,
": "))) fully_connected_size = int(input("Fully connected layer size: ")) epochs_phase_1 = int(input("Phase 1 epochs: ")) epochs_phase_2 = int(input("Phase 2 epochs: ")) batch_size = int(input("Batch size: ")) dropout_rate = float(input("Dropout rate: ")) # Clear previous layer session to match continiously constructed layer names in weights file K.clear_session() # Construct the classifier NN(input -> encoder -> Flatten -> FC -> output with 10 classes(0 - 9)) encoded = encoder(input_img, convolutional_layers, convolutional_filter_size, convolutional_filters_per_layer, dropout_rate) flatten = layers.Flatten()(encoded) fc = layers.Dense(fully_connected_size, activation='relu')(flatten) dropout = layers.Dropout(rate=dropout_rate)(fc) output_layer = layers.Dense(training_labels.num_classes(), activation='softmax')(dropout) classifier = Model(input_img, output_layer) classifier.compile(loss='categorical_crossentropy', optimizer=optimizers.Adam()) # Print it's summary classifier.summary() # Load encoder weights classifier.load_weights(autoencoder_weights_file, by_name=True)
def Conv2DMultiTaskIn1(x_train, y_train, ddg_train, x_test, y_test, ddg_test, x_val, y_val, ddg_val, class_weights_dict, modeldir): K.clear_session() summary = False verbose = 0 # setHyperParams------------------------------------------------------------------------------------------------ batch_size = 64 epochs = 200 lr = 0.0049 optimizer = 'sgd' activator = 'elu' basic_conv2D_layers = 1 basic_conv2D_filter_num = 16 loop_dilation2D_layers = 2 loop_dilation2D_filter_num = 64 loop_dilation2D_dropout_rate = 0.2008 dilation_lower = 2 dilation_upper = 16 reduce_layers = 3 # conv 3 times: 120 => 60 => 30 => 15 # print(reduce_layers) reduce_conv2D_filter_num = 16 reduce_conv2D_dropout_rate = 0.1783 residual_stride = 2 dense1_num = 128 dense2_num = 32 drop_num = 0.2605 kernel_size = (3, 3) pool_size = (2, 2) initializer = 'random_uniform' padding_style = 'same' loss_type = 'mse' metrics = ('mae', pearson_r) my_callbacks = [ callbacks.ReduceLROnPlateau( monitor='val_loss', factor=0.5, patience=5, ) ] if lr > 0: if optimizer == 'adam': chosed_optimizer = optimizers.Adam(lr=lr) elif optimizer == 'sgd': chosed_optimizer = optimizers.SGD(lr=lr) elif optimizer == 'rmsprop': chosed_optimizer = optimizers.RMSprop(lr=lr) # build -------------------------------------------------------------------------------------------------------- ## basic Conv2D input_layer = Input(shape=x_train.shape[1:]) y = layers.Conv2D(basic_conv2D_filter_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(input_layer) y = layers.BatchNormalization(axis=-1)(y) if basic_conv2D_layers == 2: y = layers.Conv2D(basic_conv2D_filter_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(y) y = layers.BatchNormalization(axis=-1)(y) ## loop with Conv2D with dilation (padding='same') for _ in range(loop_dilation2D_layers): y = layers.Conv2D(loop_dilation2D_filter_num, kernel_size, padding=padding_style, dilation_rate=dilation_lower, kernel_initializer=initializer, activation=activator)(y) y = layers.BatchNormalization(axis=-1)(y) y = layers.Dropout(loop_dilation2D_dropout_rate)(y) dilation_lower *= 2 if dilation_lower > dilation_upper: dilation_lower = 2 ## Conv2D with dilation (padding='valaid') and residual block to reduce dimention. for _ in range(reduce_layers): y = layers.Conv2D(reduce_conv2D_filter_num, kernel_size, padding=padding_style, kernel_initializer=initializer, activation=activator)(y) y = layers.BatchNormalization(axis=-1)(y) y = layers.Dropout(reduce_conv2D_dropout_rate)(y) y = layers.MaxPooling2D(pool_size, padding=padding_style)(y) residual = layers.Conv2D(reduce_conv2D_filter_num, 1, strides=residual_stride, padding='same')(input_layer) y = layers.add([y, residual]) residual_stride *= 2 ## flat & dense y = layers.Flatten()(y) y = layers.Dense(dense1_num, activation=activator)(y) y = layers.BatchNormalization(axis=-1)(y) y = layers.Dropout(drop_num)(y) y = layers.Dense(dense2_num, activation=activator)(y) y = layers.BatchNormalization(axis=-1)(y) y = layers.Dropout(drop_num)(y) output_layer = layers.Dense(1)(y) model = models.Model(inputs=input_layer, outputs=output_layer) if summary: model.summary() model.compile( optimizer=chosed_optimizer, loss=loss_type, metrics=list(metrics) # accuracy ) K.set_session(tf.Session(graph=model.output.graph)) init = K.tf.global_variables_initializer() K.get_session().run(init) result = model.fit( x=x_train, y=ddg_train, batch_size=batch_size, epochs=epochs, verbose=verbose, callbacks=my_callbacks, validation_data=(x_test, ddg_test), shuffle=True, ) # print('\n----------History:\n%s'%result.history) model_json = model.to_json() with open(modeldir + '/model.json', 'w') as json_file: json_file.write(model_json) return model, result.history
# # Then, we develop a `discriminator` model, that takes as input a candidate image (real or synthetic) and classifies it into one of two # classes, either "generated image" or "real image that comes from the training set". # In[2]: discriminator_input = layers.Input(shape=(height, width, channels)) x = layers.Conv2D(128, 3)(discriminator_input) x = layers.LeakyReLU()(x) x = layers.Conv2D(128, 4, strides=2)(x) x = layers.LeakyReLU()(x) x = layers.Conv2D(128, 4, strides=2)(x) x = layers.LeakyReLU()(x) x = layers.Conv2D(128, 4, strides=2)(x) x = layers.LeakyReLU()(x) x = layers.Flatten()(x) # One dropout layer - important trick! x = layers.Dropout(0.4)(x) # Classification layer x = layers.Dense(1, activation='sigmoid')(x) discriminator = keras.models.Model(discriminator_input, x) discriminator.summary() # To stabilize training, we use learning rate decay # and gradient clipping (by value) in the optimizer. discriminator_optimizer = keras.optimizers.RMSprop(lr=0.0008, clipvalue=1.0, decay=1e-8)
def build_model(**params): # TODO: get all these from **params CNN = 'resnet' INCLUDE_TOP = False LEARNABLE_CNN_LAYERS = params['learnable_cnn_layers'] RNN_TYPE = 'LSTM' RNN_SIZE = 1024 WORDVEC_SIZE = params['wordvec_size'] ACTIVATION = 'relu' USE_CGRU = params['use_cgru'] CGRU_SIZE = params['cgru_size'] REDUCE_MEAN = params['reduce_visual'] max_words = params['max_words'] if CNN == 'vgg16': cnn = applications.vgg16.VGG16(include_top=INCLUDE_TOP) elif CNN == 'resnet': cnn = applications.resnet50.ResNet50(include_top=INCLUDE_TOP) # Pop the mean pooling layer cnn = models.Model(inputs=cnn.inputs, outputs=cnn.layers[-2].output) for layer in cnn.layers[:-LEARNABLE_CNN_LAYERS]: layer.trainable = False # Context Vector input # normalized to [0,1] the values: # left, top, right, bottom, (box area / image area) input_ctx = layers.Input(shape=(5, )) ctx = layers.BatchNormalization()(input_ctx) repeat_ctx = layers.RepeatVector(max_words)(ctx) # Global Image featuers (convnet output for the whole image) input_img_global = layers.Input(shape=(IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS)) image_global = cnn(input_img_global) # Add a residual CGRU layer if USE_CGRU: image_global = layers.Conv2D(CGRU_SIZE, (1, 1), padding='same', activation='relu')(image_global) res_cgru = SpatialCGRU(image_global, CGRU_SIZE) image_global = layers.add([image_global, res_cgru]) if REDUCE_MEAN: image_global = layers.Lambda(lambda x: tf.reduce_mean(x, axis=1))( image_global) image_global = layers.Lambda(lambda x: tf.reduce_mean(x, axis=1))( image_global) else: image_global = layers.Conv2D(WORDVEC_SIZE / 4, (3, 3), activation='relu')(image_global) image_global = layers.Conv2D(WORDVEC_SIZE / 2, (3, 3), activation='relu')(image_global) image_global = layers.Flatten()(image_global) image_global = layers.Concatenate()([image_global, ctx]) image_global = layers.Dense(1024, activation='relu')(image_global) image_global = layers.BatchNormalization()(image_global) image_global = layers.Dense(WORDVEC_SIZE / 2, activation=ACTIVATION)(image_global) image_global = layers.BatchNormalization()(image_global) image_global = layers.RepeatVector(max_words)(image_global) # Local Image featuers (convnet output for just the bounding box) input_img_local = layers.Input(shape=(IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS)) image_local = cnn(input_img_local) if USE_CGRU: image_local = layers.Conv2D(CGRU_SIZE, (1, 1), padding='same', activation='relu')(image_local) res_cgru = SpatialCGRU(image_local, CGRU_SIZE) image_local = layers.add([image_local, res_cgru]) if REDUCE_MEAN: image_local = layers.Lambda(lambda x: tf.reduce_mean(x, axis=1))( image_local) image_local = layers.Lambda(lambda x: tf.reduce_mean(x, axis=1))( image_local) else: image_local = layers.Conv2D(WORDVEC_SIZE / 4, (3, 3), activation='relu')(image_local) image_local = layers.Conv2D(WORDVEC_SIZE / 2, (3, 3), activation='relu')(image_local) image_local = layers.Flatten()(image_local) image_local = layers.Concatenate()([image_local, ctx]) image_local = layers.Dense(1024, activation='relu')(image_local) image_local = layers.BatchNormalization()(image_local) image_local = layers.Dense(WORDVEC_SIZE / 2, activation=ACTIVATION)(image_local) image_local = layers.BatchNormalization()(image_local) image_local = layers.RepeatVector(max_words)(image_local) language_model = models.Sequential() input_words = layers.Input(shape=(max_words, ), dtype='int32') language = layers.Embedding(words.VOCABULARY_SIZE, WORDVEC_SIZE, input_length=max_words)(input_words) x = layers.concatenate([image_global, image_local, repeat_ctx, language]) if RNN_TYPE == 'LSTM': x = layers.LSTM(RNN_SIZE)(x) else: x = layers.GRU(RNN_SIZE)(x) x = layers.BatchNormalization()(x) x = layers.Dense(words.VOCABULARY_SIZE, activation='softmax')(x) return models.Model( inputs=[input_img_global, input_img_local, input_words, input_ctx], outputs=x)
def Conv2DClassifierIn1(x_train,y_train,x_test,y_test): summary = True verbose = 1 # setHyperParams------------------------------------------------------------------------------------------------ batch_size = {{choice([32,64,128,256,512])}} epoch = {{choice([25,50,75,100,125,150,175,200])}} conv_block={{choice(['two', 'three', 'four'])}} conv1_num={{choice([8, 16, 32, 64])}} conv2_num={{choice([16,32,64,128])}} conv3_num={{choice([32,64,128])}} conv4_num={{choice([32, 64, 128, 256])}} dense1_num={{choice([128, 256, 512])}} dense2_num={{choice([64, 128, 256])}} l1_regular_rate = {{uniform(0.00001, 1)}} l2_regular_rate = {{uniform(0.000001, 1)}} drop1_num={{uniform(0.1, 1)}} drop2_num={{uniform(0.0001, 1)}} activator={{choice(['elu','relu','tanh'])}} optimizer={{choice(['adam','rmsprop','SGD'])}} #--------------------------------------------------------------------------------------------------------------- kernel_size = (3, 3) pool_size = (2, 2) initializer = 'random_uniform' padding_style = 'same' loss_type='binary_crossentropy' metrics=['accuracy'] my_callback = None # early_stopping = EarlyStopping(monitor='val_loss', patience=4) # checkpointer = ModelCheckpoint(filepath='keras_weights.hdf5', # verbose=1, # save_best_only=True) # my_callback = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2, # patience=5, min_lr=0.0001) # build -------------------------------------------------------------------------------------------------------- input_layer = Input(shape=x_train.shape[1:]) conv = layers.Conv2D(conv1_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(input_layer) conv = layers.Conv2D(conv1_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(conv) pool = layers.MaxPooling2D(pool_size,padding=padding_style)(conv) if conv_block == 'two': conv = layers.Conv2D(conv2_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(pool) conv = layers.Conv2D(conv2_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(conv) BatchNorm = layers.BatchNormalization(axis=-1)(conv) pool = layers.MaxPooling2D(pool_size,padding=padding_style)(BatchNorm) elif conv_block == 'three': conv = layers.Conv2D(conv2_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(pool) conv = layers.Conv2D(conv2_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(conv) BatchNorm = layers.BatchNormalization(axis=-1)(conv) pool = layers.MaxPooling2D(pool_size,padding=padding_style)(BatchNorm) conv = layers.Conv2D(conv3_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(pool) conv = layers.Conv2D(conv3_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(conv) BatchNorm = layers.BatchNormalization(axis=-1)(conv) pool = layers.MaxPooling2D(pool_size,padding=padding_style)(BatchNorm) elif conv_block == 'four': conv = layers.Conv2D(conv2_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(pool) conv = layers.Conv2D(conv2_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(conv) BatchNorm = layers.BatchNormalization(axis=-1)(conv) pool = layers.MaxPooling2D(pool_size,padding=padding_style)(BatchNorm) conv = layers.Conv2D(conv3_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(pool) conv = layers.Conv2D(conv3_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(conv) BatchNorm = layers.BatchNormalization(axis=-1)(conv) pool = layers.MaxPooling2D(pool_size,padding=padding_style)(BatchNorm) conv = layers.Conv2D(conv4_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(pool) conv = layers.Conv2D(conv4_num,kernel_size,padding=padding_style,kernel_initializer=initializer,activation=activator)(conv) BatchNorm = layers.BatchNormalization(axis=-1)(conv) pool = layers.MaxPooling2D(pool_size,padding=padding_style)(BatchNorm) flat = layers.Flatten()(pool) drop = layers.Dropout(drop1_num)(flat) dense = layers.Dense(dense1_num, activation=activator, kernel_regularizer=regularizers.l1_l2(l1=l1_regular_rate,l2=l2_regular_rate))(drop) BatchNorm = layers.BatchNormalization(axis=-1)(dense) drop = layers.Dropout(drop2_num)(BatchNorm) dense = layers.Dense(dense2_num, activation=activator, kernel_regularizer=regularizers.l1_l2(l1=l1_regular_rate,l2=l2_regular_rate))(drop) output_layer = layers.Dense(len(np.unique(y_train)),activation='softmax')(dense) model = models.Model(inputs=input_layer, outputs=output_layer) if summary: model.summary() # train(self): class_weights = class_weight.compute_class_weight('balanced', np.unique(y_train), y_train.reshape(-1)) class_weights_dict = dict(enumerate(class_weights)) model.compile(optimizer=optimizer, loss=loss_type, metrics=metrics # accuracy ) result = model.fit(x=x_train, y=y_train, batch_size=batch_size, epochs=epoch, verbose=verbose, callbacks=my_callback, validation_data=(x_test, y_test), shuffle=True, class_weight=class_weights_dict ) validation_acc = np.amax(result.history['val_acc']) print('Best validation acc of epoch:', validation_acc) return {'loss': -validation_acc, 'status': STATUS_OK, 'model': model}
from keras import layers from keras import models from keras.datasets import cifar10 from keras.utils import to_categorical from sklearn.model_selection import train_test_split import numpy as np model1 = models.Sequential([ layers.Conv2D(32, (3, 3), activation='relu', input_shape=(32, 32, 3)), layers.MaxPooling2D((2, 2)), layers.Flatten(), layers.Dense(32, activation='relu'), layers.Dense(3, activation='softmax') ]) model1.summary() print("") model2 = models.Sequential([ layers.Conv2D(32, (3, 3), activation='relu', input_shape=(32, 32, 3)), layers.MaxPooling2D((2, 2)), layers.Conv2D(64, (3, 3), activation='relu'), layers.MaxPooling2D((2, 2)), layers.Flatten(), layers.Dense(32, activation='relu'), layers.Dense(3, activation='softmax') ]) model2.summary() print("") model3 = models.Sequential([ layers.Conv2D(32, (3, 3), activation='relu', input_shape=(32, 32, 3)),
# Simple ConvNet # ConvNet uses (image_height, image_width, image_channels) size input tensor # in this example, input shape is (28, 28, 1) for MNIST image format model = models.Sequential() model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1))) model.add(layers.MaxPooling2D((2, 2))) model.add(layers.Conv2D(64, (3, 3), activation='relu')) model.add(layers.MaxPooling2D((2, 2))) model.add(layers.Conv2D(64, (3, 3), activation='relu')) model.summary() # Add more layers model.add(layers.Flatten()) model.add(layers.Dense(64, activation='relu')) model.add(layers.Dense(10, activation='softmax')) model.summary() # ConvNet Training (train_images, train_labels), (test_images, test_labels) = mnist.load_data() train_images = train_images.reshape((60000, 28, 28, 1)) train_images = train_images.astype('float32') / 255 test_images = test_images.reshape((10000, 28, 28, 1)) test_images = test_images.astype('float32') / 255 train_labels = to_categorical(train_labels)
kernel_size=8, strides=2, activation='relu')(encoded_summary) # Size: 29 encoded_summary = layers.MaxPool1D(pool_size=2)(encoded_summary) encoded_summary = layers.Activation('relu')(encoded_summary) # Size: 14 encoded_summary = layers.Conv1D(filters=60, kernel_size=4, strides=1, activation='relu')(encoded_summary) # Size: 11 encoded_summary = layers.MaxPool1D(pool_size=2)(encoded_summary) encoded_summary = layers.Activation('relu')(encoded_summary) # Size: 5 encoded_summary = layers.Flatten()(encoded_summary) encoded_summary = layers.RepeatVector(MAX_DOCUMENT_LENGTH)(encoded_summary) document = layers.Input(shape=(MAX_DOCUMENT_LENGTH, )) encoded_document = layers.Embedding(np.shape(embedding_matrix)[0], EMBEDDINGS_SIZE, weights=[embedding_matrix], input_length=MAX_DOCUMENT_LENGTH, trainable=False)(document) merged = layers.add([encoded_summary, encoded_document]) merged = layers.Bidirectional( layers.LSTM((int)(EMBEDDINGS_SIZE / 2), return_sequences=True))(merged) merged = layers.Dropout(0.3)(merged) merged = layers.Bidirectional( layers.LSTM((int)(EMBEDDINGS_SIZE / 4), return_sequences=True))(merged)