def model_from_paper(multi_gpu=True, num_gpus=4): """ Defines hyperparameters and compiles the CNN model used in the paper: https://arxiv.org/abs/1804.06812 Returns ------- A Keras sequential model object """ model = Sequential() model.add(Conv2D(64, (3, 3), strides=(1, 1), input_shape=(128, 128, 1), kernel_initializer='glorot_uniform')) model.add(Activation('elu')) model.add(BatchNormalization()) model.add(Conv2D(64, (3, 3), strides=(1, 1), kernel_initializer='glorot_uniform')) model.add(Activation('elu')) model.add(BatchNormalization()) model.add(MaxPool2D(pool_size=(2, 2), strides=(2, 2))) model.add(Conv2D(128, (3, 3), strides=(1, 1), kernel_initializer='glorot_uniform')) model.add(Activation('elu')) model.add(BatchNormalization()) model.add(Conv2D(128, (3, 3), strides=(1, 1), kernel_initializer='glorot_uniform')) model.add(Activation('elu')) model.add(BatchNormalization()) model.add(MaxPool2D(pool_size=(2, 2), strides=(2, 2))) model.add(Conv2D(256, (3, 3), strides=(1, 1), kernel_initializer='glorot_uniform')) model.add(Activation('elu')) model.add(BatchNormalization()) model.add(Conv2D(256, (3, 3), strides=(1, 1), kernel_initializer='glorot_uniform')) model.add(Activation('elu')) model.add(BatchNormalization()) model.add(MaxPool2D(pool_size=(2, 2), strides=(2, 2))) model.add(Flatten()) model.add(Dense(2048)) model.add(Activation('elu')) model.add(BatchNormalization()) model.add(Dropout(0.5)) model.add(Dense(8, activation='softmax')) # compile model adam = optimizers.Adam(lr=1e-3, decay=0.95e-3) model.compile(optimizer=adam, loss='sparse_categorical_crossentropy', metrics=['accuracy']) if(multi_gpu == True): parallel_model = multi_gpu_model(model, gpus=num_gpus) parallel_model.compile(optimizer=adam, loss='sparse_categorical_crossentropy', metrics=['accuracy']) return model, parallel_model return model
#Read training data and convert target to one-hot for use with categorical_cross_entropy per documentation dataset = np.loadtxt(open(train_dataset_location), delimiter=',') X_train = dataset[:,0:30].astype(np.float64) Y_train = dataset[:,30].astype(int) Y_train = tf.keras.utils.to_categorical(Y_train) #Custom loss function def my_cat_crossentropy(target,output,from_logits=False,axis=-1): return tf.nn.softmax_cross_entropy_with_logits_v2(labels=target,logits=output) my_batch_size = 20 #Model defintion: my_model = Sequential() my_model.add(Dense(15, input_dim=30, kernel_initializer='glorot_normal')) my_model.add(Dropout(0.1)) my_model.add(Dense(2, kernel_initializer='glorot_normal',kernel_regularizer=tf.keras.regularizers.l2(l=0.01))) #Training model using multi-stage optimiser: #Stage 1 my_model.compile(optimizer=tf.train.AdamOptimizer(learning_rate=0.005,beta1=0.85,beta2=0.95), loss=my_cat_crossentropy, metrics=['accuracy']) my_model.fit(x=X_train, y=Y_train, batch_size=my_batch_size, epochs=100, verbose=VERBOSE, shuffle=True) #Stage 2 my_model.compile(optimizer=tf.train.AdamOptimizer(learning_rate=0.001,beta1=0.9,beta2=0.99), loss=my_cat_crossentropy, metrics=['accuracy']) my_model.fit(x=X_train, y=Y_train, batch_size=my_batch_size, epochs=150, verbose=VERBOSE, shuffle=True)# new #Stage 3 my_model.compile(optimizer=tf.train.AdamOptimizer(learning_rate=0.0005,beta1=0.9,beta2=0.99), loss=my_cat_crossentropy, metrics=['accuracy']) my_model.fit(x=X_train, y=Y_train, batch_size=my_batch_size, epochs=200, verbose=VERBOSE, shuffle=True)# new
def create_inception_resnet_v2(input_shape, reshape, nb_classes, scale=True): """ create create_inception_resnet_v2 model :param input_shape: :param reshape: :param nb_classes: :param scale: :return: """ # if K.image_dim_ordering() == 'th': # init = Input((3, 299, 299)) # else: # init = Input((input_shape)) init = Input(shape=input_shape) if reshape: init = Reshape(reshape)(init) else: init = init # input_x = tf.pad(init, [[0, 0], [32, 32], [32, 32], [0, 0]]) input_x = layers.ZeroPadding2D((32, 32))(init) # Input Shape is 299 x 299 x 3 (tf) or 3 x 299 x 299 (th) x = inception_resnet_stem(input_x) print("###### inception_resnet_stem : ", x.shape) # 10 x Inception Resnet A for i in range(10): x = inception_resnet_v2_A(x, scale_residual=scale) print("###### 10 x Inception Resnet A : ", x.shape) # Reduction A x = reduction_A(x, k=256, l=256, m=384, n=384) print("###### Reduction A : ", x.shape) # 20 x Inception Resnet B for i in range(20): x = inception_resnet_v2_B(x, scale_residual=scale) print("###### 20 x Inception Resnet B : ", x.shape) # Auxiliary tower aux_out = AveragePooling2D((5, 5), strides=3, padding='same')(x) aux_out = Conv2D(128, 1, 1, padding='same', activation='relu')(aux_out) aux_out = Conv2D(768, 5, 5, activation='relu', padding='same')(aux_out) aux_out = Flatten()(aux_out) aux_out = Dense(nb_classes, activation='softmax')(aux_out) # Reduction Resnet B x = reduction_resnet_v2_B(x) # 10 x Inception Resnet C for i in range(10): x = inception_resnet_v2_C(x, scale_residual=scale) # Average Pooling x = AveragePooling2D((8, 8), padding='same')(x) # Dropout x = Dropout(0.8)(x) x = Flatten()(x) # Output out = Dense(units=nb_classes, activation='softmax')(x) model = Model(inputs=init, outputs=out, name='model_Inception-Resnet-v2') return model
def sepcnn_model(input_shape, num_classes, num_features, blocks=2, filters=64, kernel_size=3, dropout_rate=0.2, pool_size=3, optimizer='adam', lr=1e-3, beta_1=0.9, beta_2=0.999, epsilon=0.003, decay=0, embedding_dim=100, use_pretrained_embedding=False, is_embedding_trainable=False, embedding_matrix=None, verbose=0): """Creates an instance of a separable CNN model. # Arguments blocks: int, number of pairs of sepCNN and pooling blocks in the model. filters: int, output dimension of the layers. kernel_size: int, length of the convolution window. embedding_dim: int, dimension of the embedding vectors. dropout_rate: float, percentage of input to drop at Dropout layers. pool_size: int, factor by which to downscale input at MaxPooling layer. input_shape: tuple, shape of input to the model. num_classes: int, number of output classes. num_features: int, number of words (embedding input dimension). use_pretrained_embedding: bool, true if pre-trained embedding is on. is_embedding_trainable: bool, true if embedding layer is trainable. embedding_matrix: dict, dictionary with embedding coefficients. # Returns A sepCNN model instance. """ op_units, op_activation = _get_last_layer_units_and_activation(num_classes) model = models.Sequential() # Add embedding layer. If pre-trained embedding is used add weights to the # embeddings layer and set trainable to input is_embedding_trainable flag. if use_pretrained_embedding: model.add( Embedding(input_dim=num_features, output_dim=embedding_dim, input_length=input_shape[0], weights=[embedding_matrix], trainable=is_embedding_trainable)) else: model.add( Embedding(input_dim=num_features, output_dim=embedding_dim, input_length=input_shape[0])) for _ in range(blocks - 1): model.add(Dropout(rate=dropout_rate)) model.add( SeparableConv1D(filters=filters, kernel_size=kernel_size, activation='relu', bias_initializer='random_uniform', depthwise_initializer='random_uniform', padding='same')) model.add( SeparableConv1D(filters=filters, kernel_size=kernel_size, activation='relu', bias_initializer='random_uniform', depthwise_initializer='random_uniform', padding='same')) model.add(MaxPooling1D(pool_size=pool_size)) model.add( SeparableConv1D(filters=filters * 2, kernel_size=kernel_size, activation='relu', bias_initializer='random_uniform', depthwise_initializer='random_uniform', padding='same')) model.add( SeparableConv1D(filters=filters * 2, kernel_size=kernel_size, activation='relu', bias_initializer='random_uniform', depthwise_initializer='random_uniform', padding='same')) model.add(GlobalAveragePooling1D()) model.add(Dropout(rate=dropout_rate)) model.add(Dense(op_units, activation=op_activation)) if num_classes == 2: loss = 'binary_crossentropy' else: loss = 'sparse_categorical_crossentropy' optimizer = Adam(lr=lr, beta_1=beta_1, beta_2=beta_2, epsilon=epsilon, decay=decay) model.compile(optimizer=optimizer, loss=loss, metrics=['acc']) return model
def Deeplabv3(weights='pascal_voc', input_tensor=None, input_shape=(512, 512, 3), classes=21, backbone='mobilenetv2', OS=16, alpha=1., activation=None): """ Instantiates the Deeplabv3+ architecture Optionally loads weights pre-trained on PASCAL VOC or Cityscapes. This model is available for TensorFlow only. # Arguments weights: one of 'pascal_voc' (pre-trained on pascal voc), 'cityscapes' (pre-trained on cityscape) or None (random initialization) input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model. input_shape: shape of input image. format HxWxC PASCAL VOC model was trained on (512,512,3) images. None is allowed as shape/width classes: number of desired classes. PASCAL VOC has 21 classes, Cityscapes has 19 classes. If number of classes not aligned with the weights used, last layer is initialized randomly backbone: backbone to use. one of {'xception','mobilenetv2'} activation: optional activation to add to the top of the network. One of 'softmax', 'sigmoid' or None OS: determines input_shape/feature_extractor_output ratio. One of {8,16}. Used only for xception backbone. alpha: controls the width of the MobileNetV2 network. This is known as the width multiplier in the MobileNetV2 paper. - If `alpha` < 1.0, proportionally decreases the number of filters in each layer. - If `alpha` > 1.0, proportionally increases the number of filters in each layer. - If `alpha` = 1, default number of filters from the paper are used at each layer. Used only for mobilenetv2 backbone. Pretrained is only available for alpha=1. # Returns A Keras model instance. # Raises RuntimeError: If attempting to run this model with a backend that does not support separable convolutions. ValueError: in case of invalid argument for `weights` or `backbone` """ if not (weights in {'pascal_voc', 'cityscapes', None}): raise ValueError( 'The `weights` argument should be either ' '`None` (random initialization), `pascal_voc`, or `cityscapes` ' '(pre-trained on PASCAL VOC)') if not (backbone in {'xception', 'mobilenetv2'}): raise ValueError('The `backbone` argument should be either ' '`xception` or `mobilenetv2` ') if input_tensor is None: img_input = Input(shape=input_shape) else: img_input = input_tensor if backbone == 'xception': if OS == 8: entry_block3_stride = 1 middle_block_rate = 2 # ! Not mentioned in paper, but required exit_block_rates = (2, 4) atrous_rates = (12, 24, 36) else: entry_block3_stride = 2 middle_block_rate = 1 exit_block_rates = (1, 2) atrous_rates = (6, 12, 18) x = Conv2D(32, (3, 3), strides=(2, 2), name='entry_flow_conv1_1', use_bias=False, padding='same')(img_input) x = BatchNormalization(name='entry_flow_conv1_1_BN')(x) x = Activation(tf.nn.relu)(x) x = _conv2d_same(x, 64, 'entry_flow_conv1_2', kernel_size=3, stride=1) x = BatchNormalization(name='entry_flow_conv1_2_BN')(x) x = Activation(tf.nn.relu)(x) x = _xception_block(x, [128, 128, 128], 'entry_flow_block1', skip_connection_type='conv', stride=2, depth_activation=False) x, skip1 = _xception_block(x, [256, 256, 256], 'entry_flow_block2', skip_connection_type='conv', stride=2, depth_activation=False, return_skip=True) x = _xception_block(x, [728, 728, 728], 'entry_flow_block3', skip_connection_type='conv', stride=entry_block3_stride, depth_activation=False) for i in range(16): x = _xception_block(x, [728, 728, 728], 'middle_flow_unit_{}'.format(i + 1), skip_connection_type='sum', stride=1, rate=middle_block_rate, depth_activation=False) x = _xception_block(x, [728, 1024, 1024], 'exit_flow_block1', skip_connection_type='conv', stride=1, rate=exit_block_rates[0], depth_activation=False) x = _xception_block(x, [1536, 1536, 2048], 'exit_flow_block2', skip_connection_type='none', stride=1, rate=exit_block_rates[1], depth_activation=True) else: OS = 8 first_block_filters = _make_divisible(32 * alpha, 8) x = Conv2D(first_block_filters, kernel_size=3, strides=(2, 2), padding='same', use_bias=False, name='Conv' if input_shape[2] == 3 else 'Conv_')(img_input) x = BatchNormalization(epsilon=1e-3, momentum=0.999, name='Conv_BN')(x) x = Activation(tf.nn.relu6, name='Conv_Relu6')(x) x = _inverted_res_block(x, filters=16, alpha=alpha, stride=1, expansion=1, block_id=0, skip_connection=False) x = _inverted_res_block(x, filters=24, alpha=alpha, stride=2, expansion=6, block_id=1, skip_connection=False) x = _inverted_res_block(x, filters=24, alpha=alpha, stride=1, expansion=6, block_id=2, skip_connection=True) x = _inverted_res_block(x, filters=32, alpha=alpha, stride=2, expansion=6, block_id=3, skip_connection=False) x = _inverted_res_block(x, filters=32, alpha=alpha, stride=1, expansion=6, block_id=4, skip_connection=True) x = _inverted_res_block(x, filters=32, alpha=alpha, stride=1, expansion=6, block_id=5, skip_connection=True) # stride in block 6 changed from 2 -> 1, so we need to use rate = 2 x = _inverted_res_block( x, filters=64, alpha=alpha, stride=1, # 1! expansion=6, block_id=6, skip_connection=False) x = _inverted_res_block(x, filters=64, alpha=alpha, stride=1, rate=2, expansion=6, block_id=7, skip_connection=True) x = _inverted_res_block(x, filters=64, alpha=alpha, stride=1, rate=2, expansion=6, block_id=8, skip_connection=True) x = _inverted_res_block(x, filters=64, alpha=alpha, stride=1, rate=2, expansion=6, block_id=9, skip_connection=True) x = _inverted_res_block(x, filters=96, alpha=alpha, stride=1, rate=2, expansion=6, block_id=10, skip_connection=False) x = _inverted_res_block(x, filters=96, alpha=alpha, stride=1, rate=2, expansion=6, block_id=11, skip_connection=True) x = _inverted_res_block(x, filters=96, alpha=alpha, stride=1, rate=2, expansion=6, block_id=12, skip_connection=True) x = _inverted_res_block( x, filters=160, alpha=alpha, stride=1, rate=2, # 1! expansion=6, block_id=13, skip_connection=False) x = _inverted_res_block(x, filters=160, alpha=alpha, stride=1, rate=4, expansion=6, block_id=14, skip_connection=True) x = _inverted_res_block(x, filters=160, alpha=alpha, stride=1, rate=4, expansion=6, block_id=15, skip_connection=True) x = _inverted_res_block(x, filters=320, alpha=alpha, stride=1, rate=4, expansion=6, block_id=16, skip_connection=False) # end of feature extractor # branching for Atrous Spatial Pyramid Pooling # Image Feature branch shape_before = tf.shape(x) b4 = GlobalAveragePooling2D()(x) b4_shape = tf.keras.backend.int_shape(b4) # from (b_size, channels)->(b_size, 1, 1, channels) b4 = Reshape((1, 1, b4_shape[1]))(b4) b4 = Conv2D(256, (1, 1), padding='same', use_bias=False, name='image_pooling')(b4) b4 = BatchNormalization(name='image_pooling_BN', epsilon=1e-5)(b4) b4 = Activation(tf.nn.relu)(b4) # upsample. have to use compat because of the option align_corners size_before = tf.keras.backend.int_shape(x) b4 = tf.keras.layers.experimental.preprocessing.Resizing( *size_before[1:3], interpolation="bilinear")(b4) # simple 1x1 b0 = Conv2D(256, (1, 1), padding='same', use_bias=False, name='aspp0')(x) b0 = BatchNormalization(name='aspp0_BN', epsilon=1e-5)(b0) b0 = Activation(tf.nn.relu, name='aspp0_activation')(b0) # there are only 2 branches in mobilenetV2. not sure why if backbone == 'xception': # rate = 6 (12) b1 = SepConv_BN(x, 256, 'aspp1', rate=atrous_rates[0], depth_activation=True, epsilon=1e-5) # rate = 12 (24) b2 = SepConv_BN(x, 256, 'aspp2', rate=atrous_rates[1], depth_activation=True, epsilon=1e-5) # rate = 18 (36) b3 = SepConv_BN(x, 256, 'aspp3', rate=atrous_rates[2], depth_activation=True, epsilon=1e-5) # concatenate ASPP branches & project x = Concatenate()([b4, b0, b1, b2, b3]) else: x = Concatenate()([b4, b0]) x = Conv2D(256, (1, 1), padding='same', use_bias=False, name='concat_projection')(x) x = BatchNormalization(name='concat_projection_BN', epsilon=1e-5)(x) x = Activation(tf.nn.relu)(x) x = Dropout(0.1)(x) # DeepLab v.3+ decoder if backbone == 'xception': # Feature projection # x4 (x2) block skip_size = tf.keras.backend.int_shape(skip1) x = tf.keras.layers.experimental.preprocessing.Resizing( *skip_size[1:3], interpolation="bilinear")(x) dec_skip1 = Conv2D(48, (1, 1), padding='same', use_bias=False, name='feature_projection0')(skip1) dec_skip1 = BatchNormalization(name='feature_projection0_BN', epsilon=1e-5)(dec_skip1) dec_skip1 = Activation(tf.nn.relu)(dec_skip1) x = Concatenate()([x, dec_skip1]) x = SepConv_BN(x, 256, 'decoder_conv0', depth_activation=True, epsilon=1e-5) x = SepConv_BN(x, 256, 'decoder_conv1', depth_activation=True, epsilon=1e-5) # you can use it with arbitary number of classes if (weights == 'pascal_voc' and classes == 21) or (weights == 'cityscapes' and classes == 19): last_layer_name = 'logits_semantic' else: last_layer_name = 'custom_logits_semantic' x = Conv2D(classes, (1, 1), padding='same', name=last_layer_name)(x) size_before3 = tf.keras.backend.int_shape(img_input) x = tf.keras.layers.experimental.preprocessing.Resizing( *size_before3[1:3], interpolation="bilinear")(x) # Ensure that the model takes into account # any potential predecessors of `input_tensor`. if input_tensor is not None: inputs = get_source_inputs(input_tensor) else: inputs = img_input if activation in {'softmax', 'sigmoid'}: x = tf.keras.layers.Activation(activation)(x) model = Model(inputs, x, name='deeplabv3plus') # load weights if weights == 'pascal_voc': if backbone == 'xception': weights_path = get_file( 'deeplabv3_xception_tf_dim_ordering_tf_kernels.h5', WEIGHTS_PATH_X, cache_subdir='models') else: weights_path = get_file( 'deeplabv3_mobilenetv2_tf_dim_ordering_tf_kernels.h5', WEIGHTS_PATH_MOBILE, cache_subdir='models') model.load_weights(weights_path, by_name=True) elif weights == 'cityscapes': if backbone == 'xception': weights_path = get_file( 'deeplabv3_xception_tf_dim_ordering_tf_kernels_cityscapes.h5', WEIGHTS_PATH_X_CS, cache_subdir='models') else: weights_path = get_file( 'deeplabv3_mobilenetv2_tf_dim_ordering_tf_kernels_cityscapes.h5', WEIGHTS_PATH_MOBILE_CS, cache_subdir='models') model.load_weights(weights_path, by_name=True) return model
def test_tf_keras_mnist_cnn(): """ This is the basic mnist cnn example from keras. """ _skip_if_no_tensorflow() import tensorflow as tf from tensorflow.python import keras from tensorflow.python.keras.models import Sequential from tensorflow.python.keras.layers import Dense, Dropout, Flatten, Activation from tensorflow.python.keras.layers import Conv2D, MaxPooling2D from tensorflow.python.keras import backend as K import shap tf.compat.v1.disable_eager_execution() batch_size = 128 num_classes = 10 epochs = 1 # input image dimensions img_rows, img_cols = 28, 28 # the data, split between train and test sets (x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data() if K.image_data_format() == 'channels_first': x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols) x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols) input_shape = (1, img_rows, img_cols) else: x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1) x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1) input_shape = (img_rows, img_cols, 1) x_train = x_train.astype('float32') x_test = x_test.astype('float32') x_train /= 255 x_test /= 255 # convert class vectors to binary class matrices y_train = keras.utils.to_categorical(y_train, num_classes) y_test = keras.utils.to_categorical(y_test, num_classes) model = Sequential() model.add( Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape)) model.add(Conv2D(64, (3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(32, activation='relu')) # 128 model.add(Dropout(0.5)) model.add(Dense(num_classes)) model.add(Activation('softmax')) model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adadelta(), metrics=['accuracy']) model.fit(x_train[:1000, :], y_train[:1000, :], batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_test[:1000, :], y_test[:1000, :])) # explain by passing the tensorflow inputs and outputs np.random.seed(0) inds = np.random.choice(x_train.shape[0], 20, replace=False) e = shap.GradientExplainer((model.layers[0].input, model.layers[-1].input), x_train[inds, :, :]) shap_values = e.shap_values(x_test[:1], nsamples=2000) sess = tf.compat.v1.keras.backend.get_session() diff = sess.run(model.layers[-1].input, feed_dict={model.layers[0].input: x_test[:1]}) - \ sess.run(model.layers[-1].input, feed_dict={model.layers[0].input: x_train[inds,:,:]}).mean(0) sums = np.array([shap_values[i].sum() for i in range(len(shap_values))]) d = np.abs(sums - diff).sum() assert d / np.abs(diff).sum( ) < 0.05, "Sum of SHAP values does not match difference! %f" % ( d / np.abs(diff).sum())
# show_images_by('dataset/shapes/triangles/*.png', 'circles-triangles-classification/dataset/output/triangles.png') ##################################################################################################################### # train the CNN modal classifier = Sequential() classifier.add( Conv2D(32, (3, 3), padding='same', input_shape=(28, 28, 3), activation='relu')) classifier.add(Conv2D(32, (3, 3), activation='relu')) classifier.add(MaxPooling2D(pool_size=(2, 2))) # antes era 0.25 classifier.add(Dropout(rate=0.5)) # Adding a second convolutional layer classifier.add(Conv2D(64, (3, 3), padding='same', activation='relu')) classifier.add(Conv2D(64, (3, 3), activation='relu')) classifier.add(MaxPooling2D(pool_size=(2, 2))) # antes era 0.25 classifier.add(Dropout(rate=0.5)) # Adding a third convolutional layer classifier.add(Conv2D(64, (3, 3), padding='same', activation='relu')) classifier.add(Conv2D(64, (3, 3), activation='relu')) classifier.add(MaxPooling2D(pool_size=(2, 2))) classifier.add(Dropout(rate=0.5)) # antes era 0.25 # Step 3 - Flattening
train_generator = train_datagen.flow_from_directory(training_dir, target_size=(HEIGHT, WIDTH), batch_size=batch_size) valid_generator = valid_datagen.flow_from_directory(validating_dir, target_size=(HEIGHT, WIDTH), batch_size=batch_size) # === Model === # ResNet structure without classification layer model = Sequential() model.add(VGG16( include_top=False, pooling='avg', weights='imagenet' )) # Output layer model.add(Dense(512, activation='relu')) model.add(Dropout(0.2)) model.add(Dense(64, activation='relu')) model.add(Dropout(0.2)) model.add(Dense(NUM_CLASSES, activation='softmax')) model.layers[0].trainable = False # Callback callbacks = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 10, restore_best_weights=True) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) model.fit(train_generator, validation_data=valid_generator, epochs=epochs, callbacks = [callbacks])
block5_pool (MaxPooling2D) (None, 8, 8, 512) 0 ================================================================= Total params: 20,024,384.0 Trainable params: 20,024,384.0 Non-trainable params: 0.0 """ # Freeze the layers which you don't want to train. Here I am freezing the first 5 layers. for layer in model.layers[:5]: layer.trainable = False #Adding custom Layers x = model.output x = Flatten()(x) x = Dense(1024, activation="relu")(x) x = Dropout(0.5)(x) x = Dense(1024, activation="relu")(x) predictions = Dense(11, activation="softmax")(x) # creating the final model # compile the model model_final.compile(loss="categorical_crossentropy", optimizer=optimizers.SGD(lr=0.0001, momentum=0.9), metrics=["accuracy"]) plot_model(model_final, to_file='model_plot.png', show_shapes=True, show_layer_names=True) # Initiate the train and test generators with data Augumentation train_datagen = ImageDataGenerator(rescale=1. / 255,
def __init__(self, num_units, dropout): self.dense1 = Dense(num_units*4, activation = tf.nn.relu) self.dense2 = Dense(num_units) self.dropout = Dropout(dropout)
batch_size=batch_size, class_mode='categorical'#Categorica etiquetas perro gato gorila ) #Crea una red convolucional cnn=Sequential()#Varias capas apiladas cnn.add(Convolution2D(filtrosConv1, tamano_filtro1, padding='same', input_shape=(altura, longitud,3), activation='relu')) cnn.add(MaxPooling2D(pool_size=tamano_pool)) cnn.add(Convolution2D(filtrosConv2, tamano_filtro2, padding='same', activation='relu')) cnn.add(MaxPooling2D(pool_size=tamano_pool)) cnn.add(Flatten())#Una dimension con toda la informacion de la red cnn.add(Dense(256,activation='relu'))#256 neuronas cnn.add(Dropout(0.5))#Apagar el 50 porciento de neuronas cada paso,para que aprenda varios caminos cnn.add(Dense(clases, activation='softmax')) cnn.compile(loss='categorical_crossentropy', optimizer= optimizers.Adam(lr=lr), metrics=['accuracy']) cnn.fit_generator(imagen_entrenamiento,steps_per_epoch=pasos, epochs=epocas, validation_data=imagen_validacion, validation_steps=pasos_Validacion) dir= './modelo/' if not os.path.exists(dir): os.mkdir(dir) cnn.save('./modelo/modelo.h5') cnn.save_weights('./modelo/pesos.h5')
def __init__(self, dropout): self.dropout = Dropout(dropout) self.eps = 1e-6
def __init__(self, num_units, dropout): self.conv1 = Conv1D(filters = num_units*4, kernel_size = 1, activation = tf.nn.relu) self.conv2 = Conv1D(filters = num_units, kernel_size = 1) self.dropout = Dropout(dropout)
"rb") #Replace the dots with the directory X = pickle.load(pickle_in) pickle_in = open("./y_concrete.pickle", "rb") #Replace the dots with the directory y = pickle.load(pickle_in) print("Data successfully loaded!") X = X / 255.0 model = Sequential() model.add( Conv2D(128, (3, 3), activation="relu", input_shape=(IMG_SIZE, IMG_SIZE, 1))) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(.3)) model.add(Conv2D(128, (3, 3), activation="relu")) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(.3)) model.add(Flatten()) model.add(Dense(258, activation="relu")) model.add(Dense(1, activation="sigmoid")) print("Compiling the model...") model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"]) print("Model successfully compiled!!")
def _init_graph(self): np.random.seed(self.seed) tf.set_random_seed(self.seed) self.feat_index = Input(shape=(self.field_size,), name='feat_index') # None*F self.feat_value = Input(shape=(self.field_size,), name='feat_value') # None*F self.embeddings = Embedding(self.feature_size, self.k, name='feature_embeddings', embeddings_regularizer=l2_reg(self.l2_fm))(self.feat_index) # None*F*k feat_value = Reshape((self.field_size, 1))(self.feat_value) self.embeddings = Multiply()([self.embeddings, feat_value]) # None*F*8 ###----first order------###### self.y_first_order = Embedding(self.feature_size, 1, name='feature_bias', embeddings_regularizer=l2_reg(self.l2))(self.feat_index) # None*F*1 self.y_first_order = Multiply()([self.y_first_order, feat_value]) # None*F*1 self.y_first_order = MySumLayer(axis=1)(self.y_first_order) # None*1 self.y_first_order = Dropout(self.dropout_keep_fm[0], seed=self.seed)(self.y_first_order) # None*1 ###------second order term-------### # sum_square part self.summed_feature_emb = MySumLayer(axis=1)(self.embeddings) # None*k self.summed_feature_emb_squred = Multiply()([self.summed_feature_emb, self.summed_feature_emb]) # None*k # square_sum part self.squared_feature_emb = Multiply()([self.embeddings, self.embeddings]) self.squared_sum_feature_emb = MySumLayer(axis=1)(self.squared_feature_emb) # None*k # second order self.y_second_order = Lambda(lambda x: x[0] - x[1])( [self.summed_feature_emb_squred, self.squared_sum_feature_emb]) # None*k self.y_second_order = MySumLayer(axis=1)(self.y_second_order) # None*1 self.y_second_order = Lambda(lambda x: x * 0.5)(self.y_second_order) # None*k ##deep self.y_deep = Reshape((self.field_size * self.k,))(self.embeddings) for i in range(0, len(self.deep_layers)): self.y_deep = Dense(self.deep_layers[i], activation='relu')(self.y_deep) self.y_deep = Dropout(self.dropout_keep_deep[i], seed=self.seed)(self.y_deep) # None*32 # deepFM if self.use_fm and self.use_deep: self.concat_y = Concatenate()([self.y_first_order, self.y_second_order, self.y_deep]) elif self.use_fm: self.concat_y = Concatenate()([self.y_first_order, self.y_second_order]) elif self.use_deep: self.concat_y = self.y_deep self.y = Dense(1, activation='sigmoid', name='output')(self.concat_y) # None*1 self.model = tf.keras.Model(inputs=[self.feat_index, self.feat_value], outputs=self.y, name='model') if self.optimizer_type == 'adam': self.optimizer = Adam(lr=self.learning_rate, decay=0.1) elif self.optimizer_type == 'adagrad': self.optimizer = Adagrad(lr=self.learning_rate, decay=0.1) if self.loss_type == 'logloss': self.loss = 'binary_crossentropy' print('use logloss') elif self.loss_type == 'mse': self.loss = 'mean_squared_error' print('use mse') if self.eval_metric == 'auc': self.metrics = auc else: self.metrics = self.eval_metric self.model.compile(optimizer=self.optimizer, loss=self.loss, metrics=[self.metrics])
activation='relu')) model.add(MaxPooling2D( pool_size=2, strides=2, padding='same', )) model.add(Conv2D(64, 3, strides=2, padding='same', activation='relu')) model.add(MaxPooling2D(2, 2, 'same')) model.add(Conv2D(128, 3, strides=2, padding='same', activation='relu')) model.add(MaxPooling2D(2, 2, 'same')) #把第二个池化层的输出扁平化为1维 model.add(Flatten()) model.add(Dense(1024, activation='relu')) model.add(Dropout(0.25)) model.add(Dense(10, activation='softmax')) # 定义优化器 sgd = SGD(lr=0.01) # 定义优化器,loss function,训练过程中计算准确率 model.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy']) model.fit(x_train_data, y_train_data, batch_size=64, epochs=100) # 评估模型 loss, accuracy = model.evaluate(x_test_data, y_test_data)
def build_vgg(img_rows: int = 224, img_cols: int = 224, num_classes: int = 1000): vgg = Sequential() vgg.add( Conv2D(64, kernel_size=(3, 3), activation='relu', padding='same', kernel_regularizer=l2(0.0005), input_shape=(img_rows, img_cols, 3))) vgg.add( Conv2D(64, kernel_size=(3, 3), activation='relu', padding='same', kernel_regularizer=l2(0.0005), input_shape=(img_rows, img_cols, 3))) vgg.add(MaxPooling2D()) # initial size /2 vgg.add( Conv2D(128, kernel_size=(3, 3), activation='relu', padding='same', kernel_regularizer=l2(0.0005))) vgg.add( Conv2D(128, kernel_size=(3, 3), activation='relu', padding='same', kernel_regularizer=l2(0.0005))) vgg.add(MaxPooling2D()) # initial size /4 vgg.add( Conv2D(256, kernel_size=(3, 3), activation='relu', padding='same', kernel_regularizer=l2(0.0005))) vgg.add( Conv2D(256, kernel_size=(3, 3), activation='relu', padding='same', kernel_regularizer=l2(0.0005))) vgg.add( Conv2D(256, kernel_size=(3, 3), activation='relu', padding='same', kernel_regularizer=l2(0.0005))) vgg.add( Conv2D(256, kernel_size=(3, 3), activation='relu', padding='same', kernel_regularizer=l2(0.0005))) vgg.add(MaxPooling2D()) # initial size /8 vgg.add( Conv2D(512, kernel_size=(3, 3), activation='relu', padding='same', kernel_regularizer=l2(0.0005))) vgg.add( Conv2D(512, kernel_size=(3, 3), activation='relu', padding='same', kernel_regularizer=l2(0.0005))) vgg.add( Conv2D(512, kernel_size=(3, 3), activation='relu', padding='same', kernel_regularizer=l2(0.0005))) vgg.add( Conv2D(512, kernel_size=(3, 3), activation='relu', padding='same', kernel_regularizer=l2(0.0005))) vgg.add(MaxPooling2D()) # initial size /16 vgg.add( Conv2D(512, kernel_size=(3, 3), activation='relu', padding='same', kernel_regularizer=l2(0.0005))) vgg.add( Conv2D(512, kernel_size=(3, 3), activation='relu', padding='same', kernel_regularizer=l2(0.0005))) vgg.add( Conv2D(512, kernel_size=(3, 3), activation='relu', padding='same', kernel_regularizer=l2(0.0005))) vgg.add( Conv2D(512, kernel_size=(3, 3), activation='relu', padding='same', kernel_regularizer=l2(0.0005))) vgg.add(MaxPooling2D()) # initial size /32 vgg.add(Flatten()) vgg.add(Dense(4096, activation='relu', kernel_regularizer=l2(0.0005))) vgg.add(Dense(4096, activation='relu', kernel_regularizer=l2(0.0005))) vgg.add(Dropout(0.5)) vgg.add( Dense(num_classes, activation='softmax', kernel_regularizer=l2(0.0005))) return vgg
input_shape = (img_width, img_height, 3) if os.path.isfile(MODEL): model = keras.models.load_model(MODEL) else: base_model = InceptionV3(input_shape=input_shape, weights='imagenet', include_top=False, pooling='avg') # freeze all layers of the based model that is already pre-trained. for layer in base_model.layers: layer.trainable = False x = base_model.output #x = GlobalMaxPooling2D()(x) x = Dense(128, activation='relu')(x) x = Dropout(0.2)(x) predictions = Dense(nb_classes, activation='softmax', name='predictions')(x) # add top layer block to your base model model = Model(inputs=base_model.input, outputs=predictions) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) train_datagen = ImageDataGenerator(rescale=1. / 255, rotation_range=transformation_ratio, shear_range=transformation_ratio,
def runCNN(params): os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' for key in params: if 'Filter' in key or 'Kernel' in key: params[key] = int(params[key]) num_classes = 10 filepath = os.path.dirname(os.path.abspath(__file__)) + '/data/fashion/' x_train, y_train = mnist_reader.load_mnist(filepath, kind='train') x_test, y_test = mnist_reader.load_mnist(filepath, kind='t10k') x_train = x_train.reshape((60000, 28, 28, 1)) x_test = x_test.reshape((10000, 28, 28, 1)) y_train = tf.keras.utils.to_categorical(y_train, num_classes=num_classes) y_test = tf.keras.utils.to_categorical(y_test, num_classes=num_classes) model = None tf.reset_default_graph() sess = tf.InteractiveSession() model = tf.keras.Sequential() model.add( Conv2D(params['layer1Filters'], params['layer1Kernel'], padding='same', input_shape=x_train.shape[1:])) model.add(Activation('relu')) model.add(Conv2D(params['layer2Filters'], params['layer2Kernel'])) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(params['firstDropout'])) model.add( Conv2D(params['layer3Filters'], params['layer3Kernel'], padding='same')) model.add(Activation('relu')) model.add(Conv2D(params['layer4Filters'], params['layer4Kernel'])) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(params['secondDropout'])) model.add(Flatten()) model.add(Dense(params['denseNodes'])) model.add(Activation('relu')) model.add(Dense(num_classes)) model.add(Activation('softmax')) optimizer = Adam(lr=params['learningRate']) model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy']) model.fit(x=x_train, y=y_train, batch_size=params['batchSize'], epochs=1) #Turns out that according to the 6 tests I ran, 1 epoch is enough to see which network is the most accurate, testing wasn't extensive though and there was definitely a great deal of variation score = model.evaluate(x_test, y_test, batch_size=128) print("Accuracy on Testing Data:", str(score[1] * 100) + "%") print("Hyperparameters: " + str(params)) sess.close() return {'loss': score[0], 'status': STATUS_OK}
model.add(Conv2D(64, (3, 3))) model.add(Activation('relu')) # слой подвыборки model.add(MaxPooling2D(pool_size=(2, 2))) # 4 св. слой model.add(Conv2D(128, (3, 3))) model.add(Activation('relu')) # слой подвыборки model.add(MaxPooling2D(pool_size=(2, 2))) model.add( Flatten()) # слой преобразования из двумерного в одномерное представление model.add(Dense(256)) # полносвязный слой model.add(Activation('relu')) model.add(Dropout(0.5)) # слой регуляризации model.add(Dense(100)) # выходной слой model.add(Activation('softmax')) # компилируем модель - задаем параметры для обучения model.compile(loss="categorical_crossentropy", optimizer='SGD', metrics=["accuracy"]) print(model.summary()) # обучаем модель с использованием генераторов model.fit_generator(train_generator, steps_per_epoch=nb_train_samples // batch_size, epochs=epochs, validation_data=val_generator, validation_steps=nb_validation_samples // batch_size)
def VGG_16(): model = Sequential() model.add( Conv2D(64, (3, 3), strides=(1, 1), input_shape=(224, 224, 3), padding='same', activation='relu', kernel_initializer='uniform')) model.add( Conv2D(64, (3, 3), strides=(1, 1), padding='same', activation='relu', kernel_initializer='uniform')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add( Conv2D(128, (3, 2), strides=(1, 1), padding='same', activation='relu', kernel_initializer='uniform')) model.add( Conv2D(128, (3, 3), strides=(1, 1), padding='same', activation='relu', kernel_initializer='uniform')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add( Conv2D(256, (3, 3), strides=(1, 1), padding='same', activation='relu', kernel_initializer='uniform')) model.add( Conv2D(256, (3, 3), strides=(1, 1), padding='same', activation='relu', kernel_initializer='uniform')) model.add( Conv2D(256, (3, 3), strides=(1, 1), padding='same', activation='relu', kernel_initializer='uniform')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add( Conv2D(512, (3, 3), strides=(1, 1), padding='same', activation='relu', kernel_initializer='uniform')) model.add( Conv2D(512, (3, 3), strides=(1, 1), padding='same', activation='relu', kernel_initializer='uniform')) model.add( Conv2D(512, (3, 3), strides=(1, 1), padding='same', activation='relu', kernel_initializer='uniform')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add( Conv2D(512, (3, 3), strides=(1, 1), padding='same', activation='relu', kernel_initializer='uniform')) model.add( Conv2D(512, (3, 3), strides=(1, 1), padding='same', activation='relu', kernel_initializer='uniform')) model.add( Conv2D(512, (3, 3), strides=(1, 1), padding='same', activation='relu', kernel_initializer='uniform')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Flatten()) model.add(Dense(4096, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(4096, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(1000, activation='softmax')) return model
def VGG19(input_shape, nb_classes, dropout=False, dropout_rate=0.2): """ Creates a VGG19 network. Parameters ---------- input_shape : tuple The shape of the input tensor not including the sample axis. Tensorflow uses the NHWC dimention ordering convention. nb_class : int The number of output class. The network will have this number of output nodes for one-hot encoding. dropout : bool Where or not to implement dropout in the fully-connected layers. dropout_rate : float Dropout rate. Returns ------- keras.models.Sequential() : The create VGG19 network. """ vgg19 = Sequential() # sub-net 1 vgg19.add( Conv2D(filters=64, kernel_size=3, padding='same', activation='relu', input_shape=input_shape)) vgg19.add( Conv2D(filters=64, kernel_size=3, padding='same', activation='relu')) vgg19.add(MaxPool2D(pool_size=2)) # sub-net 2 vgg19.add( Conv2D(filters=128, kernel_size=3, padding='same', activation='relu')) vgg19.add( Conv2D(filters=128, kernel_size=3, padding='same', activation='relu')) vgg19.add(MaxPool2D(pool_size=2)) # sub-net 3 vgg19.add( Conv2D(filters=256, kernel_size=3, padding='same', activation='relu')) vgg19.add( Conv2D(filters=256, kernel_size=3, padding='same', activation='relu')) vgg19.add( Conv2D(filters=256, kernel_size=3, padding='same', activation='relu')) vgg19.add( Conv2D(filters=256, kernel_size=3, padding='same', activation='relu')) vgg19.add(MaxPool2D(pool_size=2)) # sub-net 4 vgg19.add( Conv2D(filters=512, kernel_size=3, padding='same', activation='relu')) vgg19.add( Conv2D(filters=512, kernel_size=3, padding='same', activation='relu')) vgg19.add( Conv2D(filters=512, kernel_size=3, padding='same', activation='relu')) vgg19.add( Conv2D(filters=512, kernel_size=3, padding='same', activation='relu')) vgg19.add(MaxPool2D(pool_size=2)) # sub-net 5 vgg19.add( Conv2D(filters=512, kernel_size=3, padding='same', activation='relu')) vgg19.add( Conv2D(filters=512, kernel_size=3, padding='same', activation='relu')) vgg19.add( Conv2D(filters=512, kernel_size=3, padding='same', activation='relu')) vgg19.add( Conv2D(filters=512, kernel_size=3, padding='same', activation='relu')) vgg19.add(MaxPool2D(pool_size=2)) # dense layers vgg19.add(Flatten()) vgg19.add(Dense(units=4096, activation='relu')) vgg19.add(Dropout(dropout_rate)) if dropout else None vgg19.add(Dense(units=4096, activation='relu')) vgg19.add(Dropout(dropout_rate)) if dropout else None vgg19.add(Dense(units=nb_classes, activation='softmax')) return vgg19
from tensorflow.python.keras.layers import Conv1D, MaxPooling1D, Dense, Flatten, Input, AveragePooling1D, Dropout, Concatenate from tensorflow.python.keras import regularizers from readfile import load_data, embed_and_token REG = 0.0001 DROP = 0.1 tweets, labels = load_data() data, labels, embedding_layer = embed_and_token(tweets, labels) print(data.shape) print(labels.shape) sequence_input = Input(shape=(data.shape[1],), dtype='int32') # (Batch size, embedded_sequences = embedding_layer(sequence_input) embedded_sequences = Dropout(DROP)(embedded_sequences) x = Conv1D(256, 5, activation='relu', padding='same', kernel_regularizer=regularizers.l2(REG))(embedded_sequences) a = MaxPooling1D(5, strides=1, padding='same')(x) a = Dense(256, activation='relu')(a) x = Dropout(DROP)(a) x = Conv1D(256, 5, activation='relu', padding='same', kernel_regularizer=regularizers.l2(REG))(x) b = MaxPooling1D(5, strides=1, padding='same')(x) b = Dense(256, activation='relu')(b) x = Dropout(DROP)(b) x = Conv1D(128, 5, activation='relu', padding='same', kernel_regularizer=regularizers.l2(REG))(x) x = MaxPooling1D(31, padding='same')(x) x = Flatten()(x) # a = Flatten()(a) # b = Flatten()(b) # x = Concatenate()([a, b, x]) x = Dense(128, activation='relu')(x)
model.add(Conv2D(8, (2, 2), input_shape=smiles.shape[1:], kernel_regularizer=regularizers.l2(0.02))) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Conv2D(8, (2, 2), kernel_regularizer=regularizers.l2(0.02))) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Conv2D(8, (2, 2), kernel_regularizer=regularizers.l2(0.02))) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.5)) model.add(Flatten()) # this converts our 3D feature maps to 1D feature vectors model.add(Dense(1)) model.add(Activation('sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) # Use TensorBoard to visualize the TensorFlow graph model.fit(smiles, labels, batch_size=32, epochs=epoach, validation_data=(test_smiles, test_labels), callbacks=[tensorboard]) test_results = model.evaluate(test_smiles, test_labels)
from tensorflow.python.keras.models import Sequential from tensorflow.python.keras.layers import Dense, Dropout, Flatten from tensorflow.python.keras.layers import Conv2D, MaxPooling2D # In[10]: desain jaringan model = Sequential() model.add( Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=np.shape(train_input[0]))) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Conv2D(32, (3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Flatten()) model.add(Dense(1024, activation='tanh')) model.add(Dropout(0.5)) model.add(Dense(num_classes, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) print(model.summary()) # In[11]: import sequential import keras.callbacks tensorboard = keras.callbacks.TensorBoard(log_dir='./logs/mnist-style') # In[12]: 5menit kali 10 epoch = 50 menit model.fit(train_input,
padding='same', activation='relu')) cnn.add(MaxPooling2D(pool_size=tamano_pool)) cnn.add( Convolution2D(filtrosCon4, tamano_filtro2, padding='same', activation='relu')) cnn.add(MaxPooling2D(pool_size=tamano_pool)) cnn.add(Flatten()) cnn.add(Dense(256, activation='relu')) cnn.add(Dropout(0.5)) cnn.add(Dense(clases, activation='softmax')) cnn.compile(loss='categorical_crossentropy', optimizer=Adam(), metrics=['accuracy']) checkpointer = ModelCheckpoint(filepath='./modelo/best.hdf5', verbose=1, save_best_only=True) history = cnn.fit_generator(imagen_entrenamiento, epochs=epocas, validation_data=imagen_validacion, callbacks=[checkpointer])
def keras_estimator(model_dir, config, learning_rate, blocks=2, filters=64, dropout_rate=0.2, embedding_dim=200, kernel_size=3, pool_size=3, embedding_path=None, word_index=None): # Create model instance. model = models.Sequential() num_features = min(len(word_index) + 1, TOP_K) # Add embedding layer. If pre-trained embedding is used add weights to the # embeddings layer and set trainable to input is_embedding_trainable flag. if embedding_path != None: embedding_matrix = get_embedding_matrix(word_index, embedding_path, embedding_dim) is_embedding_trainable = True # set to False to freeze embedding weights model.add( Embedding(input_dim=num_features, output_dim=embedding_dim, input_length=MAX_SEQUENCE_LENGTH, weights=[embedding_matrix], trainable=is_embedding_trainable)) else: model.add( Embedding(input_dim=num_features, output_dim=embedding_dim, input_length=MAX_SEQUENCE_LENGTH)) for _ in range(blocks - 1): model.add(Dropout(rate=dropout_rate)) model.add( SeparableConv1D(filters=filters, kernel_size=kernel_size, activation='relu', bias_initializer='random_uniform', depthwise_initializer='random_uniform', padding='same')) model.add( SeparableConv1D(filters=filters, kernel_size=kernel_size, activation='relu', bias_initializer='random_uniform', depthwise_initializer='random_uniform', padding='same')) model.add(MaxPooling1D(pool_size=pool_size)) model.add( SeparableConv1D(filters=filters * 2, kernel_size=kernel_size, activation='relu', bias_initializer='random_uniform', depthwise_initializer='random_uniform', padding='same')) model.add( SeparableConv1D(filters=filters * 2, kernel_size=kernel_size, activation='relu', bias_initializer='random_uniform', depthwise_initializer='random_uniform', padding='same')) model.add(GlobalAveragePooling1D()) model.add(Dropout(rate=dropout_rate)) model.add(Dense(len(CLASSES), activation='softmax')) # Compile model with learning parameters. optimizer = tf.keras.optimizers.Adam(lr=learning_rate) model.compile(optimizer=optimizer, loss='sparse_categorical_crossentropy', metrics=['acc']) estimator = tf.keras.estimator.model_to_estimator(keras_model=model, model_dir=model_dir, config=config) return estimator
def InceptionV3(input_shape=None, classes=3, weights=None): img_input = Input(shape=input_shape) if image_data_format() == 'channels_first': channel_axis = 1 else: channel_axis = 3 x = conv2d_bn(img_input, 32, 3, 3, strides=(2, 2), padding='valid') x = conv2d_bn(x, 32, 3, 3, padding='valid') x = conv2d_bn(x, 64, 3, 3) x = MaxPooling2D((3, 3), strides=(2, 2))(x) x = conv2d_bn(x, 80, 1, 1, padding='valid') x = conv2d_bn(x, 192, 3, 3, padding='valid') x = MaxPooling2D((3, 3), strides=(2, 2))(x) # mixed 0: 35 x 35 x 256 branch1x1 = conv2d_bn(x, 64, 1, 1) branch5x5 = conv2d_bn(x, 48, 1, 1) branch5x5 = conv2d_bn(branch5x5, 64, 5, 5) branch3x3dbl = conv2d_bn(x, 64, 1, 1) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 32, 1, 1) x = concatenate( [branch1x1, branch5x5, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed0') # mixed 1: 35 x 35 x 288 branch1x1 = conv2d_bn(x, 64, 1, 1) branch5x5 = conv2d_bn(x, 48, 1, 1) branch5x5 = conv2d_bn(branch5x5, 64, 5, 5) branch3x3dbl = conv2d_bn(x, 64, 1, 1) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 64, 1, 1) x = concatenate( [branch1x1, branch5x5, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed1') # mixed 2: 35 x 35 x 288 branch1x1 = conv2d_bn(x, 64, 1, 1) branch5x5 = conv2d_bn(x, 48, 1, 1) branch5x5 = conv2d_bn(branch5x5, 64, 5, 5) branch3x3dbl = conv2d_bn(x, 64, 1, 1) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 64, 1, 1) x = concatenate( [branch1x1, branch5x5, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed2') # mixed 3: 17 x 17 x 768 branch3x3 = conv2d_bn(x, 384, 3, 3, strides=(2, 2), padding='valid') branch3x3dbl = conv2d_bn(x, 64, 1, 1) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch3x3dbl = conv2d_bn( branch3x3dbl, 96, 3, 3, strides=(2, 2), padding='valid') branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x) x = concatenate( [branch3x3, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed3') # mixed 4: 17 x 17 x 768 branch1x1 = conv2d_bn(x, 192, 1, 1) branch7x7 = conv2d_bn(x, 128, 1, 1) branch7x7 = conv2d_bn(branch7x7, 128, 1, 7) branch7x7 = conv2d_bn(branch7x7, 192, 7, 1) branch7x7dbl = conv2d_bn(x, 128, 1, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 1, 7) branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 192, 1, 1) x = concatenate( [branch1x1, branch7x7, branch7x7dbl, branch_pool], axis=channel_axis, name='mixed4') # mixed 5, 6: 17 x 17 x 768 for i in range(2): branch1x1 = conv2d_bn(x, 192, 1, 1) branch7x7 = conv2d_bn(x, 160, 1, 1) branch7x7 = conv2d_bn(branch7x7, 160, 1, 7) branch7x7 = conv2d_bn(branch7x7, 192, 7, 1) branch7x7dbl = conv2d_bn(x, 160, 1, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 1, 7) branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7) branch_pool = AveragePooling2D( (3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 192, 1, 1) x = concatenate( [branch1x1, branch7x7, branch7x7dbl, branch_pool], axis=channel_axis, name='mixed' + str(5 + i)) # mixed 7: 17 x 17 x 768 branch1x1 = conv2d_bn(x, 192, 1, 1) branch7x7 = conv2d_bn(x, 192, 1, 1) branch7x7 = conv2d_bn(branch7x7, 192, 1, 7) branch7x7 = conv2d_bn(branch7x7, 192, 7, 1) branch7x7dbl = conv2d_bn(x, 192, 1, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 192, 1, 1) x = concatenate( [branch1x1, branch7x7, branch7x7dbl, branch_pool], axis=channel_axis, name='mixed7') # mixed 8: 8 x 8 x 1280 branch3x3 = conv2d_bn(x, 192, 1, 1) branch3x3 = conv2d_bn(branch3x3, 320, 3, 3, strides=(2, 2), padding='valid') branch7x7x3 = conv2d_bn(x, 192, 1, 1) branch7x7x3 = conv2d_bn(branch7x7x3, 192, 1, 7) branch7x7x3 = conv2d_bn(branch7x7x3, 192, 7, 1) branch7x7x3 = conv2d_bn( branch7x7x3, 192, 3, 3, strides=(2, 2), padding='valid') branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x) x = concatenate( [branch3x3, branch7x7x3, branch_pool], axis=channel_axis, name='mixed8') # mixed 9: 8 x 8 x 2048 for i in range(2): branch1x1 = conv2d_bn(x, 320, 1, 1) branch3x3 = conv2d_bn(x, 384, 1, 1) branch3x3_1 = conv2d_bn(branch3x3, 384, 1, 3) branch3x3_2 = conv2d_bn(branch3x3, 384, 3, 1) branch3x3 = concatenate( [branch3x3_1, branch3x3_2], axis=channel_axis, name='mixed9_' + str(i)) branch3x3dbl = conv2d_bn(x, 448, 1, 1) branch3x3dbl = conv2d_bn(branch3x3dbl, 384, 3, 3) branch3x3dbl_1 = conv2d_bn(branch3x3dbl, 384, 1, 3) branch3x3dbl_2 = conv2d_bn(branch3x3dbl, 384, 3, 1) branch3x3dbl = concatenate( [branch3x3dbl_1, branch3x3dbl_2], axis=channel_axis) branch_pool = AveragePooling2D( (3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 192, 1, 1) x = concatenate( [branch1x1, branch3x3, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed' + str(9 + i)) x = GlobalAveragePooling2D(name='avg_pool')(x) x = Dropout(hyperparameters.dropout)(x) # softmax classifier x = Flatten()(x) x = Dense(classes)(x) x = Activation("softmax")(x) inputs = img_input # Create model. return Model(inputs, x, name='inception_v3')
def build_model(self): # Build the network of vgg for 10 classes with massive dropout and weight decay as described in the paper. model = Sequential() weight_decay = self.weight_decay model.add( Conv2D(64, (3, 3), padding='same', input_shape=self.x_shape, kernel_regularizer=regularizers.l2(weight_decay), trainable=self.mode)) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(Dropout(0.3)) model.add( Conv2D(64, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay), trainable=self.mode)) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(MaxPooling2D(pool_size=(2, 2))) model.add( Conv2D(128, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay), trainable=self.mode)) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(Dropout(0.4)) model.add( Conv2D(128, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay), trainable=self.mode)) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(MaxPooling2D(pool_size=(2, 2))) model.add( Conv2D(256, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay), trainable=self.mode)) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(Dropout(0.4)) model.add( Conv2D(256, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay), trainable=self.mode)) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(Dropout(0.4)) model.add( Conv2D(256, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay), trainable=self.mode)) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(MaxPooling2D(pool_size=(2, 2))) model.add( Conv2D(512, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay), trainable=self.mode)) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(Dropout(0.4)) model.add( Conv2D(512, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay), trainable=self.mode)) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(Dropout(0.4)) model.add( Conv2D(512, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay), trainable=self.mode)) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(MaxPooling2D(pool_size=(2, 2))) model.add( Conv2D(512, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay), trainable=self.mode)) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(Dropout(0.4)) model.add( Conv2D(512, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay), trainable=self.mode)) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(Dropout(0.4)) model.add( Conv2D(512, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay), name='vgg', trainable=self.mode)) model.add(Activation('relu')) model.add(BatchNormalization()) if self.include_top: model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.5)) model.add(Flatten()) model.add( Dense(512, kernel_regularizer=regularizers.l2(weight_decay), trainable=self.mode)) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(Dropout(0.5)) model.add(Dense(self.num_classes, trainable=self.mode)) model.add(Activation('softmax')) # model.add(MaxPooling2D(pool_size=(2, 2))) # model.add(Dropout(0.5)) # # model.add(Flatten()) # model.add(Dense(512,kernel_regularizer=regularizers.l2(weight_decay),trainable=self.mode)) # model.add(Activation('relu')) # model.add(BatchNormalization()) # # model.add(Dropout(0.5)) # model.add(Dense(self.num_classes,trainable=self.mode)) # model.add(Activation('softmax')) return model
def main(batch_size=100, n_paired_per_batch=100, cvset=0, p_dropT=0.5, p_dropE=0.1, stdE=0.05, fc_dimT=[50,50,50,50],fc_dimE=[60,60,60,60],latent_dim=3, recon_strT=1.0, recon_strE=0.1, cpl_str=10.0, n_epoch=2000, steps_per_epoch = 500, run_iter=0, model_id='crossval_noadaptloss',exp_name='patchseq_v2_noadapt'): train_dat, val_dat, train_ind_T, train_ind_E, val_ind, dir_pth = dataset_50fold(exp_name=exp_name,cvset=cvset) train_generator = DatagenTE(dataset=train_dat, batch_size=batch_size, n_paired_per_batch=n_paired_per_batch, steps_per_epoch=steps_per_epoch) chkpt_save_period = 1e7 #Architecture parameters ------------------------------ input_dim = [train_dat['T'].shape[1],train_dat['E'].shape[1]] #'_fcT_' + '-'.join(map(str, fc_dimT)) + \ #'_fcE_' + '-'.join(map(str, fc_dimE)) + \ fileid = model_id + \ '_rT_' + str(recon_strT) + \ '_rE_' + str(recon_strE) + \ '_cs_' + str(cpl_str) + \ '_pdT_' + str(p_dropT) + \ '_pdE_' + str(p_dropE) + \ '_sdE_' + str(stdE) + \ '_bs_' + str(batch_size) + \ '_np_' + str(n_paired_per_batch) + \ '_se_' + str(steps_per_epoch) +\ '_ne_' + str(n_epoch) + \ '_cv_' + str(cvset) + \ '_ri_' + str(run_iter) fileid = fileid.replace('.', '-') print(fileid) out_actfcn = ['elu','linear'] def add_gauss_noise(x): '''Injects additive gaussian noise independently into each element of input x''' x_noisy = x + tf.random.normal(shape=tf.shape(x), mean=0., stddev=stdE, dtype = tf.float32) return tf.keras.backend.in_train_phase(x_noisy, x) #Model inputs ----------------------------------------- M = {} M['in_ae_0'] = Input(shape=(input_dim[0],), name='in_ae_0') M['in_ae_1'] = Input(shape=(input_dim[1],), name='in_ae_1') M['ispaired_ae_0'] = Input(shape=(1,), name='ispaired_ae_0') M['ispaired_ae_1'] = Input(shape=(1,), name='ispaired_ae_1') #Transcriptomics arm--------------------------------------------------------------------------------- M['dr_ae_0'] = Dropout(p_dropT, name='dr_ae_0')(M['in_ae_0']) X = 'dr_ae_0' for j, units in enumerate(fc_dimT): Y = 'fc'+ format(j,'02d') +'_ae_0' M[Y] = Dense(units, activation='elu', name=Y)(M[X]) X = Y M['ldx_ae_0'] = Dense(latent_dim, activation='linear',name='ldx_ae_0')(M[X]) M['ld_ae_0'] = BatchNormalization(scale = False, center = False ,epsilon = 1e-10, momentum = 0.99, name='ld_ae_0')(M['ldx_ae_0']) X = 'ld_ae_0' for j, units in enumerate(reversed(fc_dimT)): Y = 'fc'+ format(j+len(fc_dimT),'02d') +'_ae_0' M[Y] = Dense(units, activation='elu', name=Y)(M[X]) X = Y M['ou_ae_0'] = Dense(input_dim[0], activation=out_actfcn[0], name='ou_ae_0')(M[X]) #Electrophysiology arm-------------------------------------------------------------------------------- M['no_ae_1'] = Lambda(add_gauss_noise,name='no_ae_1')(M['in_ae_1']) M['dr_ae_1'] = Dropout(p_dropE, name='dr_ae_1')(M['no_ae_1']) X = 'dr_ae_1' for j, units in enumerate(fc_dimE): Y = 'fc'+ format(j,'02d') +'_ae_1' M[Y] = Dense(units, activation='elu', name=Y)(M[X]) X = Y M['ldx_ae_1'] = Dense(latent_dim, activation='linear',name='ldx_ae_1')(M[X]) M['ld_ae_1'] = BatchNormalization(scale = False, center = False ,epsilon = 1e-10, momentum = 0.99, name='ld_ae_1')(M['ldx_ae_1']) X = 'ld_ae_1' for j, units in enumerate(reversed(fc_dimE)): Y = 'fc'+ format(j+len(fc_dimE),'02d') +'_ae_1' M[Y] = Dense(units, activation='elu', name=Y)(M[X]) X = Y M['ou_ae_1'] = Dense(input_dim[1], activation=out_actfcn[1], name='ou_ae_1')(M[X]) cplAE = Model(inputs=[M['in_ae_0'], M['in_ae_1'], M['ispaired_ae_0'], M['ispaired_ae_1']], outputs=[M['ou_ae_0'], M['ou_ae_1'],M['ld_ae_0'], M['ld_ae_1']]) def coupling_loss(zi, pairedi, zj, pairedj): '''Minimum singular value based loss. \n SVD is calculated over all datapoints \n MSE is calculated over only `paired` datapoints''' batch_size = tf.shape(zi)[0] paired_i = tf.reshape(pairedi, [tf.shape(pairedi)[0],]) paired_j = tf.reshape(pairedj, [tf.shape(pairedj)[0],]) zi_paired = tf.boolean_mask(zi, tf.equal(paired_i, 1.0)) zj_paired = tf.boolean_mask(zj, tf.equal(paired_j, 1.0)) vars_j_ = tf.square(tf.linalg.svd(zj - tf.reduce_mean(zj, axis=0), compute_uv=False))/tf.cast(batch_size - 1, tf.float32) vars_j = tf.where(tf.math.is_nan(vars_j_), tf.zeros_like(vars_j_) + tf.cast(1e-1,dtype=tf.float32), vars_j_) L_ij = tf.compat.v1.losses.mean_squared_error(zi_paired, zj_paired)/tf.maximum(tf.reduce_min(vars_j, axis=None),tf.cast(1e-2,dtype=tf.float32)) def loss(y_true, y_pred): #Adaptive version:#tf.multiply(tf.stop_gradient(L_ij), L_ij) return L_ij return loss #Create loss dictionary loss_dict = {'ou_ae_0': mse, 'ou_ae_1': mse, 'ld_ae_0': coupling_loss(zi=M['ld_ae_0'], pairedi=M['ispaired_ae_0'],zj=M['ld_ae_1'], pairedj=M['ispaired_ae_1']), 'ld_ae_1': coupling_loss(zi=M['ld_ae_1'], pairedi=M['ispaired_ae_1'],zj=M['ld_ae_0'], pairedj=M['ispaired_ae_0'])} #Loss weights dictionary loss_wt_dict = {'ou_ae_0': recon_strT, 'ou_ae_1': recon_strE, 'ld_ae_0': cpl_str, 'ld_ae_1': cpl_str} #Add loss definitions to the model cplAE.compile(optimizer='adam', loss=loss_dict, loss_weights=loss_wt_dict) #Checkpoint function definitions checkpoint_cb = ModelCheckpoint(filepath=(dir_pth['checkpoint']+fileid + '-checkpoint-' + '{epoch:04d}' + '.h5'), verbose=1, save_best_only=False, save_weights_only=True, mode='auto', period=chkpt_save_period) val_in = {'in_ae_0': val_dat['T'], 'in_ae_1': val_dat['E'], 'ispaired_ae_0': val_dat['T_ispaired'], 'ispaired_ae_1': val_dat['E_ispaired']} val_out = {'ou_ae_0': val_dat['T'], 'ou_ae_1': val_dat['E'], 'ld_ae_0': np.zeros((val_dat['T'].shape[0], latent_dim)), 'ld_ae_1': np.zeros((val_dat['E'].shape[0], latent_dim))} #Custom callback object log_cb = CSVLogger(filename=dir_pth['logs']+fileid+'.csv') last_checkpoint_epoch = 0 start_time = timeit.default_timer() cplAE.fit_generator(train_generator, validation_data=(val_in,val_out), epochs=n_epoch, max_queue_size=100, use_multiprocessing=False, workers=1, initial_epoch=last_checkpoint_epoch, verbose=2, callbacks=[checkpoint_cb,log_cb]) elapsed = timeit.default_timer() - start_time print('-------------------------------') print('Training time:',elapsed) print('-------------------------------') #Saving weights cplAE.save_weights(dir_pth['result']+fileid+'-modelweights'+'.h5') matsummary = {} matsummary['cvset'] = cvset matsummary['val_ind'] = val_ind matsummary['train_ind_T'] = train_ind_T matsummary['train_ind_E'] = train_ind_E #Trained model predictions i = 0 encoder = Model(inputs=M['in_ae_'+str(i)], outputs=M['ld_ae_'+str(i)]) matsummary['z_val_'+str(i)] = encoder.predict({'in_ae_'+str(i): val_dat['T']}) matsummary['z_train_'+str(i)] = encoder.predict({'in_ae_'+str(i): train_dat['T']}) i = 1 encoder = Model(inputs=M['in_ae_'+str(i)], outputs=M['ld_ae_'+str(i)]) matsummary['z_val_'+str(i)] = encoder.predict({'in_ae_'+str(i): val_dat['E']}) matsummary['z_train_'+str(i)] = encoder.predict({'in_ae_'+str(i): train_dat['E']}) sio.savemat(dir_pth['result']+fileid+'-summary', matsummary) return