conv_11 = layers.Conv2D(64, (3, 3), padding='same', activation='relu')(merge_2) conv_12 = layers.Conv2D(64, (3, 3), padding='same', activation='relu')(conv_11) # up convolutional layers 64 > 128 up_conv_3 = layers.Conv2DTranspose(32, (2, 2), strides=2, padding='same')(conv_12) merge_3 = layers.Concatenate()([up_conv_3, conv_2]) # convolutional layers 128 > 128 > 128 conv_13 = layers.Conv2D(32, (3, 3), padding='same', activation='relu')(merge_3) conv_14 = layers.Conv2D(32, (3, 3), padding='same', activation='relu')(conv_13) drop_1 = layers.Dropout(0.5)(conv_14) # convolutional layers 128 128 output_layer = layers.Conv2D(1, (1, 1), padding='same')(drop_1) model = models.Model(inputs=input_layer, outputs=output_layer) model.compile(loss='binary_crossentropy', optimizer=optimizers.Adam(learning_rate=0.001), metrics=['acc']) NAME = "u-net-{}".format(datetime.now().strftime("%H%M%S")) tensorboard = callbacks.TensorBoard(log_dir='u-net\\logs\\%s' % NAME) model.fit(x=x_train, y=y_train, batch_size=32, epochs=60, validation_split=0.3, callbacks=[tensorboard]) # %%
def grid_search(dropout_level_lst=[0.10], beta2_lst=[0.999], beta1_lst=[0.9], lr_rate_lst=[0.001], epsilon_lst=[1e-08], epo_lst=[10], bat_size_lst=[10]): # initialize values val_loss = np.inf best_params = dict() for dp_level in dropout_level_lst: inputs_h2 = layers.Input(shape=(64, 32, 2)) # same as (rscale, cscale, 2) # also same as the dimension for EACH image in the training set encoder0_pool_h2, encoder0_h2 = encoder_block_h2( inputs_h2, 8, dropout_level=dp_level) encoder1_pool_h2, encoder1_h2 = encoder_block_h2( encoder0_pool_h2, 16, dropout_level=dp_level) encoder2_pool_h2, encoder2_h2 = encoder_block_h2( encoder1_pool_h2, 32, dropout_level=dp_level) encoder3_pool_h2, encoder3_h2 = encoder_block_h2( encoder2_pool_h2, 64, dropout_level=dp_level) center_h2 = conv_block_h2(encoder3_pool_h2, 128, dropout_level=dp_level) decoder3_h2 = decoder_block_h2(center_h2, encoder3_h2, 64, dropout_level=dp_level) decoder2_h2 = decoder_block_h2(decoder3_h2, encoder2_h2, 32, dropout_level=dp_level) decoder1_h2 = decoder_block_h2(decoder2_h2, encoder1_h2, 16, dropout_level=dp_level) outputs_h2 = layers.Conv2D(3, (1, 1), padding="same")( decoder1_h2 ) # simply set number of output channels here, seems legit model_ht2b = models.Model(inputs=[inputs_h2], outputs=[outputs_h2]) for beta2 in beta2_lst: for beta1 in beta1_lst: for lr_rate in lr_rate_lst: for eps in epsilon_lst: adam = keras.optimizers.Adam(learning_rate=lr_rate, beta_1=beta1, beta_2=beta2, epsilon=eps) model_ht2b.compile( optimizer=adam, loss=custom_loss_rmse ) # let's use rmse for optimization becuase it is a bigger target than mse # construct checkpoint for saving the best model for current training filepath = "current.best.h5" checkpoint = ModelCheckpoint( filepath, monitor= 'val_loss', # this must be the same string as a metric from your model training verbose output verbose=1, save_best_only=True, mode='min', # we want minimum loss save_weights_only= False # we want to save the entire model, not just the weights ) callbacks_list = [checkpoint] for epo in epo_lst: for bat_size in bat_size_lst: start = time.time() history_ht2b = model_ht2b.fit( train_images_t2b, train_labels_t2b, validation_data=(val_images_t2b, val_labels_t2b), epochs=epo, batch_size=bat_size, shuffle=True, callbacks=callbacks_list, verbose=1) training_time = time.time() - start # load best model from current training b/c the best model might not be the last model model_ht2b = tf.keras.models.load_model( 'current.best.h5', custom_objects={ 'custom_loss_rmse': custom_loss_rmse }) new_loss = custom_loss_rmse( val_labels_t2b, model_ht2b.predict(val_images_t2b)) if new_loss.numpy() < val_loss: print() print( 'final validation loss decreased from ', val_loss, ' to ', new_loss.numpy()) print( 'saving the current best model as the overall best model' ) print(100 * '*') val_loss = new_loss.numpy() best_params['best_dropout_rate'] = dp_level best_params['best_beta_2'] = beta2 best_params['best_beta_1'] = beta1 best_params['best_learning_rate'] = lr_rate best_params['best_epsilon'] = eps best_params['best_epochs'] = epo best_params['best_batch_size'] = bat_size best_params[ 'best_val_loss_reached'] = val_loss best_params[ 'training_time'] = training_time # best_params['val_loss_his'] = history_ht2b.history['val_loss'] # best_params['train_loss_his'] = history_ht2b.history['loss'] # comment these out for now because they take way too much space when printed out # save the best overall grid-searched model found so far model_ht2b.save('model.best.h5') # save history of validation-loss from the best model to observe epochs effect with open('best_val_loss_history.db', 'wb') as file_pi: pk.dump( history_ht2b.history['val_loss'], file_pi) # later open with # val_loss_history_ht2b = pk.load(open('best_val_loss_history.db', "rb")) # save history of training-loss from the best model to observe epochs effect with open('best_train_loss_history.db', 'wb') as file_pi: pk.dump(history_ht2b.history['loss'], file_pi) # later open with # train_loss_history_ht2b = pk.load(open('best_train_loss_history.db', "rb")) # save the best_params dictionary along the way incase training gets killed mid-way and the function doesn't get to finish # "w" mode automatically overwrites if the file already exists param_json = json.dumps(best_params) f = open("best_params.json", "w") f.write(param_json) f.close() # save a plot of the val_loss_history for the best performing model for observation fig, ax = get_figure() fig.set_size_inches(20, 10) num_epochs = len( history_ht2b.history['val_loss']) startpoints = 0 ax.set_yscale( 'log' ) # set y-axis to log_10 scale for better viewing ax.plot((np.arange(num_epochs * 1) + 1)[startpoints:], history_ht2b.history['loss'] [startpoints:], linewidth=1, color="orange", label="training_loss") ax.plot((np.arange(num_epochs * 1) + 1)[startpoints:], history_ht2b.history['val_loss'] [startpoints:], linewidth=1, color="blue", label="validation loss") ax.set_xlabel('epochs') ax.set_ylabel('log loss') ax.legend(frameon=False) fig.savefig('best_model_loss_history.png') else: print( 'final validation loss did not decrease for this set of parameters' ) print( 'current overall best model and parameters does not get updated' ) print(100 * '*') return best_params
import tensorflow.keras.layers as KL import tensorflow.keras.models as KM ## Dataset mnist = tf.keras.datasets.mnist (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train, x_test = x_train/255.0, x_test/255.0 x_train, x_test = np.expand_dims(x_train, axis=-1), np.expand_dims(x_test, axis=-1) ## Model inputs = KL.Input(shape=(28, 28, 1)) c = KL.Conv2D(32, (3, 3), padding="valid", activation=tf.nn.relu)(inputs) m = KL.MaxPool2D((2, 2), (2, 2))(c) d = KL.Dropout(0.5)(m) c = KL.Conv2D(64, (3, 3), padding="valid", activation=tf.nn.relu)(d) m = KL.MaxPool2D((2, 2), (2, 2))(c) d = KL.Dropout(0.5)(m) c = KL.Conv2D(128, (3, 3), padding="valid", activation=tf.nn.relu)(d) f = KL.Flatten()(c) outputs = KL.Dense(10, activation=tf.nn.softmax)(f) model = KM.Model(inputs, outputs) model.summary() model.compile(optimizer="adam", loss="sparse_categorical_crossentropy", metrics=["accuracy"]) model.fit(x_train, y_train, epochs=5) test_loss, test_acc = model.evaluate(x_test, y_test) print("Test Loss: {0} - Test Acc: {1}".format(test_loss, test_acc))
def unet(input_size, n_filters=64, kernel_size=3, dropout_rate=0.1, output_channerl=1): input = layers.Input(input_size) # conv conv1_1 = down_conv(input, n_filters, kernel_size, dropout_rate) conv1_2 = down_conv(conv1_1, n_filters, kernel_size, dropout_rate) pool1 = layers.MaxPooling2D(pool_size=(2, 2))(conv1_2) pool1 = layers.Dropout(rate=dropout_rate)(pool1) conv2_1 = down_conv(pool1, 2 * n_filters, kernel_size, dropout_rate) conv2_2 = down_conv(conv2_1, 2 * n_filters, kernel_size, dropout_rate) pool2 = layers.MaxPooling2D(pool_size=(2, 2))(conv2_2) pool2 = layers.Dropout(rate=dropout_rate)(pool2) conv3_1 = down_conv(pool2, 4 * n_filters, kernel_size, dropout_rate) conv3_2 = down_conv(conv3_1, 4 * n_filters, kernel_size, dropout_rate) pool3 = layers.MaxPooling2D(pool_size=(2, 2))(conv3_2) pool3 = layers.Dropout(rate=dropout_rate)(pool3) conv4_1 = down_conv(pool3, 8 * n_filters, kernel_size, dropout_rate) conv4_2 = down_conv(conv4_1, 8 * n_filters, kernel_size, dropout_rate) pool4 = layers.MaxPooling2D(pool_size=(2, 2))(conv4_2) pool4 = layers.Dropout(rate=dropout_rate)(pool4) conv5_1 = down_conv(pool4, 16 * n_filters, kernel_size, dropout_rate) conv5_2 = down_conv(conv5_1, 16 * n_filters, kernel_size, dropout_rate) # deconv upconv6 = layers.Conv2DTranspose(filters=8 * n_filters, kernel_size=(kernel_size, kernel_size), strides=(2, 2), padding='same')(conv5_2) upconv6 = layers.concatenate([conv4_2, upconv6]) conv6_1 = down_conv(upconv6, 8 * n_filters, kernel_size, dropout_rate) conv6_2 = down_conv(conv6_1, 8 * n_filters, kernel_size, dropout_rate) upconv7 = layers.Conv2DTranspose(filters=4 * n_filters, kernel_size=(kernel_size, kernel_size), strides=(2, 2), padding='same')(conv6_2) upconv7 = layers.concatenate([conv3_2, upconv7]) conv7_1 = down_conv(upconv7, 4 * n_filters, kernel_size, dropout_rate) conv7_2 = down_conv(conv7_1, 4 * n_filters, kernel_size, dropout_rate) upconv8 = layers.Conv2DTranspose(filters=2 * n_filters, kernel_size=(kernel_size, kernel_size), strides=(2, 2), padding='same')(conv7_2) upconv8 = layers.concatenate([conv2_2, upconv8]) conv8_1 = down_conv(upconv8, 2 * n_filters, kernel_size, dropout_rate) conv8_2 = down_conv(conv8_1, 2 * n_filters, kernel_size, dropout_rate) upconv9 = layers.Conv2DTranspose(filters=n_filters, kernel_size=(kernel_size, kernel_size), strides=(2, 2), padding='same')(conv8_2) upconv9 = layers.concatenate([conv1_2, upconv9]) conv9_1 = down_conv(upconv9, n_filters, kernel_size, dropout_rate) conv9_2 = down_conv(conv9_1, n_filters, kernel_size, dropout_rate) # last layer output = layers.Conv2D(filters=output_channerl, kernel_size=(1, 1), activation='sigmoid')(conv9_2) model = models.Model(input, output) return model
def InceptionV3(): img_input = layers.Input(shape=(256, 256, 3)) channel_axis = 3 x = conv2d_bn(img_input, 32, 3, 3, strides=(2, 2), padding='valid') x = conv2d_bn(x, 32, 3, 3, padding='valid') x = conv2d_bn(x, 64, 3, 3) x = layers.MaxPooling2D((3, 3), strides=(2, 2))(x) x = conv2d_bn(x, 80, 1, 1, padding='valid') x = conv2d_bn(x, 192, 3, 3, padding='valid') x = layers.MaxPooling2D((3, 3), strides=(2, 2))(x) # mixed 0: 35 x 35 x 256 branch1x1 = conv2d_bn(x, 64, 1, 1) branch5x5 = conv2d_bn(x, 48, 1, 1) branch5x5 = conv2d_bn(branch5x5, 64, 5, 5) branch3x3dbl = conv2d_bn(x, 64, 1, 1) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch_pool = layers.AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 32, 1, 1) x = layers.concatenate([branch1x1, branch5x5, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed0') # mixed 1: 35 x 35 x 288 branch1x1 = conv2d_bn(x, 64, 1, 1) branch5x5 = conv2d_bn(x, 48, 1, 1) branch5x5 = conv2d_bn(branch5x5, 64, 5, 5) branch3x3dbl = conv2d_bn(x, 64, 1, 1) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch_pool = layers.AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 64, 1, 1) x = layers.concatenate([branch1x1, branch5x5, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed1') # mixed 2: 35 x 35 x 288 branch1x1 = conv2d_bn(x, 64, 1, 1) branch5x5 = conv2d_bn(x, 48, 1, 1) branch5x5 = conv2d_bn(branch5x5, 64, 5, 5) branch3x3dbl = conv2d_bn(x, 64, 1, 1) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch_pool = layers.AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 64, 1, 1) x = layers.concatenate([branch1x1, branch5x5, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed2') # mixed 3: 17 x 17 x 768 branch3x3 = conv2d_bn(x, 384, 3, 3, strides=(2, 2), padding='valid') branch3x3dbl = conv2d_bn(x, 64, 1, 1) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3, strides=(2, 2), padding='valid') branch_pool = layers.MaxPooling2D((3, 3), strides=(2, 2))(x) x = layers.concatenate([branch3x3, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed3') # mixed 4: 17 x 17 x 768 branch1x1 = conv2d_bn(x, 192, 1, 1) branch7x7 = conv2d_bn(x, 128, 1, 1) branch7x7 = conv2d_bn(branch7x7, 128, 1, 7) branch7x7 = conv2d_bn(branch7x7, 192, 7, 1) branch7x7dbl = conv2d_bn(x, 128, 1, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 1, 7) branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7) branch_pool = layers.AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 192, 1, 1) x = layers.concatenate([branch1x1, branch7x7, branch7x7dbl, branch_pool], axis=channel_axis, name='mixed4') # mixed 5, 6: 17 x 17 x 768 for i in range(2): branch1x1 = conv2d_bn(x, 192, 1, 1) branch7x7 = conv2d_bn(x, 160, 1, 1) branch7x7 = conv2d_bn(branch7x7, 160, 1, 7) branch7x7 = conv2d_bn(branch7x7, 192, 7, 1) branch7x7dbl = conv2d_bn(x, 160, 1, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 1, 7) branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7) branch_pool = layers.AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 192, 1, 1) x = layers.concatenate( [branch1x1, branch7x7, branch7x7dbl, branch_pool], axis=channel_axis, name='mixed' + str(5 + i)) # mixed 7: 17 x 17 x 768 branch1x1 = conv2d_bn(x, 192, 1, 1) branch7x7 = conv2d_bn(x, 192, 1, 1) branch7x7 = conv2d_bn(branch7x7, 192, 1, 7) branch7x7 = conv2d_bn(branch7x7, 192, 7, 1) branch7x7dbl = conv2d_bn(x, 192, 1, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7) branch_pool = layers.AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 192, 1, 1) x = layers.concatenate([branch1x1, branch7x7, branch7x7dbl, branch_pool], axis=channel_axis, name='mixed7') # mixed 8: 8 x 8 x 1280 branch3x3 = conv2d_bn(x, 192, 1, 1) branch3x3 = conv2d_bn(branch3x3, 320, 3, 3, strides=(2, 2), padding='valid') branch7x7x3 = conv2d_bn(x, 192, 1, 1) branch7x7x3 = conv2d_bn(branch7x7x3, 192, 1, 7) branch7x7x3 = conv2d_bn(branch7x7x3, 192, 7, 1) branch7x7x3 = conv2d_bn(branch7x7x3, 192, 3, 3, strides=(2, 2), padding='valid') branch_pool = layers.MaxPooling2D((3, 3), strides=(2, 2))(x) x = layers.concatenate([branch3x3, branch7x7x3, branch_pool], axis=channel_axis, name='mixed8') # mixed 9: 8 x 8 x 2048 for i in range(2): branch1x1 = conv2d_bn(x, 320, 1, 1) branch3x3 = conv2d_bn(x, 384, 1, 1) branch3x3_1 = conv2d_bn(branch3x3, 384, 1, 3) branch3x3_2 = conv2d_bn(branch3x3, 384, 3, 1) branch3x3 = layers.concatenate([branch3x3_1, branch3x3_2], axis=channel_axis, name='mixed9_' + str(i)) branch3x3dbl = conv2d_bn(x, 448, 1, 1) branch3x3dbl = conv2d_bn(branch3x3dbl, 384, 3, 3) branch3x3dbl_1 = conv2d_bn(branch3x3dbl, 384, 1, 3) branch3x3dbl_2 = conv2d_bn(branch3x3dbl, 384, 3, 1) branch3x3dbl = layers.concatenate([branch3x3dbl_1, branch3x3dbl_2], axis=channel_axis) branch_pool = layers.AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 192, 1, 1) x = layers.concatenate( [branch1x1, branch3x3, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed' + str(9 + i)) x = layers.GlobalAveragePooling2D()(x) x = layers.Dense(256)(x) inputs = img_input # Create model. model = models.Model(inputs, x, name='inception_v3') model.load_weights("face_recog_model-k-n-face_model.h5") return model
def f_define_model(config_dict, name='1'): ''' Function that defines the model and compiles it. Reads in a dictionary with parameters for CNN model prototype and returns a keral model ''' ### Extract info from the config_dict shape = config_dict['model']['input_shape'] loss_fn = config_dict['training']['loss'] metrics = config_dict['training']['metrics'] resnet = False ### Variable storing whether the models is resnet or not. This is needed for specifying the loss function. custom_model = False ### Variable storing whether the models is a layer-by-layer build code (not using the protytype function). # Choose model if name == '3': # Simple layered, with inner dropout model_par_dict = { 'conv_size_list': [80, 80, 80], 'kernel_size': (3, 3), 'no_pool': False, 'pool_size': (2, 2), 'strides': 1, 'learn_rate': 0.00002, 'inner_dropout': None, 'outer_dropout': 0.3, 'dense_size': 51, 'final_activation': 'sigmoid', 'double_conv': False } if name == '4': # Simple layered, with inner dropout model_par_dict = { 'conv_size_list': [120, 120, 120], 'kernel_size': (3, 3), 'no_pool': False, 'pool_size': (2, 2), 'strides': 1, 'learn_rate': 0.00002, 'inner_dropout': None, 'outer_dropout': 0.3, 'dense_size': 51, 'final_activation': 'sigmoid', 'double_conv': False } if name == '8': # Simple layered, with inner dropout model_par_dict = { 'conv_size_list': [80, 80], 'kernel_size': (4, 4), 'no_pool': False, 'pool_size': (3, 3), 'strides': 1, 'learn_rate': 0.00002, 'inner_dropout': None, 'outer_dropout': 0.3, 'dense_size': 51, 'final_activation': 'sigmoid', 'double_conv': True } if name == '9': # Simple layered, with inner dropout model_par_dict = { 'conv_size_list': [120, 120], 'kernel_size': (4, 4), 'no_pool': False, 'pool_size': (3, 3), 'strides': 1, 'learn_rate': 0.00002, 'inner_dropout': None, 'outer_dropout': 0.3, 'dense_size': 51, 'final_activation': 'sigmoid', 'double_conv': True } if name == '15': # Striding single conv model_par_dict = { 'conv_size_list': [80, 100, 120], 'kernel_size': (6, 6), 'no_pool': True, 'pool_size': (2, 2), 'strides': [2, 2, 1], 'learn_rate': 0.00002, 'inner_dropout': 0.1, 'outer_dropout': 0.3, 'dense_size': 51, 'final_activation': 'sigmoid', 'double_conv': False } if name == '16': # Striding single conv model_par_dict = { 'conv_size_list': [40, 60, 80], 'kernel_size': (6, 6), 'no_pool': True, 'pool_size': (2, 2), 'strides': [2, 2, 1], 'learn_rate': 0.00002, 'inner_dropout': 0.1, 'outer_dropout': 0.3, 'dense_size': 51, 'final_activation': 'sigmoid', 'double_conv': False } if name == '17': # Simple layered, with inner dropout model_par_dict = { 'conv_size_list': [400, 400, 400, 400], 'kernel_size': (4, 4), 'no_pool': False, 'pool_size': (2, 2), 'strides': 1, 'learn_rate': 0.000002, 'inner_dropout': None, 'outer_dropout': 0.5, 'dense_size': 40, 'final_activation': 'sigmoid', 'double_conv': False } if name == '18': # Simple layered, with inner dropout model_par_dict = { 'conv_size_list': [160, 200, 240, 320], 'kernel_size': (4, 4), 'no_pool': False, 'pool_size': (2, 2), 'strides': 1, 'learn_rate': 0.000002, 'inner_dropout': None, 'outer_dropout': 0.3, 'dense_size': 10, 'final_activation': 'sigmoid', 'double_conv': False } if name == '19': # Simple layered, with inner dropout model_par_dict = { 'conv_size_list': [160, 320], 'kernel_size': (4, 4), 'no_pool': False, 'pool_size': (3, 3), 'strides': 1, 'learn_rate': 0.000002, 'inner_dropout': None, 'outer_dropout': 0.3, 'dense_size': 10, 'final_activation': 'sigmoid', 'double_conv': True } if name == '20': # Simple layered, with inner dropout model_par_dict = { 'conv_size_list': [200, 320], 'kernel_size': (4, 4), 'no_pool': False, 'pool_size': (3, 3), 'strides': 1, 'learn_rate': 0.000002, 'inner_dropout': None, 'outer_dropout': 0.3, 'dense_size': 40, 'final_activation': 'sigmoid', 'double_conv': True } elif name == '0': custom_model = True learn_rate = 0.001 inputs = layers.Input(shape=shape) h = inputs # Convolutional layers h = Conv2D(64, kernel_size=(3, 3), activation='relu', strides=1, padding='same')(h) h = Conv2D(128, kernel_size=(3, 3), activation='relu', strides=2, padding='same')(h) h = Conv2D(256, kernel_size=(3, 3), activation='relu', strides=1, padding='same')(h) h = Conv2D(256, kernel_size=(3, 3), activation='relu', strides=2, padding='same')(h) h = Flatten()(h) h = Dense(512, activation='relu')(h) # Ouptut layer outputs = layers.Dense(1, activation='sigmoid')(h) elif name == '30': custom_model = True learn_rate = 0.000005 inputs = layers.Input(shape=shape) h = inputs h = layers.Conv2D(80, kernel_size=(4, 4), strides=1, activation='relu', padding='same')(h) h = layers.BatchNormalization(epsilon=1e-5, momentum=0.99)(h) h = layers.Conv2D(160, kernel_size=(4, 4), strides=1, activation='relu', padding='same')(h) h = layers.BatchNormalization(epsilon=1e-5, momentum=0.99)(h) h = layers.MaxPooling2D(pool_size=(3, 3))(h) h = layers.Conv2D(240, kernel_size=(4, 4), strides=1, activation='relu', padding='same')(h) h = layers.BatchNormalization(epsilon=1e-5, momentum=0.99)(h) h = layers.Conv2D(320, kernel_size=(4, 4), strides=1, activation='relu', padding='same')(h) h = layers.BatchNormalization(epsilon=1e-5, momentum=0.99)(h) h = layers.MaxPooling2D(pool_size=(3, 3))(h) h = layers.Flatten()(h) h = layers.Dropout(rate=0.3)(h) h = layers.Dense(51, activation='relu')(h) h = layers.BatchNormalization(epsilon=1e-5, momentum=0.99)(h) # Ouptut layer outputs = layers.Dense(1, activation='sigmoid')(h) elif name == '31': custom_model = True learn_rate = 0.000002 inputs = layers.Input(shape=shape) h = inputs # Convolutional layers conv_sizes = [200, 200] conv_args = dict(kernel_size=(4, 4), activation='relu', padding='same') for conv_size in conv_sizes: h = layers.Conv2D(conv_size, **conv_args)(h) h = layers.BatchNormalization(epsilon=1e-5, momentum=0.99)(h) h = layers.Conv2D(conv_size, **conv_args)(h) h = layers.BatchNormalization(epsilon=1e-5, momentum=0.99)(h) h = layers.MaxPooling2D(pool_size=(3, 3))(h) h = layers.Flatten()(h) h = layers.Dense(51, activation='relu')(h) h = layers.BatchNormalization(epsilon=1e-5, momentum=0.99)(h) h = layers.Dropout(rate=0.5)(h) outputs = layers.Dense(1, activation='sigmoid')(h) elif name == '100': # Resnet 50 inputs = layers.Input(shape=shape) model = ResNet50(img_input=inputs) learn_rate = 0.0005 resnet = True elif name == '101': # Resnet 18 inputs = layers.Input(shape=shape) model = ResNet18(img_input=inputs) learn_rate = 0.0005 resnet = True ############################################ ### Add more models above ############################################ ####### Compile model ###################### ############################################ if resnet: print("resnet model name", name) opt, loss_fn = optimizers.Adam( lr=learn_rate), 'sparse_categorical_crossentropy' else: ## For non resnet models if not custom_model: ### For non-custom models, use prototype function outputs, inputs = f_model_prototype(shape, **model_par_dict) learn_rate = model_par_dict['learn_rate'] model = models.Model(inputs, outputs) opt = optimizers.Adam(lr=learn_rate) model.compile(optimizer=opt, loss=loss_fn, metrics=metrics) #print("model %s"%name) return model
def Xception(include_top=True, input_shape=(224, 224, 3), pooling=None, classes=1000): img_input = layers.Input(shape=input_shape) channel_axis = -1 x = layers.Conv2D(32, (3, 3), strides=(2, 2), use_bias=False, name='block1_conv1')(img_input) x = layers.BatchNormalization(axis=channel_axis, name='block1_conv1_bn')(x) x = layers.Activation('relu', name='block1_conv1_act')(x) x = layers.Conv2D(64, (3, 3), use_bias=False, name='block1_conv2')(x) x = layers.BatchNormalization(axis=channel_axis, name='block1_conv2_bn')(x) x = layers.Activation('relu', name='block1_conv2_act')(x) residual = layers.Conv2D(128, (1, 1), strides=(2, 2), padding='same', use_bias=False)(x) residual = layers.BatchNormalization(axis=channel_axis)(residual) x = layers.SeparableConv2D(128, (3, 3), padding='same', use_bias=False, name='block2_sepconv1')(x) x = layers.BatchNormalization(axis=channel_axis, name='block2_sepconv1_bn')(x) x = layers.Activation('relu', name='block2_sepconv2_act')(x) x = layers.SeparableConv2D(128, (3, 3), padding='same', use_bias=False, name='block2_sepconv2')(x) x = layers.BatchNormalization(axis=channel_axis, name='block2_sepconv2_bn')(x) x = layers.MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='block2_pool')(x) x = layers.add([x, residual]) residual = layers.Conv2D(256, (1, 1), strides=(2, 2), padding='same', use_bias=False)(x) residual = layers.BatchNormalization(axis=channel_axis)(residual) x = layers.Activation('relu', name='block3_sepconv1_act')(x) x = layers.SeparableConv2D(256, (3, 3), padding='same', use_bias=False, name='block3_sepconv1')(x) x = layers.BatchNormalization(axis=channel_axis, name='block3_sepconv1_bn')(x) x = layers.Activation('relu', name='block3_sepconv2_act')(x) x = layers.SeparableConv2D(256, (3, 3), padding='same', use_bias=False, name='block3_sepconv2')(x) x = layers.BatchNormalization(axis=channel_axis, name='block3_sepconv2_bn')(x) x = layers.MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='block3_pool')(x) x = layers.add([x, residual]) residual = layers.Conv2D(728, (1, 1), strides=(2, 2), padding='same', use_bias=False)(x) residual = layers.BatchNormalization(axis=channel_axis)(residual) x = layers.Activation('relu', name='block4_sepconv1_act')(x) x = layers.SeparableConv2D(728, (3, 3), padding='same', use_bias=False, name='block4_sepconv1')(x) x = layers.BatchNormalization(axis=channel_axis, name='block4_sepconv1_bn')(x) x = layers.Activation('relu', name='block4_sepconv2_act')(x) x = layers.SeparableConv2D(728, (3, 3), padding='same', use_bias=False, name='block4_sepconv2')(x) x = layers.BatchNormalization(axis=channel_axis, name='block4_sepconv2_bn')(x) x = layers.MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='block4_pool')(x) x = layers.add([x, residual]) for i in range(8): residual = x prefix = 'block' + str(i + 5) x = layers.Activation('relu', name=prefix + '_sepconv1_act')(x) x = layers.SeparableConv2D(728, (3, 3), padding='same', use_bias=False, name=prefix + '_sepconv1')(x) x = layers.BatchNormalization(axis=channel_axis, name=prefix + '_sepconv1_bn')(x) x = layers.Activation('relu', name=prefix + '_sepconv2_act')(x) x = layers.SeparableConv2D(728, (3, 3), padding='same', use_bias=False, name=prefix + '_sepconv2')(x) x = layers.BatchNormalization(axis=channel_axis, name=prefix + '_sepconv2_bn')(x) x = layers.Activation('relu', name=prefix + '_sepconv3_act')(x) x = layers.SeparableConv2D(728, (3, 3), padding='same', use_bias=False, name=prefix + '_sepconv3')(x) x = layers.BatchNormalization(axis=channel_axis, name=prefix + '_sepconv3_bn')(x) x = layers.add([x, residual]) residual = layers.Conv2D(1024, (1, 1), strides=(2, 2), padding='same', use_bias=False)(x) residual = layers.BatchNormalization(axis=channel_axis)(residual) x = layers.Activation('relu', name='block13_sepconv1_act')(x) x = layers.SeparableConv2D(728, (3, 3), padding='same', use_bias=False, name='block13_sepconv1')(x) x = layers.BatchNormalization(axis=channel_axis, name='block13_sepconv1_bn')(x) x = layers.Activation('relu', name='block13_sepconv2_act')(x) x = layers.SeparableConv2D(1024, (3, 3), padding='same', use_bias=False, name='block13_sepconv2')(x) x = layers.BatchNormalization(axis=channel_axis, name='block13_sepconv2_bn')(x) x = layers.MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='block13_pool')(x) x = layers.add([x, residual]) x = layers.SeparableConv2D(1536, (3, 3), padding='same', use_bias=False, name='block14_sepconv1')(x) x = layers.BatchNormalization(axis=channel_axis, name='block14_sepconv1_bn')(x) x = layers.Activation('relu', name='block14_sepconv1_act')(x) x = layers.SeparableConv2D(2048, (3, 3), padding='same', use_bias=False, name='block14_sepconv2')(x) x = layers.BatchNormalization(axis=channel_axis, name='block14_sepconv2_bn')(x) x = layers.Activation('relu', name='block14_sepconv2_act')(x) if include_top: x = layers.GlobalAveragePooling2D(name='avg_pool')(x) x = layers.Dense(classes, activation='softmax', name='predictions')(x) else: if pooling == 'avg': x = layers.GlobalAveragePooling2D()(x) elif pooling == 'max': x = layers.GlobalMaxPooling2D()(x) # Create model. model = models.Model(img_input, x, name='xception') return model
conv2d_8_1 = layers.Conv2D(128, kernel_size=3, padding='same', activation='relu', name='conv2d_8_1', kernel_initializer='he_normal')(layers.UpSampling2D(size=(2, 2))(conv2d_7_3)) concatenate_8 = layers.Concatenate(axis=-1, name='concatenate_8')([conv2d_2_2, conv2d_8_1]) conv2d_8_2 = layers.Conv2D(128, kernel_size=3, padding='same', activation='relu', name='conv2d_8_2', kernel_initializer='he_normal')(concatenate_8) conv2d_8_3 = layers.Conv2D(128, kernel_size=3, padding='same', activation='relu', name='conv2d_8_3', kernel_initializer='he_normal')(conv2d_8_2) conv2d_9_1 = layers.Conv2D(64, kernel_size=3, padding='same', activation='relu', name='conv2d_9_1', kernel_initializer='he_normal')(layers.UpSampling2D(size=(2, 2))(conv2d_8_3)) concatenate_9 = layers.Concatenate(axis=-1, name='concatenate_9')([conv2d_1_2, conv2d_9_1]) conv2d_9_2 = layers.Conv2D(64, kernel_size=3, padding='same', activation='relu', name='conv2d_9_2', kernel_initializer='he_normal')(concatenate_9) conv2d_9_3 = layers.Conv2D(64, kernel_size=3, padding='same', activation='relu', name='conv2d_9_3', kernel_initializer='he_normal')(conv2d_9_2) conv2d_9_4 = layers.Conv2D(2, kernel_size=3, padding='same', activation='relu', name='conv2d_9_4', kernel_initializer='he_normal')(conv2d_9_3) conv2d_10 = layers.Conv2D(1, kernel_size=1, activation='sigmoid', name='conv2d_10')(conv2d_9_4) model = models.Model(inputs=input, outputs=conv2d_10) model.summary() model.compile(optimizer=optimizers.SGD(lr=1e-5, momentum=0.9, nesterov=True), loss='binary_crossentropy', metrics=['accuracy']) model.fit(x=images_new, y=results_new, batch_size=16, epochs=16) model.save_weights('unet_test_10.h5', save_format='h5') new = model.predict(images_new).reshape(N, 256, 256) plt.imshow(new[0]) plt.show()
def fit(self, X, Y=None, val_X=None, val_Y=None, num_epochs=300, batch_size=None, start_temp=10.0, min_temp=0.1, tryout_limit=1, class_weight=None): if Y is None: Y = X assert len(X) == len(Y) validation_data = None if val_X is not None and val_Y is not None: assert len(val_X) == len(val_Y) validation_data = (val_X, val_Y) if batch_size is None: batch_size = max(len(X) // 256, 16) steps_per_epoch = (len(X) + batch_size - 1) // batch_size for i in range(tryout_limit): K.set_learning_phase(1) inputs = layers.Input(shape=X.shape[1:]) alpha = np.exp( np.log(min_temp / start_temp) / (num_epochs * steps_per_epoch)) self.concrete_select = ConcreteSelect(self.K, start_temp, min_temp, alpha, name='concrete_select') selected_features = self.concrete_select(inputs) outputs = self.output_function(selected_features) self.model = models.Model(inputs, outputs) self.model.compile(loss=LinearSVC.loss_function( loss_function, class_weight), optimizer=optimizer_class(lr=initial_lr), metrics=[LinearSVC.accuracy]) print(self.model.summary()) stopper_callback = StopperCallback() hist = self.model.fit( X, Y, batch_size, num_epochs, verbose=0, callbacks=[stopper_callback], validation_data=validation_data) # , validation_freq = 10) if K.get_value( K.mean( K.max(K.softmax( self.concrete_select.logits, axis=-1)))) >= stopper_callback.mean_max_target: break num_epochs *= 2 self.probabilities = K.get_value( K.softmax(self.model.get_layer('concrete_select').logits)) self.indices = K.get_value( K.argmax(self.model.get_layer('concrete_select').logits)) return self
def ResNet50(include_top=True, weights='imagenet', input_tensor=None, input_shape=None, pooling=None, classes=1000, **kwargs): """Instantiates the ResNet50 architecture. Optionally loads weights pre-trained on ImageNet. Note that the data format convention used by the model is the one specified in your Keras config at `~/.keras/keras.json`. # Arguments include_top: whether to include the fully-connected layer at the top of the network. weights: one of `None` (random initialization), 'imagenet' (pre-training on ImageNet), or the path to the weights file to be loaded. input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model. input_shape: optional shape tuple, only to be specified if `include_top` is False (otherwise the input shape has to be `(224, 224, 3)` (with `channels_last` data format) or `(3, 224, 224)` (with `channels_first` data format). It should have exactly 3 inputs channels, and width and height should be no smaller than 32. E.g. `(200, 200, 3)` would be one valid value. pooling: Optional pooling mode for feature extraction when `include_top` is `False`. - `None` means that the output of the model will be the 4D tensor output of the last convolutional block. - `avg` means that global average pooling will be applied to the output of the last convolutional block, and thus the output of the model will be a 2D tensor. - `max` means that global max pooling will be applied. classes: optional number of classes to classify images into, only to be specified if `include_top` is True, and if no `weights` argument is specified. # Returns A Keras model instance. # Raises ValueError: in case of invalid argument for `weights`, or invalid input shape. """ if not (weights in {'imagenet', None} or os.path.exists(weights)): raise ValueError('The `weights` argument should be either ' '`None` (random initialization), `imagenet` ' '(pre-training on ImageNet), ' 'or the path to the weights file to be loaded.') if weights == 'imagenet' and include_top and classes != 1000: raise ValueError('If using `weights` as `"imagenet"` with `include_top`' ' as true, `classes` should be 1000') # Determine proper input shape input_shape = _obtain_input_shape(input_shape, default_size=224, min_size=32, data_format=backend.image_data_format(), require_flatten=include_top, weights=weights) if input_tensor is None: img_input = layers.Input(shape=input_shape) else: if not backend.is_keras_tensor(input_tensor): img_input = layers.Input(tensor=input_tensor, shape=input_shape) else: img_input = input_tensor if backend.image_data_format() == 'channels_last': bn_axis = 3 else: bn_axis = 1 x = layers.ZeroPadding2D(padding=(3, 3), name='conv1_pad')(img_input) x = layers.Conv2D(64, (7, 7), strides=(2, 2), padding='valid', kernel_initializer='he_normal', name='conv1')(x) x = layers.BatchNormalization(axis=bn_axis, name='bn_conv1')(x) x = layers.Activation('relu')(x) x = layers.ZeroPadding2D(padding=(1, 1), name='pool1_pad')(x) x = layers.MaxPooling2D((3, 3), strides=(2, 2))(x) x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1)) x = identity_block(x, 3, [64, 64, 256], stage=2, block='b') x = identity_block(x, 3, [64, 64, 256], stage=2, block='c') x = conv_block(x, 3, [128, 128, 512], stage=3, block='a') x = identity_block(x, 3, [128, 128, 512], stage=3, block='b') x = identity_block(x, 3, [128, 128, 512], stage=3, block='c') x = identity_block(x, 3, [128, 128, 512], stage=3, block='d') x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f') x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a') x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b') x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c') if include_top: x = layers.GlobalAveragePooling2D(name='avg_pool')(x) x = layers.Dense(classes, activation='softmax', name='fc1000')(x) else: if pooling == 'avg': x = layers.GlobalAveragePooling2D()(x) elif pooling == 'max': x = layers.GlobalMaxPooling2D()(x) else: warnings.warn('The output shape of `ResNet50(include_top=False)` ' 'has been changed since Keras 2.2.0.') # Ensure that the model takes into account # any potential predecessors of `input_tensor`. if input_tensor is not None: inputs = keras_utils.get_source_inputs(input_tensor) else: inputs = img_input # Create model. model = models.Model(inputs, x, name='resnet50') # Load weights. if weights == 'imagenet': if include_top: weights_path = keras_utils.get_file( 'resnet50_weights_tf_dim_ordering_tf_kernels.h5', WEIGHTS_PATH, cache_subdir='models', md5_hash='a7b3fe01876f51b976af0dea6bc144eb') else: weights_path = keras_utils.get_file( 'resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5', WEIGHTS_PATH_NO_TOP, cache_subdir='models', md5_hash='a268eb855778b3df3c7506639542a6af') model.load_weights(weights_path) if backend.backend() == 'theano': keras_utils.convert_all_kernels_in_model(model) elif weights is not None: model.load_weights(weights) return model
def model(nbr): entree = layers.Input(shape=(576, 560, 3), dtype='float32') result = layers.Conv2D(nbr, 3, activation='relu', padding='same')(entree) result = layers.BatchNormalization()(result) result = layers.Conv2D(nbr, 3, activation='relu', padding='same')(result) result1 = layers.BatchNormalization()(result) result = layers.MaxPool2D()(result1) result = layers.Conv2D(2 * nbr, 3, activation='relu', padding='same')(result) result = layers.BatchNormalization()(result) result = layers.Conv2D(2 * nbr, 3, activation='relu', padding='same')(result) result2 = layers.BatchNormalization()(result) result = layers.MaxPool2D()(result2) result = layers.Conv2D(4 * nbr, 3, activation='relu', padding='same')(result) result = layers.BatchNormalization()(result) result = layers.Conv2D(4 * nbr, 3, activation='relu', padding='same')(result) result3 = layers.BatchNormalization()(result) result = layers.MaxPool2D()(result3) result = layers.Conv2D(4 * nbr, 3, activation='relu', padding='same')(result) result = layers.BatchNormalization()(result) result = layers.Conv2D(4 * nbr, 3, activation='relu', padding='same')(result) result4 = layers.BatchNormalization()(result) result = layers.MaxPool2D()(result4) result = layers.Conv2D(8 * nbr, 3, activation='relu', padding='same')(result) result = layers.BatchNormalization()(result) result = layers.Conv2D(4 * nbr, 3, activation='relu', padding='same')(result) result = layers.BatchNormalization()(result) result = layers.UpSampling2D()(result) result = tf.concat([result, result4], axis=3) result = layers.Conv2D(8 * nbr, 3, activation='relu', padding='same')(result) result = layers.BatchNormalization()(result) result = layers.Conv2D(4 * nbr, 3, activation='relu', padding='same')(result) result = layers.BatchNormalization()(result) result = layers.UpSampling2D()(result) result = tf.concat([result, result3], axis=3) result = layers.Conv2D(4 * nbr, 3, activation='relu', padding='same')(result) result = layers.BatchNormalization()(result) result = layers.Conv2D(2 * nbr, 3, activation='relu', padding='same')(result) result = layers.BatchNormalization()(result) result = layers.UpSampling2D()(result) result = tf.concat([result, result2], axis=3) result = layers.Conv2D(2 * nbr, 3, activation='relu', padding='same')(result) result = layers.BatchNormalization()(result) result = layers.Conv2D(nbr, 3, activation='relu', padding='same')(result) result = layers.BatchNormalization()(result) result = layers.UpSampling2D()(result) result = tf.concat([result, result1], axis=3) result = layers.Conv2D(nbr, 3, activation='relu', padding='same')(result) result = layers.BatchNormalization()(result) result = layers.Conv2D(nbr, 3, activation='relu', padding='same')(result) result = layers.BatchNormalization()(result) sortie = layers.Conv2D(1, 1, activation='sigmoid', padding='same')(result) model = models.Model(inputs=entree, outputs=sortie) return model
X = np.array(X_l) y = np.array(y_l) print(X.shape, y.shape) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=100) # define model m_x = layers.Input(shape=X_train.shape[1:]) m_h = layers.LSTM(10)(m_x) m_y = layers.Dense(1)(m_h) m = models.Model(m_x, m_y) #m = keras.utils.multi_gpu_model(m, gpus=2) m.compile('adam', 'mse') m.summary() # fit #from tensorflow.keras import backend as K #K.tensorflow_backend._get_available_gpus() history = m.fit(X_train, y_train, epochs=500, validation_data=(X_test, y_test), verbose=0) # Then explore the hist object history.history #gives you a dictionary history.epoch #gives you a list
def build_mobilenet(input_shape=(None, None, 3), num_classes=1, depth_multiplier=1, alpha=1, name='mobile'): last_act = 'sigmoid' if num_classes == 1 else 'softmax' input = layers.Input(shape=input_shape, name=name + "_input") x = conv_block(input, 32, 3, 2, "same", name + "_Stem") x = depthwise_separable_block(x, 64, depth_multiplier=depth_multiplier, alpha=alpha, name=name + "_Block_1") x = depthwise_separable_block(x, 128, strides=2, depth_multiplier=depth_multiplier, alpha=alpha, name=name + "_Block_2") x = depthwise_separable_block(x, 128, depth_multiplier=depth_multiplier, alpha=alpha, name=name + "_Block_3") x = depthwise_separable_block(x, 256, strides=2, depth_multiplier=depth_multiplier, alpha=alpha, name=name + "_Block_4") x = depthwise_separable_block(x, 256, depth_multiplier=depth_multiplier, alpha=alpha, name=name + "_Block_5") x = depthwise_separable_block(x, 512, strides=2, depth_multiplier=depth_multiplier, alpha=alpha, name=name + "_Block_6") for i in range(5): x = depthwise_separable_block(x, 512, depth_multiplier=depth_multiplier, alpha=alpha, name=name + "_Block_%d" % (i + 1 + 6)) x = depthwise_separable_block(x, 1024, strides=2, depth_multiplier=depth_multiplier, alpha=alpha, name=name + "_Block_12") x = depthwise_separable_block(x, 1024, depth_multiplier=depth_multiplier, alpha=alpha, name=name + "_Block_13") x = layers.GlobalAveragePooling2D(name=name + "_GAP")(x) x = layers.Dense(num_classes, activation=last_act, name=name + "_Output")(x) return models.Model(input, x)
def EfficientNet(input_shape, block_args_list, global_params, include_top=True, pooling=None): batch_norm_momentum = global_params.batch_norm_momentum batch_norm_epsilon = global_params.batch_norm_epsilon if global_params.data_format == 'channels_first': channel_axis = 1 else: channel_axis = -1 # Stem part inputs = KL.Input(shape=input_shape) x = inputs x = KL.Conv2D( filters=round_filters(32, global_params), kernel_size=[3, 3], strides=[2, 2], kernel_initializer=ConvKernalInitializer(), padding='same', use_bias=False )(x) x = KL.BatchNormalization( axis=channel_axis, momentum=batch_norm_momentum, epsilon=batch_norm_epsilon )(x) x = Swish()(x) # Blocks part block_idx = 1 n_blocks = sum([block_args.num_repeat for block_args in block_args_list]) drop_rate = global_params.drop_connect_rate or 0 drop_rate_dx = drop_rate / n_blocks for block_args in block_args_list: assert block_args.num_repeat > 0 # Update block input and output filters based on depth multiplier. block_args = block_args._replace( input_filters=round_filters(block_args.input_filters, global_params), output_filters=round_filters(block_args.output_filters, global_params), num_repeat=round_repeats(block_args.num_repeat, global_params) ) # The first block needs to take care of stride and filter size increase. x = MBConvBlock(block_args, global_params, drop_connect_rate=drop_rate_dx * block_idx)(x) block_idx += 1 if block_args.num_repeat > 1: block_args = block_args._replace(input_filters=block_args.output_filters, strides=[1, 1]) for _ in xrange(block_args.num_repeat - 1): x = MBConvBlock(block_args, global_params, drop_connect_rate=drop_rate_dx * block_idx)(x) block_idx += 1 # Head part x = KL.Conv2D( filters=round_filters(1280, global_params), kernel_size=[1, 1], strides=[1, 1], kernel_initializer=ConvKernalInitializer(), padding='same', use_bias=False )(x) x = KL.BatchNormalization( axis=channel_axis, momentum=batch_norm_momentum, epsilon=batch_norm_epsilon )(x) x = Swish()(x) if include_top: x = KL.GlobalAveragePooling2D(data_format=global_params.data_format)(x) if global_params.dropout_rate > 0: x = KL.Dropout(global_params.dropout_rate)(x) x = KL.Dense(global_params.num_classes, kernel_initializer=DenseKernalInitializer())(x) x = KL.Activation('softmax')(x) else: if pooling == 'avg': x = KL.GlobalAveragePooling2D(data_format=global_params.data_format)(x) elif pooling == 'max': x = KL.GlobalMaxPooling2D(data_format=global_params.data_format)(x) outputs = x model = KM.Model(inputs, outputs) return model
def main(): dataset = load_dataset() for network_name in network_names: train_data = np.asarray(dataset['train']['data']) train_labels = dataset['train']['label'] num_classes = len(np.unique(train_labels)) test_data = np.asarray(dataset['test']['data']) test_labels = dataset['test']['label'] train_labels = to_categorical(train_labels, num_classes=num_classes) test_labels = to_categorical(test_labels, num_classes=num_classes) generator = dataset['generator'] generator_kwargs = { 'batch_size': batch_size } print('reps : ', reps) name = 'fashion_mnist_' + network_name + '_r_' + str(regularization) print(name) model_kwargs = { 'nclasses': num_classes, 'regularization': regularization } total_features = int(np.prod(train_data.shape[1:])) model_filename = directory + network_name + '_trained_model.h5' if not os.path.exists(model_filename) and warming_up: model = getattr(network_models, network_name)(input_shape=train_data.shape[1:], **model_kwargs) print('training_model') model.fit_generator( generator.flow(train_data, train_labels, **generator_kwargs), steps_per_epoch=train_data.shape[0] // batch_size, epochs=80, callbacks=[ callbacks.LearningRateScheduler(scheduler()) ], validation_data=(test_data, test_labels), validation_steps=test_data.shape[0] // batch_size, verbose=verbose ) if not os.path.isdir(directory): os.makedirs(directory) model.save(model_filename) del model K.clear_session() nfeats = [] accuracies = [] model_accuracies = [] times = [] for factor in [.05, .1, .25, .5]: n_features = int(total_features * factor) n_accuracies = [] n_model_accuracies = [] n_times = [] for r in range(reps): print('factor : ', factor, ' , rep : ', r) l2x_model = get_l2x_model(train_data.shape[1:], n_features) classifier = load_model(model_filename) if warming_up else getattr(network_models, network_name)(input_shape=train_data.shape[1:], **model_kwargs) classifier_input = layers.Multiply()([l2x_model.input, l2x_model.output]) output = classifier(classifier_input) model = models.Model(l2x_model.input, output) # optimizer = optimizers.SGD(lr=1e-1) # optimizers.adam(lr=1e-2) optimizer = optimizers.Adam(lr=1e-2) model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['acc']) model.classifier = classifier model.summary() start_time = time.time() model.fit_generator( generator.flow(train_data, train_labels, **generator_kwargs), steps_per_epoch=train_data.shape[0] // batch_size, epochs=80, callbacks=[ callbacks.LearningRateScheduler(scheduler(extra=0)), ], validation_data=(test_data, test_labels), validation_steps=test_data.shape[0] // batch_size, verbose=verbose ) n_model_accuracies.append(model.evaluate(test_data, test_labels, verbose=0)[-1]) scores = l2x_model.predict(train_data, verbose=0, batch_size=batch_size).reshape((-1, np.prod(train_data.shape[1:]))).sum(axis=0) pos = np.argsort(scores)[::-1][:n_features] n_times.append(time.time() - start_time) mask = np.zeros_like(scores) mask[pos] = 1. mask = mask.reshape(train_data.shape[1:]) del l2x_model, classifier, model K.clear_session() classifier = load_model(model_filename) if warming_up else getattr(network_models, network_name)( input_shape=train_data.shape[1:], **model_kwargs) classifier.fit_generator( generator.flow(mask * train_data, train_labels, **generator_kwargs), steps_per_epoch=train_data.shape[0] // batch_size, epochs=80, callbacks=[ callbacks.LearningRateScheduler(scheduler(extra=0)), ], validation_data=(mask * test_data, test_labels), validation_steps=test_data.shape[0] // batch_size, verbose=verbose ) n_accuracies.append(classifier.evaluate(mask * test_data, test_labels, verbose=0)[-1]) del classifier K.clear_session() print( 'n_features : ', n_features, ', acc : ', n_accuracies, ', time : ', n_times ) accuracies.append(n_accuracies) model_accuracies.append(n_model_accuracies) nfeats.append(n_features) times.append(n_times) output_filename = directory + network_name + '_l2x_results_warming_' + str(warming_up) + '.json' try: with open(output_filename) as outfile: info_data = json.load(outfile) except: info_data = {} if name not in info_data: info_data[name] = [] info_data[name].append( { 'regularization': regularization, 'reps': reps, 'classification': { 'n_features': nfeats, 'accuracy': accuracies, 'model_accuracy': model_accuracies, 'times': times } } ) with open(output_filename, 'w') as outfile: json.dump(info_data, outfile)
input_s = layers.Input(shape=(92, ), name='situation') # Specify base model img_shape = (230, 119, 3) base_model = tf.keras.applications.vgg19.VGG19(input_shape=img_shape, include_top=False, weights='imagenet') # Get earlier layers from VGG19 # Idea: Only want low level features # Will mess around with this low_name = "block" + str(NUM_BASE_LAYERS) + "_pool" base_model_low_out = base_model.get_layer(low_name).output base_model_low = models.Model(base_model.input, base_model_low_out) # Freeze all layers base_model_low.trainable = False base_low_out = base_model_low(input_v) # Testing to see if it matches lower level features #low_level_feature_model = models.Model(inputs=base_model.input, # outputs=base_model.get_layer('block3_pool').output) #low_level_output = low_level_feature_model.predict(X_v_train[1,].reshape(1,230,119,3)) # Flatten base model flatten = layers.Flatten()(base_low_out) # Add in situational data concat = layers.concatenate([flatten, input_s]) # Pass through dense layer
3. Keras Modeling ''' '''3.1 Functional -- More Flexible''' inputs = layers.Input(shape=(32, 32, 3)) # The shape of input tensor # The actual kernel_size is (3, 3, 3). They move in 2 dimensions. x = layers.Conv2D(32, kernel_size=(3, 3), activation="relu")(inputs) # x = layers.Conv3D(32, kernel_size=(3, 3, 3))(inputs) # It's WRONG! # x = layers.BatchNormalization()(x) x = layers.MaxPool2D(pool_size=(2, 2))(x) x = layers.Conv2D(64, kernel_size=(3, 3), activation="relu")(x) x = layers.MaxPool2D(pool_size=(2, 2))(x) x = layers.Dropout(rate=0.2)(x) x = layers.Flatten()(x) x = layers.Dense(32, activation='relu')(x) outputs = layers.Dense(1, activation='sigmoid')(x) model = models.Model(inputs=inputs, outputs=outputs) # '''3.2 Sequential -- Easier''' # model = tf.keras.models.Sequential( # [ # layers.Input(shape=(32, 32, 3)), # layers.Conv2D(32, kernel_size=(3, 3), activation="relu"), # layers.MaxPooling2D(pool_size=(2, 2)), # layers.Conv2D(64, kernel_size=(3, 3), activation="relu"), # layers.MaxPooling2D(pool_size=(2, 2)), # layers.Dropout(0.1), # layers.Flatten(), # layers.Dense(32, activation="relu"), # layers.Dense(1, activation="sigmoid"), # ] # )
def SCRUNet(): inp = layers.Input(shape=(256, 256, 3)) # Convolution layers to help learn some basic kernels pre_conv = layers.Conv2D(8, (3, 3), strides=(1, 1), padding='same', activation='relu')(inp) # Down sampling down_sample_0 = layers.Conv2D(16, kernel_size=(3, 3), strides=(2, 2), padding='same', activation='relu')(pre_conv) down_sample_1 = layers.Conv2D(24, kernel_size=(3, 3), strides=(2, 2), padding='same', activation='relu')(down_sample_0) down_sample_2 = layers.Conv2D(24, kernel_size=(3, 3), strides=(2, 2), padding='same', activation='relu')(down_sample_1) # Most compressed layer in the network latent = layers.Conv2D(32, kernel_size=(3, 3), strides=(2, 2), padding='same', activation='relu')(down_sample_2) # Upsampling with skip connections up_sample_0 = layers.Conv2DTranspose(24, (3, 3), strides=(2, 2), padding='same', activation='relu')(latent) skip_0 = layers.Concatenate()([up_sample_0, down_sample_2]) up_sample_1 = layers.Conv2DTranspose(24, (3, 3), strides=(2, 2), padding='same', activation='relu')(skip_0) skip_1 = layers.Concatenate()([up_sample_1, down_sample_1]) up_sample_2 = layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding='same', activation='relu')(skip_1) skip_2 = layers.Concatenate()([up_sample_2, down_sample_0]) up_sample_3 = layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding='same', activation='relu')(skip_2) skip_3 = layers.Concatenate()([up_sample_3, pre_conv]) # Post convolution layers post_conv = layers.Conv2DTranspose(8, (3, 3), strides=(1, 1), padding='same', activation='relu')(skip_3) output = layers.Conv2DTranspose(1, (1, 1), strides=(1, 1), padding='same', activation='sigmoid')(post_conv) model = models.Model(inputs=inp, outputs=output) # Bind the optimizer and the loss function to the model model.compile(optimizer='adam', loss=tf.keras.losses.BinaryCrossentropy(), metrics=[tf.keras.metrics.BinaryAccuracy()]) return model
if __name__ == "__main__": content_image = load_img(CONTENT_IMAGE) style_image = load_img(STYLE_IMAGE) imshow(content_image, 'Content Image') imshow(style_image, 'Style Image') vgg = VGG16(input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3), include_top=False, weights='imagenet') vgg.trainable = False print(vgg.summary()) # define content target content_model = models.Model( inputs=vgg.inputs, outputs=[vgg.get_layer(name).output for name in CONTENT_LAYERS]) print(f"Content layer : {CONTENT_LAYERS}") # define style target style_model = models.Model( inputs=vgg.inputs, outputs=[vgg.get_layer(name).output for name in STYLE_LAYERS]) print(f"Style layers : {STYLE_LAYERS}") # set targets content_targets = content_model(content_image * 255) style_targets = calc_style_outputs(style_image * 255) # set optimizer opt = tf.optimizers.Adam(learning_rate=LR)
def objetivo(params): tf.keras.backend.clear_session() batch_size = 32 args = read_args() print_selected_hyperparams(params) epochs = hyperparam_value("epochs", params) learning_rate = hyperparam_value("learning_rate", params) hidden_layer_1 = hyperparam_value("hidden_layer_1", params) hidden_layer_2 = hyperparam_value("hidden_layer_2", params) hidden_layer_3 = hyperparam_value("hidden_layer_3", params) hidden_layer_4 = hyperparam_value("hidden_layer_4", params) hidden_layer_5 = hyperparam_value("hidden_layer_5", params) drop_1 = hyperparam_value("dropout_1", params) drop_2 = hyperparam_value("dropout_2", params) dataset, dev_dataset, test_dataset = load_dataset(args.dataset_dir, batch_size) nlabels = dataset[TARGET_COL].unique().shape[0] # It's important to always use the same one-hot length one_hot_columns = { one_hot_col: max(dataset[one_hot_col].max(), dev_dataset[one_hot_col].max(), test_dataset[one_hot_col].max()) for one_hot_col in [ 'Gender', 'Color1', 'Type', 'MaturitySize', 'Sterilized', 'Color2', 'Color3', 'Vaccinated', 'Health' ] } embedded_columns = { embedded_col: dataset[embedded_col].max() + 1 for embedded_col in ['Breed1', 'Breed2'] } numeric_columns = ['Age', 'Fee'] # TODO (optional) put these three types of columns in the same dictionary with "column types" X_train, y_train = process_features(dataset, one_hot_columns, numeric_columns, embedded_columns) direct_features_input_shape = (X_train['direct_features'].shape[1], ) X_dev, y_dev = process_features(dev_dataset, one_hot_columns, numeric_columns, embedded_columns) # Create the tensorflow Dataset # TODO shuffle the train dataset! train_ds = tf.data.Dataset.from_tensor_slices( (X_train, y_train)).batch(batch_size).shuffle(buffer_size=800) dev_ds = tf.data.Dataset.from_tensor_slices( (X_dev, y_dev)).batch(batch_size) test_ds = tf.data.Dataset.from_tensor_slices( process_features(test_dataset, one_hot_columns, numeric_columns, embedded_columns, test=True)[0]).batch(batch_size) # TODO: Build the Keras model # model = .... # Add one input and one embedding for each embedded column embedding_layers = [] inputs = [] for embedded_col, max_value in embedded_columns.items(): input_layer = layers.Input(shape=(1, ), name=embedded_col) inputs.append(input_layer) # Define the embedding layer embedding_size = int(max_value / 4) embedding_layers.append( tf.squeeze(layers.Embedding( input_dim=max_value, output_dim=embedding_size)(input_layer), axis=-2)) print('Adding embedding of size {} for layer {}'.format( embedding_size, embedded_col)) # Add the direct features already calculated direct_features_input = layers.Input(shape=direct_features_input_shape, name='direct_features') inputs.append(direct_features_input) # Concatenate everything together features = layers.concatenate(embedding_layers + [direct_features_input]) # Modelo 4 bn = layers.BatchNormalization(momentum=0)(features) dense1 = layers.Dense(hidden_layer_1, activation='relu', use_bias=True)(bn) dropout_1 = layers.Dropout(drop_1)(dense1) dense2 = layers.Dense( hidden_layer_2, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.01))(dropout_1) dense3 = layers.Dense(hidden_layer_3, activation='relu')(dense2) dropout_2 = layers.Dropout(drop_2)(dense3) dense4 = layers.Dense(hidden_layer_4, activation='relu')(dropout_2) dense_ = layers.Dense(hidden_layer_5, activation='relu')(dense4) ## Modelo 2 # bn = layers.BatchNormalization(momentum=0)(features) # dense1 = layers.Dense(hidden_layer_1, activation='relu')(bn) # dropout_1 = layers.Dropout(drop_1)(dense1) #dense2 = layers.Dense(hidden_layer_2, activation='relu')(dropout_1) # dense_ = layers.Dense(hidden_layer_3, activation='relu')(dense2) ### Modelo 1 # dense1 = layers.Dense(hidden_layer_1, activation='relu')(features) # dense_ = layers.Dense(hidden_layer_2, activation='relu')(dense1) ### Modelo == 3: #dense1 = layers.Dense(hidden_layer_1,activation='relu')(features) #dense2 = layers.Dense(hidden_layer_2,activation='relu')(dense1) #dropout_1 = layers.Dropout(drop_1)(dense2) #dense_ = layers.Dense(hidden_layer_3,activation='relu')(dropout_1) output_layer = layers.Dense(nlabels, activation='softmax')(dense_) model = models.Model(inputs=inputs, outputs=output_layer) optimizador = tf.keras.optimizers.Adam(learning_rate=learning_rate) model.compile(loss='categorical_crossentropy', optimizer=optimizador, metrics=['accuracy']) # TODO: Fit the model mlflow.set_experiment(args.experiment_name) with mlflow.start_run(nested=True): # Log model hiperparameters first mlflow.log_param('hidden_layer_1', hidden_layer_1) mlflow.log_param('hidden_layer_2', hidden_layer_2) mlflow.log_param('hidden_layer_3', hidden_layer_3) mlflow.log_param('hidden_layer_4', hidden_layer_4) mlflow.log_param('hidden_layer_5', hidden_layer_5) mlflow.log_param('dropout_1', drop_1) mlflow.log_param('dropout_2', drop_2) mlflow.log_param('batch_size', batch_size) mlflow.log_param('learning_rate', learning_rate) mlflow.log_param('epochs', epochs) # Train history = model.fit(train_ds, epochs=epochs) loss, accuracy = 0, 0 loss, accuracy = model.evaluate(dev_ds) print("*** Dev loss: {} - accuracy: {}".format(loss, accuracy)) mlflow.log_metric('loss', loss) mlflow.log_metric('accuracy', accuracy) return (accuracy * (-1))
def ResNet(stack_fn, preact, use_bias, model_name='resnet', include_top=True, weights='imagenet', input_tensor=None, input_shape=None, pooling=None, classes=1000, actfun='relu', **kwargs): """Instantiates the ResNet, ResNetV2, and ResNeXt architecture. Optionally loads weights pre-trained on ImageNet. Note that the data format convention used by the model is the one specified in your Keras config at `~/.keras/keras.json`. # Arguments stack_fn: a function that returns output tensor for the stacked residual blocks. preact: whether to use pre-activation or not (True for ResNetV2, False for ResNet and ResNeXt). use_bias: whether to use biases for convolutional layers or not (True for ResNet and ResNetV2, False for ResNeXt). model_name: string, model name. include_top: whether to include the fully-connected layer at the top of the network. weights: one of `None` (random initialization), 'imagenet' (pre-training on ImageNet), or the path to the weights file to be loaded. input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model. input_shape: optional shape tuple, only to be specified if `include_top` is False (otherwise the input shape has to be `(224, 224, 3)` (with `channels_last` data format) or `(3, 224, 224)` (with `channels_first` data format). It should have exactly 3 inputs channels. pooling: optional pooling mode for feature extraction when `include_top` is `False`. - `None` means that the output of the model will be the 4D tensor output of the last convolutional layer. - `avg` means that global average pooling will be applied to the output of the last convolutional layer, and thus the output of the model will be a 2D tensor. - `max` means that global max pooling will be applied. classes: optional number of classes to classify images into, only to be specified if `include_top` is True, and if no `weights` argument is specified. # Returns A Keras model instance. # Raises ValueError: in case of invalid argument for `weights`, or invalid input shape. """ if not (weights in {'imagenet', None} or os.path.exists(weights)): raise ValueError('The `weights` argument should be either ' '`None` (random initialization), `imagenet` ' '(pre-training on ImageNet), ' 'or the path to the weights file to be loaded.') if weights == 'imagenet' and include_top and classes != 1000: raise ValueError( 'If using `weights` as `"imagenet"` with `include_top`' ' as true, `classes` should be 1000') # Determine proper input shape input_shape = _obtain_input_shape(input_shape, default_size=224, min_size=32, data_format=backend.image_data_format(), require_flatten=include_top, weights=weights) if input_tensor is None: img_input = layers.Input(shape=input_shape) else: if not backend.is_keras_tensor(input_tensor): img_input = layers.Input(tensor=input_tensor, shape=input_shape) else: img_input = input_tensor bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1 ## modified JJ # x = layers.ZeroPadding2D(padding=((3, 3), (3, 3)), name='conv1_pad')(img_input) # x = layers.Conv2D(64, 7, strides=2, use_bias=use_bias, name='conv1_conv')(x) x = layers.Conv2D(64, 7, strides=2, use_bias=use_bias, name='conv1_conv')(img_input) ## modified JJ if preact is False: x = batch_norm_agg(axis=bn_axis, epsilon=1.001e-5, name='conv1_bn')(x) x = layers.Activation(actfun, name='conv1_' + actfun)(x) ## modified JJ # x = layers.ZeroPadding2D(padding=((1, 1), (1, 1)), name='pool1_pad')(x) x = layers.MaxPooling2D(3, strides=2, name='pool1_pool')(x) x = stack_fn(x) if preact is True: x = batch_norm_agg(axis=bn_axis, epsilon=1.001e-5, name='post_bn')(x) x = layers.Activation(actfun, name='post_' + actfun)(x) if include_top: x = layers.GlobalAveragePooling2D(name='avg_pool')(x) x = layers.Dense(classes, activation='softmax', name='probs')(x) else: if pooling == 'avg': x = layers.GlobalAveragePooling2D(name='avg_pool')(x) elif pooling == 'max': x = layers.GlobalMaxPooling2D(name='max_pool')(x) # Ensure that the model takes into account # any potential predecessors of `input_tensor`. if input_tensor is not None: inputs = keras_utils.get_source_inputs(input_tensor) else: inputs = img_input # Create model. model = models.Model(inputs, x, name=model_name) # Load weights. if weights is not None: model.load_weights(weights) return model
padding='same')(pool1) #14 x 14 x 64 pool2 = MaxPooling2D(pool_size=(2, 2))(conv2) #7 x 7 x 64 conv3 = Conv2D(64, (3, 3), activation='relu', padding='same')(pool2) #7 x 7 x 128 (small and thick) #decoder conv4 = Conv2D(32, (3, 3), activation='relu', padding='same')(conv3) #7 x 7 x 128 up1 = layers.UpSampling2D((2, 2))(conv4) # 14 x 14 x 128 conv5 = Conv2D(16, (3, 3), activation='relu', padding='same')(up1) # 14 x 14 x 64 up2 = layers.UpSampling2D((2, 2))(conv5) # 28 x 28 x 64 decoded = Conv2D(1, (3, 3), activation='sigmoid', padding='same')(up2) # 28 x 28 x 1 model = models.Model(input=inputs, output=decoded) model.compile(loss='mean_squared_error', optimizer=optimizers.RMSprop()) with tf.device('/gpu:0'): model.fit(x_train, x_train, batch_size=100, epochs=10, shuffle=True, validation_data=(x_test, x_test)) prediction = model.predict(x_train) plt.imshow(np.reshape(prediction[0], (28, 28))) plt.show() plt.imshow(np.reshape(x_train[0], (28, 28))) plt.show()
n_latent = 100 input_shape = (n_latent, ) img_size = train_x.shape[1:] D = Build_Discriminator(input_shape=img_size, name="Discriminator") D.compile(optimizer=optimizers.RMSprop(), loss=losses.binary_crossentropy, metrics=['acc']) D.trainable = False G = Build_Generator(input_shape=input_shape, output_size=img_size, name="Generator") A = models.Model(inputs=G.input, outputs=D(G.output), name="GAN") A.compile(optimizer=optimizers.RMSprop(), loss=losses.binary_crossentropy) D.summary() G.summary() A.summary() # %% # Training Network epochs = 100 batch_size = 100 fake_label = np.zeros((batch_size, 1)) real_label = np.ones((batch_size, 1))
# merged model_final_merge = layers.Concatenate(axis=-1)( [model_l_conv1_mp_do, model_r_conv1_mp_do]) model_final_conv1 = layers.Conv2D(32, (3, 3), activation='relu', padding='same')(model_final_merge) model_final_conv1_mp = layers.MaxPooling2D(pool_size=(2, 2))(model_final_conv1) model_final_conv1_mp_do = layers.Dropout(0.2)(model_final_conv1_mp) model_final_flatten = layers.Flatten()(model_final_conv1_mp_do) model_final_dropout = layers.Dropout(0.2)( model_final_flatten) # dropout for regularization predicted_coords = layers.Dense(2, activation='tanh')( model_final_dropout ) # I have used the tanh activation because our outputs should be between -1 and 1 #------------------------------------------------------------------------------ # Create model #------------------------------------------------------------------------------ # create model = models.Model(inputs=[in1, in2], outputs=predicted_coords) # create # compile model.compile(loss=cust_mean_squared_error, optimizer=optimizers.Adam(), metrics=['cosine_proximity', 'mse', cos_distmet_2D]) # print summary model.summary() # save model.save(dir_wrfiles + '/DNN_' + modelname + '.h5') # save model
def QuantizedMobileNetV1(input_shape=None, alpha=1.0, depth_multiplier=1, dropout=1e-3, include_top=True, weights='imagenet', input_tensor=None, pooling=None, classes=1000, batch_size=None, nbits=16, fbits=8, BN_nbits=None, BN_fbits=None, rounding_method='nearest', quant_mode='hybrid', overflow_mode=False, stop_gradient=False, verbose=True, **kwargs): """Instantiates the MobileNet architecture. To load a MobileNet model via `load_model`, import the custom objects `relu6` and pass them to the `custom_objects` parameter. E.g. model = load_model('mobilenet.h5', custom_objects={ 'relu6': mobilenet.relu6}) # Arguments input_shape: optional shape tuple, only to be specified if `include_top` is False (otherwise the input shape has to be `(224, 224, 3)` (with `channels_last` data format) or (3, 224, 224) (with `channels_first` data format). It should have exactly 3 inputs channels, and width and height should be no smaller than 32. E.g. `(200, 200, 3)` would be one valid value. alpha: controls the width of the network. - If `alpha` < 1.0, proportionally decreases the number of filters in each layer. - If `alpha` > 1.0, proportionally increases the number of filters in each layer. - If `alpha` = 1, default number of filters from the paper are used at each layer. depth_multiplier: depth multiplier for depthwise convolution (also called the resolution multiplier) dropout: dropout rate include_top: whether to include the fully-connected layer at the top of the network. weights: one of `None` (random initialization), 'imagenet' (pre-training on ImageNet), or the path to the weights file to be loaded. input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model. pooling: Optional pooling mode for feature extraction when `include_top` is `False`. - `None` means that the output of the model will be the 4D tensor output of the last convolutional layer. - `avg` means that global average pooling will be applied to the output of the last convolutional layer, and thus the output of the model will be a 2D tensor. - `max` means that global max pooling will be applied. classes: optional number of classes to classify images into, only to be specified if `include_top` is True, and if no `weights` argument is specified. # Returns A Keras model instance. # Raises ValueError: in case of invalid argument for `weights`, or invalid input shape. RuntimeError: If attempting to run this model with a backend that does not support separable convolutions. """ if verbose: print('\nBuilding model : Quantized MobileNet V1') pbar=tqdm(total=16) if BN_nbits is None: BN_nbits=nbits if BN_fbits is None: BN_fbits=fbits layer_quantizer=build_layer_quantizer(nbits,fbits,rounding_method,overflow_mode,stop_gradient) layer_BN_quantizer=build_layer_quantizer(BN_nbits,BN_fbits,rounding_method,overflow_mode,stop_gradient) if verbose: pbar.set_postfix_str('Preparation') pbar.update() if not (weights in {'imagenet', None} or os.path.exists(weights)): raise ValueError('The `weights` argument should be either ' '`None` (random initialization), `imagenet` ' '(pre-training on ImageNet), ' 'or the path to the weights file to be loaded.') if weights == 'imagenet' and include_top and classes != 1000: raise ValueError('If using `weights` as ImageNet with `include_top` ' 'as true, `classes` should be 1000') # Determine proper input shape and default size. if input_shape is None: default_size = 224 else: if backend.image_data_format() == 'channels_first': rows = input_shape[1] cols = input_shape[2] else: rows = input_shape[0] cols = input_shape[1] if rows == cols and rows in [128, 160, 192, 224]: default_size = rows else: default_size = 224 input_shape = _obtain_input_shape(input_shape, default_size=default_size, min_size=32, data_format=backend.image_data_format(), require_flatten=include_top, weights=weights) if backend.image_data_format() == 'channels_last': row_axis, col_axis = (0, 1) else: row_axis, col_axis = (1, 2) rows = input_shape[row_axis] cols = input_shape[col_axis] if weights == 'imagenet': if depth_multiplier != 1: raise ValueError('If imagenet weights are being loaded, ' 'depth multiplier must be 1') if alpha not in [0.25, 0.50, 0.75, 1.0]: raise ValueError('If imagenet weights are being loaded, ' 'alpha can be one of' '`0.25`, `0.50`, `0.75` or `1.0` only.') if rows != cols or rows not in [128, 160, 192, 224]: if rows is None: rows = 224 warnings.warn('MobileNet shape is undefined.' ' Weights for input shape ' '(224, 224) will be loaded.') else: raise ValueError('If imagenet weights are being loaded, ' 'input must have a static square shape ' '(one of (128, 128), (160, 160), ' '(192, 192), or (224, 224)). ' 'Input shape provided = %s' % (input_shape,)) if backend.image_data_format() != 'channels_last': warnings.warn('The MobileNet family of models is only available ' 'for the input data format "channels_last" ' '(width, height, channels). ' 'However your settings specify the default ' 'data format "channels_first" (channels, width, height).' ' You should set `image_data_format="channels_last"` ' 'in your Keras config located at ~/.keras/keras.json. ' 'The model being returned right now will expect inputs ' 'to follow the "channels_last" data format.') backend.set_image_data_format('channels_last') old_data_format = 'channels_first' else: old_data_format = None if input_tensor is None: img_input = layers.Input(batch_shape=(batch_size,)+input_shape) else: if not backend.is_keras_tensor(input_tensor): img_input = layers.Input(tensor=input_tensor, batch_shape=(batch_size,)+input_shape) else: img_input = input_tensor if verbose: pbar.set_postfix_str('building standard conv block') x = _conv_block(img_input, 32, alpha, strides=(2, 2), layer_quantizer=layer_quantizer, layer_BN_quantizer=layer_BN_quantizer, quant_mode=quant_mode) if verbose: pbar.update() pbar.set_postfix_str('building depthwise conv block 1') x = _depthwise_conv_block(x, 64, alpha, depth_multiplier, block_id=1, layer_quantizer=layer_quantizer, layer_BN_quantizer=layer_BN_quantizer, quant_mode=quant_mode) if verbose: pbar.update() pbar.set_postfix_str('building depthwise conv block 2') x = _depthwise_conv_block(x, 128, alpha, depth_multiplier, strides=(2, 2), block_id=2, layer_quantizer=layer_quantizer, layer_BN_quantizer=layer_BN_quantizer, quant_mode=quant_mode) if verbose: pbar.update() pbar.set_postfix_str('building depthwise conv block 3') x = _depthwise_conv_block(x, 128, alpha, depth_multiplier, block_id=3, layer_quantizer=layer_quantizer, layer_BN_quantizer=layer_BN_quantizer, quant_mode=quant_mode) if verbose: pbar.update() pbar.set_postfix_str('building depthwise conv block 4') x = _depthwise_conv_block(x, 256, alpha, depth_multiplier, strides=(2, 2), block_id=4, layer_quantizer=layer_quantizer, layer_BN_quantizer=layer_BN_quantizer, quant_mode=quant_mode) if verbose: pbar.update() pbar.set_postfix_str('building depthwise conv block 5') x = _depthwise_conv_block(x, 256, alpha, depth_multiplier, block_id=5, layer_quantizer=layer_quantizer, layer_BN_quantizer=layer_BN_quantizer, quant_mode=quant_mode) if verbose: pbar.update() pbar.set_postfix_str('building depthwise conv block 6') x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, strides=(2, 2), block_id=6, layer_quantizer=layer_quantizer, layer_BN_quantizer=layer_BN_quantizer, quant_mode=quant_mode) if verbose: pbar.update() pbar.set_postfix_str('building depthwise conv block 7') x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=7, layer_quantizer=layer_quantizer, layer_BN_quantizer=layer_BN_quantizer, quant_mode=quant_mode) if verbose: pbar.update() pbar.set_postfix_str('building depthwise conv block 8') x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=8, layer_quantizer=layer_quantizer, layer_BN_quantizer=layer_BN_quantizer, quant_mode=quant_mode) if verbose: pbar.update() pbar.set_postfix_str('building depthwise conv block 9') x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=9, layer_quantizer=layer_quantizer, layer_BN_quantizer=layer_BN_quantizer, quant_mode=quant_mode) if verbose: pbar.update() pbar.set_postfix_str('building depthwise conv block 10') x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=10, layer_quantizer=layer_quantizer, layer_BN_quantizer=layer_BN_quantizer, quant_mode=quant_mode) if verbose: pbar.update() pbar.set_postfix_str('building depthwise conv block 11') x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=11, layer_quantizer=layer_quantizer, layer_BN_quantizer=layer_BN_quantizer, quant_mode=quant_mode) if verbose: pbar.update() pbar.set_postfix_str('building depthwise conv block 12') x = _depthwise_conv_block(x, 1024, alpha, depth_multiplier, strides=(2, 2), block_id=12, layer_quantizer=layer_quantizer, layer_BN_quantizer=layer_BN_quantizer, quant_mode=quant_mode) if verbose: pbar.update() pbar.set_postfix_str('building depthwise conv block 13') x = _depthwise_conv_block(x, 1024, alpha, depth_multiplier, block_id=13, layer_quantizer=layer_quantizer, layer_BN_quantizer=layer_BN_quantizer, quant_mode=quant_mode) if verbose: pbar.update() pbar.set_postfix_str('building output block') if include_top: if backend.image_data_format() == 'channels_first': shape = (int(1024 * alpha), 1, 1) else: shape = (1, 1, int(1024 * alpha)) x = layers.GlobalAveragePooling2D()(x) x = layers.Reshape(shape, name='reshape_1')(x) x = layers.Dropout(dropout, name='dropout')(x) x = QuantizedConv2D(classes, kernel_size=(1, 1), quantizers=layer_quantizer, padding='same', name='conv_preds', quant_mode=quant_mode, last_layer=True)(x) x = layers.Activation('softmax', name='act_softmax')(x) x = layers.Reshape((classes,), name='reshape_2')(x) else: if pooling == 'avg': x = layers.GlobalAveragePooling2D()(x) elif pooling == 'max': x = layers.GlobalMaxPooling2D()(x) if verbose: pbar.update() # Ensure that the model takes into account # any potential predecessors of `input_tensor`. if input_tensor is not None: inputs = keras_utils.get_source_inputs(input_tensor) else: inputs = img_input # Create model. model = models.Model(inputs, x, name='quantized_mobilenet_%0.2f_%s' % (alpha, rows)) if verbose: pbar.set_postfix_str('Model Built') pbar.close() # load weights if weights == 'imagenet': if backend.image_data_format() == 'channels_first': raise ValueError('Weights for "channels_first" format ' 'are not available.') if alpha == 1.0: alpha_text = '1_0' elif alpha == 0.75: alpha_text = '7_5' elif alpha == 0.50: alpha_text = '5_0' else: alpha_text = '2_5' if include_top: model_name = 'mobilenet_%s_%d_tf.h5' % (alpha_text, rows) weight_path = BASE_WEIGHT_PATH + model_name weights_path = keras_utils.get_file(model_name, weight_path, cache_subdir='models') else: model_name = 'mobilenet_%s_%d_tf_no_top.h5' % (alpha_text, rows) weight_path = BASE_WEIGHT_PATH + model_name weights_path = keras_utils.get_file(model_name, weight_path, cache_subdir='models') model.load_weights(weights_path) elif weights is not None: model.load_weights(weights) if old_data_format: backend.set_image_data_format(old_data_format) return model
def resnet_v1(input_shape, depth, num_classes=2): """ResNet Version 1 Model builder [a] Stacks of 2 x (3 x 3) Conv2D-BN-ReLU Last ReLU is after the shortcut connection. At the beginning of each stage, the feature map size is halved (downsampled) by a convolutional layer with strides=2, while the number of filters is doubled. Within each stage, the layers have the same number filters and the same number of filters. Features maps sizes: stage 0: 32x32, 16 stage 1: 16x16, 32 stage 2: 8x8, 64 The Number of parameters is approx the same as Table 6 of [a]: ResNet20 0.27M ResNet32 0.46M ResNet44 0.66M ResNet56 0.85M ResNet110 1.7M # Arguments input_shape (tensor): shape of input image tensor depth (int): number of core convolutional layers num_classes (int): number of classes (CIFAR10 has 10) # Returns model (Model): Keras model instance """ if (depth - 2) % 6 != 0: raise ValueError('depth should be 6n+2') # Start model definition. num_filters = 32 num_res_blocks = int((depth - 2) / 6) inputs = tf.keras.Input(shape=input_shape) x = resnet_layer(inputs, num_filters) # Instantiate teh stack of residual units for stack in range(3): for res_block in range(num_res_blocks): strides = 1 if stack > 0 and res_block == 0: # first layer but not first stack strides = 2 # downsample y = resnet_layer(x, num_filters, strides=strides) y = resnet_layer(y, num_filters, activation=None) if stack > 0 and res_block == 0: # first layer but not first stack # linear projection residual shortcut connection to match # change dims x = resnet_layer(x, num_filters, kernel_size=1, strides=strides, activation=None, batch_normalization=False) x = layers.add([x, y]) x = layers.Activation('relu')(x) num_filters *= 2 # Add classifier on top. # v1 does not use BN after last shortcut connection-ReLU ax = layers.GlobalAveragePooling2D()(x) #x = layers.AveragePooling2D()(x) ax = layers.Dense(num_filters // 8, activation='relu')(ax) ax = layers.Dense(num_filters // 2, activation='softmax')(ax) ax = layers.Reshape((1, 1, num_filters // 2))(ax) ax = layers.Multiply()([ax, x]) y = layers.Flatten()(ax) outputs = layers.Dense(num_classes, activation='softmax', kernel_initializer='he_normal')(y) # Instantiate model model = models.Model(inputs=inputs, outputs=outputs) return model
def unet_train(): height = 512 width = 512 path = 'D:/desktop/unet_datasets/' input_name = os.listdir(path + 'train_image') n = len(input_name) print(n) X_train, y_train = [], [] for i in range(n): print("正在读取第%d张图片" % i) img = cv2.imread(path + 'train_image/%d.png' % i) label = cv2.imread(path + 'train_label/%d.png' % i) X_train.append(img) y_train.append(label) X_train = np.array(X_train) y_train = np.array(y_train) def Conv2d_BN(x, nb_filter, kernel_size, strides=(1, 1), padding='same'): x = layers.Conv2D(nb_filter, kernel_size, strides=strides, padding=padding)(x) x = layers.BatchNormalization(axis=3)(x) x = layers.LeakyReLU(alpha=0.1)(x) return x def Conv2dT_BN(x, filters, kernel_size, strides=(2, 2), padding='same'): x = layers.Conv2DTranspose(filters, kernel_size, strides=strides, padding=padding)(x) x = layers.BatchNormalization(axis=3)(x) x = layers.LeakyReLU(alpha=0.1)(x) return x inpt = layers.Input(shape=(height, width, 3)) conv1 = Conv2d_BN(inpt, 8, (3, 3)) conv1 = Conv2d_BN(conv1, 8, (3, 3)) pool1 = layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')(conv1) conv2 = Conv2d_BN(pool1, 16, (3, 3)) conv2 = Conv2d_BN(conv2, 16, (3, 3)) pool2 = layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')(conv2) conv3 = Conv2d_BN(pool2, 32, (3, 3)) conv3 = Conv2d_BN(conv3, 32, (3, 3)) pool3 = layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')(conv3) conv4 = Conv2d_BN(pool3, 64, (3, 3)) conv4 = Conv2d_BN(conv4, 64, (3, 3)) pool4 = layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')(conv4) conv5 = Conv2d_BN(pool4, 128, (3, 3)) conv5 = layers.Dropout(0.5)(conv5) conv5 = Conv2d_BN(conv5, 128, (3, 3)) conv5 = layers.Dropout(0.5)(conv5) convt1 = Conv2dT_BN(conv5, 64, (3, 3)) concat1 = layers.concatenate([conv4, convt1], axis=3) concat1 = layers.Dropout(0.5)(concat1) conv6 = Conv2d_BN(concat1, 64, (3, 3)) conv6 = Conv2d_BN(conv6, 64, (3, 3)) convt2 = Conv2dT_BN(conv6, 32, (3, 3)) concat2 = layers.concatenate([conv3, convt2], axis=3) concat2 = layers.Dropout(0.5)(concat2) conv7 = Conv2d_BN(concat2, 32, (3, 3)) conv7 = Conv2d_BN(conv7, 32, (3, 3)) convt3 = Conv2dT_BN(conv7, 16, (3, 3)) concat3 = layers.concatenate([conv2, convt3], axis=3) concat3 = layers.Dropout(0.5)(concat3) conv8 = Conv2d_BN(concat3, 16, (3, 3)) conv8 = Conv2d_BN(conv8, 16, (3, 3)) convt4 = Conv2dT_BN(conv8, 8, (3, 3)) concat4 = layers.concatenate([conv1, convt4], axis=3) concat4 = layers.Dropout(0.5)(concat4) conv9 = Conv2d_BN(concat4, 8, (3, 3)) conv9 = Conv2d_BN(conv9, 8, (3, 3)) conv9 = layers.Dropout(0.5)(conv9) outpt = layers.Conv2D(filters=3, kernel_size=(1, 1), strides=(1, 1), padding='same', activation='relu')(conv9) model = models.Model(inpt, outpt) model.compile(optimizer='adam', loss='mean_squared_error', metrics=['accuracy']) model.summary() print("开始训练u-net") model.fit(X_train, y_train, epochs=400, batch_size=15) #batch_size不要过大,否则内存容易溢出 model.save('unet.h5') print('unet.h5保存成功!!!')
sum_mse = 'CNN_sum_K-32-64-64-128_KS-37-37-37-37_MP-12-22-22-32_DO-2-2-2-2-2_MSE_final.h5' con_ad = 'CNN_con_K-32-64-128-128_KS-35-35-35-35_MP-12-22-22-32_DO-2-2-2-2-2_AD_final.h5' sub_ad = 'CNN_sub_K-32-32-64-128_KS-37-37-37-37_MP-12-22-22-32_DO-2-2-2-2-2_AD_final.h5' sum_ad = 'CNN_sum_K-32-32-64-128_KS-35-35-35-35_MP-12-22-22-32_DO-2-2-2-2-2_AD_final.h5' # perform operation 6 times model = models.load_model(dir_models + '/' + con_mse, custom_objects={ 'GlorotUniform': glorot_uniform(), "cust_mean_squared_error": cust_mean_squared_error, "cos_distmet_2D_angular": cos_distmet_2D_angular }) # define a new model, input = testsound, output = intermediate representations for all layers in the previous model from the first # successive_outputs = [layer.output for i in model.layers[0:]] # this defines outputs for all layers successive_outputs = [model.layers[i].output for i in layers_interest] visualization_model = models.Model(inputs=model.input, outputs=successive_outputs) feature_maps = visualization_model.predict([sounds_l1, sounds_r1]) pickle.dump(feature_maps, open(dir_models + '/featuremaps_con_mse1.p', 'wb')) print('model con mse1 done') feature_maps = visualization_model.predict([sounds_l2, sounds_r2]) pickle.dump(feature_maps, open(dir_models + '/featuremaps_con_mse2.p', 'wb')) print('model con mse2 done') feature_maps = visualization_model.predict([sounds_l3, sounds_r3]) pickle.dump(feature_maps, open(dir_models + '/featuremaps_con_mse3.p', 'wb')) print('model con mse3 done') feature_maps = visualization_model.predict([sounds_l4, sounds_r4]) pickle.dump(feature_maps, open(dir_models + '/featuremaps_con_mse4.p', 'wb')) print('model con mse4 done')
print(os.path.join(file)) img = image.load_img("./data/color/" + file, target_size=(img_height, img_height)) # imgG = image.load_img("./data/gray/"+file, target_size=(img_height, img_height)) x = image.img_to_array(img) x = np.expand_dims(x, axis=0) # y = image.img_to_array(imgG) # y = np.expand_dims(y, axis=0) enc.fit(x, x) dec.fit(enc.predict(x),x) image_tensor = layers.Input(shape=(512, 512, 3)) encodig_output = Encoding(image_tensor) enc = models.Model(inputs=[image_tensor], outputs=[encodig_output]) decoding_output = Decoding(image_tensor) dec = models.Model(inputs=[image_tensor], outputs=[decoding_output]) enc.compile(optimizer='sgd', loss=lossFunction, metrics=['mse']) dec.compile(optimizer='sgd', loss=lossFunction, metrics=['mse']) # model.compile(optimizer='adam', # loss=tf.keras.losses.MeanSquaredError(), # metrics=['mse']) train(enc, dec) img_path = './data/color/christ_church_000317.jpg' img = image.load_img(img_path, target_size=(img_height, img_height))
def create_model( width=256, height=256, channels=3, activation='relu', padding='same', dropout_rate=0.2, conv_layer_opts=[(32, 8), (16, 4), (8, 2)], pooling_layer_opts=[4, 4, 2], mse_loss_factor=None, ): input_shape = (height, width, channels) encoder_input = tf.keras.Input(shape=input_shape, name='encoder_input') conv_pool_layers = [] for i, (conv_i, pool_i) in enumerate(zip(conv_layer_opts, pooling_layer_opts)): opts = { 'filters': conv_i[0], 'kernel_size': conv_i[1], 'activation': activation, 'padding': padding } l_conv = layers.Conv2D(name=f'conv_{i}', **opts) l_pool = layers.MaxPooling2D(pool_i, name=f'pool_{i}') if i == 0: x = l_conv(encoder_input) else: x = l_conv(x) x = l_pool(x) input_shape = transform_shape_conv2d(input_shape, l_conv) input_shape = transform_shape_maxpool2d(input_shape, l_pool) conv_pool_layers.append((l_conv, l_pool)) latent_shape = input_shape latent_dim = np.prod(input_shape) x = layers.Flatten()(x) mu = layers.Dense(latent_dim)(x) log_var = layers.Dense(latent_dim)(x) x = Sampling()([mu, log_var]) encoder_output = layers.Reshape(latent_shape)(x) decoder_input = tf.keras.Input(shape=latent_shape, name='decoder_input') decoder_input = encoder_output if dropout_rate > 0: dropout = tf.keras.layers.Dropout(dropout_rate) x = dropout(decoder_input) else: x = decoder_input for i, (l_conv, l_pool) in enumerate(conv_pool_layers): l_up = layers.UpSampling2D(size=l_pool.pool_size) cfg = l_conv.get_config() cfg['name'] = f'conv_trans_{i}' l_conv_trans = layers.Conv2DTranspose.from_config(cfg) x = l_up(x) x = l_conv_trans(x) conv_final = layers.Conv2D(3, 3, activation='sigmoid', padding=padding) decoder_output = conv_final(x) full_vae = models.Model(encoder_input, [decoder_output, mu, log_var]) reconstruction_loss = tf.math.reduce_mean( (encoder_input - decoder_output)**2, axis=[1, 2, 3]) * mse_loss_factor kl_loss = 1 + log_var - tf.math.square(mu) - tf.math.exp(log_var) kl_loss = tf.math.reduce_mean(kl_loss, axis=-1) kl_loss *= -0.5 full_vae.add_loss(K.mean(reconstruction_loss)) full_vae.add_loss(K.mean(kl_loss)) return full_vae