def get_vgg16(input_shape, final_activation='softmax', weights=None, classes=10, weight_decay=0.0005): if weights is not None: raise NotImplementedError('Weight load is not implemented.') model = Sequential() model.add( layers.Conv2D(64, (3, 3), padding='same', input_shape=input_shape, kernel_regularizer=regularizers.l2(weight_decay))) model.add(layers.Activation('relu')) model.add(layers.BatchNormalization()) model.add(layers.Dropout(0.3)) model.add( layers.Conv2D(64, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(layers.Activation('relu')) model.add(layers.BatchNormalization()) model.add(layers.MaxPooling2D(pool_size=(2, 2))) model.add( layers.Conv2D(128, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(layers.Activation('relu')) model.add(layers.BatchNormalization()) model.add(layers.Dropout(0.4)) model.add( layers.Conv2D(128, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(layers.Activation('relu')) model.add(layers.BatchNormalization()) model.add(layers.MaxPooling2D(pool_size=(2, 2))) model.add( layers.Conv2D(256, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(layers.Activation('relu')) model.add(layers.BatchNormalization()) model.add(layers.Dropout(0.4)) model.add( layers.Conv2D(256, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(layers.Activation('relu')) model.add(layers.BatchNormalization()) model.add(layers.Dropout(0.4)) model.add( layers.Conv2D(256, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(layers.Activation('relu')) model.add(layers.BatchNormalization()) model.add(layers.MaxPooling2D(pool_size=(2, 2))) model.add( layers.Conv2D(512, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(layers.Activation('relu')) model.add(layers.BatchNormalization()) model.add(layers.Dropout(0.4)) model.add( layers.Conv2D(512, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(layers.Activation('relu')) model.add(layers.BatchNormalization()) model.add(layers.Dropout(0.4)) model.add( layers.Conv2D(512, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(layers.Activation('relu')) model.add(layers.BatchNormalization()) model.add(layers.MaxPooling2D(pool_size=(2, 2))) model.add( layers.Conv2D(512, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(layers.Activation('relu')) model.add(layers.BatchNormalization()) model.add(layers.Dropout(0.4)) model.add( layers.Conv2D(512, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(layers.Activation('relu')) model.add(layers.BatchNormalization()) model.add(layers.Dropout(0.4)) model.add( layers.Conv2D(512, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(layers.Activation('relu')) model.add(layers.BatchNormalization()) model.add(layers.MaxPooling2D(pool_size=(2, 2))) model.add(layers.Dropout(0.5)) model.add(layers.Flatten()) model.add( layers.Dense(512, kernel_regularizer=regularizers.l2(weight_decay))) model.add(layers.Activation('relu')) model.add(layers.BatchNormalization()) model.add(layers.Dropout(0.5)) model.add(layers.Dense(classes)) model.add(layers.Activation(final_activation)) return model
def get_model(num_classes, model_name=MODEL_NAME, size_of_image=SIZE_OF_IMAGE): """ Returns a CNN model, or None if the parameter model is not recognised. Parameters: num_classes (int): Total number of different classes of Pokemon. model (string): One of the 3 models (custom, MobileNetV2, VGG19). Default: config.MODEL. size_of_image (int): Size of image after resizing. Default: config.SIZE_OF_IMAGE. Returns: model (tf.keras.Model): A CNN model. """ if model_name == 'custom': """ Custom CNN Model """ model = models.Sequential() model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(size_of_image, size_of_image, CHANNEL))) model.add(layers.MaxPooling2D((2, 2))) model.add(layers.Conv2D(64, (3, 3), activation='relu')) model.add(layers.MaxPooling2D((2, 2))) model.add(layers.Conv2D(128, (3, 3), activation='relu')) model.add(layers.MaxPooling2D((2, 2))) model.add(layers.Conv2D(256, (3, 3), activation='relu')) model.add(layers.MaxPooling2D((2, 2))) model.add(layers.Flatten()) model.add(layers.Dense(196, activation='relu')) # model.add(layers.Dropout(0.3)) model.add(layers.Dense(num_classes)) elif model_name == 'MobileNetV2': """ MobileNetV2 """ base_model = applications.MobileNetV2(input_shape=(size_of_image, size_of_image, CHANNEL), include_top=False, weights='imagenet') base_model.trainable = False global_average_layer = layers.GlobalAveragePooling2D() output_layer = layers.Dense(num_classes) model = tf.keras.Sequential([ base_model, global_average_layer, layers.Flatten(), layers.Dense(196, activation='relu'), layers.Dropout(0.3), output_layer ]) elif model_name == 'VGG19': """ VGG19 """ base_model = applications.VGG19(input_shape=(size_of_image, size_of_image, CHANNEL), include_top=False, weights='imagenet') base_model.trainable = False global_average_layer = layers.GlobalAveragePooling2D() output_layer = layers.Dense(num_classes) model = tf.keras.Sequential([ base_model, global_average_layer, layers.Flatten(), layers.Dense(196, activation='relu'), layers.Dropout(0.3), output_layer ]) else: print("ERROR: Cannot recognise model.") sys.exit(1) return model
import tensorflow as tf from tensorflow.keras import models, datasets, layers (train_images, train_labels), (test_images, test_labels) = datasets.mnist.load_data() train_images, test_images = train_images / 255.0, test_images / 255.0 model = models.Sequential([ layers.Flatten(input_shape=(28, 28)), layers.Dense(128, activation='relu'), layers.Dropout(.3), layers.Dense(128, activation='relu'), layers.Dropout(.3), layers.Dense(10, activation='softmax') ]) model.summary() model.compile( optimizer='Adam', loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy']) model.fit(train_images, train_labels, epochs=10, validation_split=.2) test_loss, test_acc = model.evaluate(test_images, test_labels) print('Test Accuracy: ', test_acc) model.save('./models')
def EfficientNet(width_coefficient, depth_coefficient, default_resolution, dropout_rate=0.2, drop_connect_rate=0.2, depth_divisor=8, blocks_args=DEFAULT_BLOCKS_ARGS, model_name='EfficientNet', include_top=False, weights='imagenet', input_tensor=None, input_shape=None, pooling=None, classes=1000): """Instantiates the EfficientNet architecture using given scaling coefficients. Optionally loads weights pre-trained on ImageNet. Note that the data format convention used by the model is the one specified in your Keras config at `~/.keras/keras.json`. # Arguments width_coefficient: float, scaling coefficient for network width. depth_coefficient: float, scaling coefficient for network depth. default_resolution: int, default input image size. dropout_rate: float, dropout rate before final classifier layer. drop_connect_rate: float, dropout rate at skip connections. depth_divisor: int. blocks_args: A list of BlockArgs to construct block modules. model_name: string, model name. include_top: whether to include the fully-connected layer at the top of the network. weights: one of `None` (random initialization), 'imagenet' (pre-training on ImageNet), or the path to the weights file to be loaded. input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model. input_shape: optional shape tuple, only to be specified if `include_top` is False. It should have exactly 3 inputs channels. pooling: optional pooling mode for feature extraction when `include_top` is `False`. - `None` means that the output of the model will be the 4D tensor output of the last convolutional layer. - `avg` means that global average pooling will be applied to the output of the last convolutional layer, and thus the output of the model will be a 2D tensor. - `max` means that global max pooling will be applied. classes: optional number of classes to classify images into, only to be specified if `include_top` is True, and if no `weights` argument is specified. # Returns A Keras model instance. # Raises ValueError: in case of invalid argument for `weights`, or invalid input shape. """ if not (weights in {'imagenet', 'noisy-student', None} or os.path.exists(weights)): raise ValueError('The `weights` argument should be either ' '`None` (random initialization), `imagenet` ' '(pre-training on ImageNet), ' 'or the path to the weights file to be loaded.') if weights == 'imagenet' and include_top and classes != 1000: raise ValueError( 'If using `weights` as `"imagenet"` with `include_top`' ' as true, `classes` should be 1000') # Determine proper input shape input_shape = _obtain_input_shape(input_shape, default_size=default_resolution, min_size=32, data_format=backend.image_data_format(), require_flatten=include_top, weights=weights) if input_tensor is None: img_input = layers.Input(shape=input_shape) else: if backend.backend() == 'tensorflow': from tensorflow.python.keras.backend import is_keras_tensor else: is_keras_tensor = backend.is_keras_tensor if not is_keras_tensor(input_tensor): img_input = layers.Input(tensor=input_tensor, shape=input_shape) else: img_input = input_tensor bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1 activation = get_swish() # Build stem x = img_input x = layers.Conv2D(round_filters(32, width_coefficient, depth_divisor), 3, strides=(2, 2), padding='same', use_bias=False, kernel_initializer=CONV_KERNEL_INITIALIZER, name='stem_conv')(x) x = layers.BatchNormalization(axis=bn_axis, name='stem_bn')(x) x = layers.Activation(activation, name='stem_activation')(x) # Build blocks num_blocks_total = sum(block_args.num_repeat for block_args in blocks_args) block_num = 0 for idx, block_args in enumerate(blocks_args): assert block_args.num_repeat > 0 # Update block input and output filters based on depth multiplier. block_args = block_args._replace( input_filters=round_filters(block_args.input_filters, width_coefficient, depth_divisor), output_filters=round_filters(block_args.output_filters, width_coefficient, depth_divisor), num_repeat=round_repeats(block_args.num_repeat, depth_coefficient)) # The first block needs to take care of stride and filter size increase. drop_rate = drop_connect_rate * float(block_num) / num_blocks_total x = mb_conv_block(x, block_args, activation=activation, drop_rate=drop_rate, prefix='block{}a_'.format(idx + 1)) block_num += 1 if block_args.num_repeat > 1: block_args = block_args._replace( input_filters=block_args.output_filters, strides=[1, 1]) for bidx in xrange(block_args.num_repeat - 1): drop_rate = drop_connect_rate * float( block_num) / num_blocks_total block_prefix = 'block{}{}_'.format( idx + 1, string.ascii_lowercase[bidx + 1]) x = mb_conv_block(x, block_args, activation=activation, drop_rate=drop_rate, prefix=block_prefix) block_num += 1 # Build top x = layers.Conv2D(round_filters(1280, width_coefficient, depth_divisor), 1, padding='same', use_bias=False, kernel_initializer=CONV_KERNEL_INITIALIZER, name='top_conv')(x) x = layers.BatchNormalization(axis=bn_axis, name='top_bn')(x) x = layers.Activation(activation, name='top_activation')(x) if include_top: x = layers.GlobalAveragePooling2D(name='avg_pool')(x) if dropout_rate and dropout_rate > 0: x = layers.Dropout(dropout_rate, name='top_dropout')(x) x = layers.Dense(classes, activation='softmax', kernel_initializer=DENSE_KERNEL_INITIALIZER, name='probs')(x) else: if pooling == 'avg': x = layers.GlobalAveragePooling2D(name='avg_pool')(x) elif pooling == 'max': x = layers.GlobalMaxPooling2D(name='max_pool')(x) # Ensure that the model takes into account # any potential predecessors of `input_tensor`. if input_tensor is not None: inputs = utils.get_source_inputs(input_tensor) else: inputs = img_input # Create model. model = models.Model(inputs, x, name=model_name) # Load weights. if weights == 'imagenet': if include_top: file_name = model_name + '_AutoAugment.h5' else: file_name = model_name + '_AutoAugment_NoTop.h5' model.load_weights(os.path.join(os.path.dirname(__file__), file_name)) elif weights == 'noisy-student': if include_top: file_name = "{}_{}.h5".format(model_name, 'NoisyStudent') else: file_name = "{}_{}_NoTop.h5".format(model_name, 'NoisyStudent') model.load_weights(os.path.join(os.path.dirname(__file__), file_name)) elif weights is not None: model.load_weights(weights) return model
layers.BatchNormalization(), layers.MaxPooling1D(pool_size=5, strides=2, padding="same"), layers.Conv1D(256, kernel_size=5, strides=1, padding="same", activation="relu"), layers.BatchNormalization(), layers.MaxPooling1D(pool_size=5, strides=2, padding="same"), layers.Conv1D(128, kernel_size=5, strides=1, padding="same", activation="relu"), layers.MaxPooling1D(pool_size=5, strides=2, padding="same"), layers.Dropout(0.2), layers.Conv1D(64, kernel_size=5, strides=1, padding="same", activation="relu"), layers.MaxPooling1D(pool_size=5, strides=2, padding="same"), layers.Flatten(), layers.Dense(units=32, activation="relu"), layers.Dropout(0.3), layers.Dense(units=8, activation="softmax"), ]) model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
def __init__(self, channels, init_block_channels, final_block_channels, final_block_groups, dilations, dropout_rate=0.2, in_channels=3, in_size=(224, 224), classes=1000, data_format="channels_last", **kwargs): super(ESPNetv2, self).__init__(**kwargs) self.in_size = in_size self.classes = classes x0_channels = in_channels self.features = DualPathSequential(return_two=False, first_ordinals=0, last_ordinals=2, name="features") self.features.add( ESPInitBlock(in_channels=in_channels, out_channels=init_block_channels, data_format=data_format, name="init_block")) in_channels = init_block_channels for i, channels_per_stage in enumerate(channels): stage = DualPathSequential(name="stage{}_".format(i + 1)) for j, out_channels in enumerate(channels_per_stage): if j == 0: unit = DownsampleBlock(in_channels=in_channels, out_channels=out_channels, x0_channels=x0_channels, dilations=dilations[i][j], data_format=data_format, name="unit{}".format(j + 1)) else: unit = ESPBlock(in_channels=in_channels, out_channels=out_channels, strides=1, dilations=dilations[i][j], data_format=data_format, name="unit{}".format(j + 1)) stage.add(unit) in_channels = out_channels self.features.add(stage) self.features.add( ESPFinalBlock(in_channels=in_channels, out_channels=final_block_channels, final_groups=final_block_groups, data_format=data_format, name="final_block")) in_channels = final_block_channels self.features.add( nn.AveragePooling2D(pool_size=7, strides=1, data_format=data_format, name="final_pool")) self.output1 = tf.keras.Sequential(name="output1") self.output1.add(nn.Dropout(rate=dropout_rate, name="dropout")) self.output1.add( nn.Dense(units=classes, input_dim=in_channels, name="fc"))
"""## Create the discriminator It maps a 64x64 image to a binary classification score. """ discriminator = keras.Sequential( [ keras.Input(shape=(64, 64, 3)), layers.Conv2D(64, kernel_size=4, strides=2, padding="same"), layers.LeakyReLU(alpha=0.2), layers.Conv2D(128, kernel_size=4, strides=2, padding="same"), layers.LeakyReLU(alpha=0.2), layers.Conv2D(128, kernel_size=4, strides=2, padding="same"), layers.LeakyReLU(alpha=0.2), layers.Flatten(), layers.Dropout(0.2), layers.Dense(1, activation="sigmoid"), ], name="discriminator", ) discriminator.summary() """## Create the generator It mirrors the discriminator, replacing `Conv2D` layers with `Conv2DTranspose` layers. """ latent_dim = 128 generator = keras.Sequential( [ keras.Input(shape=(latent_dim, )),
#load the weights in model Model.load_weights(name+"_weights.best.h5") #plot the model plot(history) #real testing of model test_loss, test_acc = Model.evaluate(test_X, test_Y, verbose=2) print("\n\nAccuracy(",name,") on Test images ==> ",test_acc*100,"% \n") print("\n\nExact 1-off Accuracy(",name,") on Test images ==> ",get_exact1off_acc(test_X,test_Y,Model),"% \n") model_age = models.Sequential() #CONV1 model_age.add(layers.Conv2D(96, (7, 7),strides=(4,4), input_shape=(227, 227, 3))) model_age.add(layers.Activation('relu')) model_age.add(layers.BatchNormalization()) model_age.add(layers.MaxPooling2D(pool_size=(2, 2))) model_age.add(layers.Dropout(0.6)) #CONV2 model_age.add(layers.Conv2D(256, (5, 5))) model_age.add(layers.Activation('relu')) model_age.add(layers.BatchNormalization()) model_age.add(layers.MaxPooling2D(pool_size=(2, 2))) model_age.add(layers.Dropout(0.6)) #CONV3 model_age.add(layers.Conv2D(384, (3, 3))) model_age.add(layers.Activation('relu')) model_age.add(layers.BatchNormalization()) model_age.add(layers.MaxPooling2D(pool_size=(2, 2))) model_age.add(layers.Dropout(0.6)) #CONV4 model_age.add(layers.Conv2D(256, (3, 3))) model_age.add(layers.Activation('relu'))
def autoencoder( trainingData, validationData, encodingDimension, trainingLabels=None, validationLabels=None, activation='relu', batchSize=256, dropoutRate=0.3, epochs=200, hiddenDimension=None, learningRate=0.0002, lossFunction='mse', metrics=['mae', 'mse'], testingData=None, validationSteps=3, regularizationRate=0, ): inputDimension = len(trainingData[0]) inputData = layers.Input(shape=(inputDimension, )) encoded = layers.Dropout(dropoutRate)(inputData) if hiddenDimension is not None: encoded = layers.Dense( hiddenDimension, activation=activation, kernel_regularizer=regularizers.l2(regularizationRate), )(encoded) encoded = layers.Dense( encodingDimension, activation=activation, kernel_regularizer=regularizers.l2(regularizationRate), )(inputData) decoded = encoded if hiddenDimension is not None: decoded = layers.Dense(hiddenDimension, activation=activation)(decoded) decoded = layers.Dense(inputDimension, activation=activation)(decoded) autoencoder = Model(inputData, decoded) encodedInput = layers.Input(shape=(encodingDimension, )) decoderLayer = (autoencoder.layers[-1](encodedInput) if hiddenDimension is None else autoencoder.layers[-1]( autoencoder.layers[-2](encodedInput))) encoder = Model(inputData, encoded) decoder = Model(encodedInput, decoderLayer) autoencoder.compile( optimizer=optimizers.Nadam(learningRate), loss=lossFunction, metrics=metrics, ) if trainingLabels is None: trainingLabels = trainingData if validationLabels is None: validationLabels = validationData autoencoder.fit( trainingData, trainingLabels, batch_size=batchSize, epochs=epochs, shuffle=True, validation_data=(validationData, validationLabels), validation_steps=validationSteps, ) return autoencoder, encoder, decoder
def build_model(local_bm_hyperparameters, local_bm_settings): model_built = 0 time_steps_days = int(local_bm_hyperparameters['time_steps_days']) epochs = int(local_bm_hyperparameters['epochs']) batch_size = int(local_bm_hyperparameters['batch_size']) workers = int(local_bm_hyperparameters['workers']) optimizer_function = local_bm_hyperparameters['optimizer'] optimizer_learning_rate = local_bm_hyperparameters['learning_rate'] if optimizer_function == 'adam': optimizer_function = optimizers.Adam(optimizer_learning_rate) elif optimizer_function == 'ftrl': optimizer_function = optimizers.Ftrl(optimizer_learning_rate) losses_list = [] loss_1 = local_bm_hyperparameters['loss_1'] loss_2 = local_bm_hyperparameters['loss_2'] loss_3 = local_bm_hyperparameters['loss_3'] union_settings_losses = [loss_1, loss_2, loss_3] if 'mape' in union_settings_losses: losses_list.append(losses.MeanAbsolutePercentageError()) if 'mse' in union_settings_losses: losses_list.append(losses.MeanSquaredError()) if 'mae' in union_settings_losses: losses_list.append(losses.MeanAbsoluteError()) if 'm_mape' in union_settings_losses: losses_list.append(modified_mape()) if 'customized_loss_function' in union_settings_losses: losses_list.append(customized_loss()) metrics_list = [] metric1 = local_bm_hyperparameters['metrics1'] metric2 = local_bm_hyperparameters['metrics2'] union_settings_metrics = [metric1, metric2] if 'rmse' in union_settings_metrics: metrics_list.append(metrics.RootMeanSquaredError()) if 'mse' in union_settings_metrics: metrics_list.append(metrics.MeanSquaredError()) if 'mae' in union_settings_metrics: metrics_list.append(metrics.MeanAbsoluteError()) if 'mape' in union_settings_metrics: metrics_list.append(metrics.MeanAbsolutePercentageError()) l1 = local_bm_hyperparameters['l1'] l2 = local_bm_hyperparameters['l2'] if local_bm_hyperparameters['regularizers_l1_l2'] == 'True': activation_regularizer = regularizers.l1_l2(l1=l1, l2=l2) else: activation_regularizer = None nof_features_for_training = local_bm_hyperparameters[ 'nof_features_for_training'] # creating model forecaster_in_block = tf.keras.Sequential() print('creating the ANN model...') # first layer (DENSE) if local_bm_hyperparameters['units_layer_1'] > 0: forecaster_in_block.add( layers.Dense( units=local_bm_hyperparameters['units_layer_1'], activation=local_bm_hyperparameters['activation_1'], input_shape=(local_bm_hyperparameters['time_steps_days'], nof_features_for_training), activity_regularizer=activation_regularizer)) forecaster_in_block.add( layers.Dropout( rate=float(local_bm_hyperparameters['dropout_layer_1']))) # second LSTM layer if local_bm_hyperparameters[ 'units_layer_2'] > 0 and local_bm_hyperparameters[ 'units_layer_1'] > 0: forecaster_in_block.add( layers.Bidirectional( layers.LSTM( units=local_bm_hyperparameters['units_layer_2'], activation=local_bm_hyperparameters['activation_2'], activity_regularizer=activation_regularizer, dropout=float(local_bm_hyperparameters['dropout_layer_2']), return_sequences=False))) forecaster_in_block.add( RepeatVector(local_bm_hyperparameters['repeat_vector'])) # third LSTM layer if local_bm_hyperparameters['units_layer_3'] > 0: forecaster_in_block.add( layers.Bidirectional( layers.LSTM( units=local_bm_hyperparameters['units_layer_3'], activation=local_bm_hyperparameters['activation_3'], activity_regularizer=activation_regularizer, dropout=float(local_bm_hyperparameters['dropout_layer_3']), return_sequences=True))) if local_bm_hyperparameters['units_layer_4'] == 0: forecaster_in_block.add( RepeatVector(local_bm_hyperparameters['repeat_vector'])) # fourth layer (DENSE) if local_bm_hyperparameters['units_layer_4'] > 0: forecaster_in_block.add( layers.Dense(units=local_bm_hyperparameters['units_layer_4'], activation=local_bm_hyperparameters['activation_4'], activity_regularizer=activation_regularizer)) forecaster_in_block.add( layers.Dropout( rate=float(local_bm_hyperparameters['dropout_layer_4']))) # final layer forecaster_in_block.add( TimeDistributed(layers.Dense(units=nof_features_for_training))) forecaster_in_block.save(''.join( [local_bm_settings['models_path'], 'in_block_NN_model_structure_']), save_format='tf') forecast_horizon_days = local_bm_settings['forecast_horizon_days'] forecaster_in_block.build(input_shape=(1, forecast_horizon_days + 1, nof_features_for_training)) forecaster_in_block.compile(optimizer=optimizer_function, loss=losses_list, metrics=metrics_list) forecaster_in_block_json = forecaster_in_block.to_json() with open( ''.join([ local_bm_settings['models_path'], 'freq_acc_forecaster_in_block.json' ]), 'w') as json_file: json_file.write(forecaster_in_block_json) json_file.close() print( 'build_model function finish (model structure saved in json and ts formats)' ) return True, model_built
def _efficientnet(input_shape, blocks_args_list, global_params): batch_norm_momentum = global_params.batch_norm_momentum batch_norm_epsilon = global_params.batch_norm_epsilon # Stem part model_input = layers.Input(shape=input_shape) x = layers.Conv2D( filters=round_filters(32, global_params), kernel_size=[3, 3], strides=[2, 2], kernel_initializer=conv_kernel_initializer, padding='same', use_bias=False, name='stem_conv2d' )(model_input) x = layers.BatchNormalization( momentum=batch_norm_momentum, epsilon=batch_norm_epsilon, name='stem_batch_norm' )(x) x = Swish(name='stem_swish')(x) # Blocks part idx = 0 drop_rate = global_params.drop_connect_rate n_blocks = sum([blocks_args.num_repeat for blocks_args in blocks_args_list]) drop_rate_dx = drop_rate / n_blocks for blocks_args in blocks_args_list: assert blocks_args.num_repeat > 0 # Update block input and output filters based on depth multiplier. blocks_args = blocks_args._replace( input_filters=round_filters(blocks_args.input_filters, global_params), output_filters=round_filters(blocks_args.output_filters, global_params), num_repeat=round_repeats(blocks_args.num_repeat, global_params) ) # The first block needs to take care of stride and filter size increase. x = MBConvBlock(blocks_args, global_params, idx, drop_connect_rate=drop_rate_dx * idx)(x) idx += 1 if blocks_args.num_repeat > 1: blocks_args = blocks_args._replace(input_filters=blocks_args.output_filters, strides=[1, 1]) for _ in range(blocks_args.num_repeat - 1): x = MBConvBlock(blocks_args, global_params, idx, drop_connect_rate=drop_rate_dx * idx)(x) idx += 1 # Head part x = layers.Conv2D( filters=round_filters(1280, global_params), kernel_size=[1, 1], strides=[1, 1], kernel_initializer=conv_kernel_initializer, padding='same', use_bias=False, name='head_conv2d' )(x) x = layers.BatchNormalization( momentum=batch_norm_momentum, epsilon=batch_norm_epsilon, name='head_batch_norm' )(x) x = Swish(name='head_swish')(x) x = layers.GlobalAveragePooling2D(name='global_average_pooling2d')(x) if global_params.dropout_rate > 0: x = layers.Dropout(global_params.dropout_rate)(x) x = layers.Dense( global_params.num_classes, kernel_initializer=dense_kernel_initializer, activation='softmax', name='head_dense' )(x) model = models.Model(model_input, x) return model
def __call__(self, inputs): if self.dropout is not None and self.sdropout is not None: raise ValueError("Can only use either dropout or spatial " "dropout, not both") dim_layers = _get_dimensional_layers(self.conv_dim) convolution_nd = dim_layers["convolution"] s_dropout_nd = dim_layers["s_dropout"] if self.kernel_l2_reg is not None: kernel_reg = ks.regularizers.l2(self.kernel_l2_reg) else: kernel_reg = None if self.batchnorm: use_bias = False else: use_bias = True block_layers = list() if isinstance(self.padding, str): padding = self.padding else: block_layers.append(dim_layers["zero_padding"](self.padding)) padding = "valid" block_layers.append(convolution_nd( filters=self.filters, kernel_size=self.kernel_size, strides=self.strides, padding=padding, kernel_initializer=self.kernel_initializer, use_bias=use_bias, kernel_regularizer=kernel_reg, dilation_rate=self.dilation_rate) ) if self.batchnorm: channel_axis = 1 if ks.backend.image_data_format() == "channels_first" else -1 block_layers.append(layers.BatchNormalization(axis=channel_axis)) if self.activation is not None: block_layers.append(layers.Activation(self.activation)) if self.pool_type == "global_average_pooling": pooling_nd = dim_layers[self.pool_type] block_layers.append(pooling_nd()) elif self.pool_size is not None: pooling_nd = dim_layers[self.pool_type] block_layers.append(pooling_nd( pool_size=self.pool_size, padding=self.pool_padding)) if self.dropout is not None: block_layers.append(layers.Dropout(self.dropout)) elif self.sdropout is not None: block_layers.append(s_dropout_nd(self.sdropout)) x = inputs for block_layer in block_layers: if self.time_distributed: x = layers.TimeDistributed(block_layer)(x) else: x = block_layer(x) return x
## Build a model We will build a convolutional reconstruction autoencoder model. The model will take input of shape `(batch_size, sequence_length, num_features)` and return output of the same shape. In this case, `sequence_length` is 288 and `num_features` is 1. """ model = keras.Sequential([ layers.Input(shape=(x_train.shape[1], x_train.shape[2])), layers.Conv1D(filters=32, kernel_size=7, padding="same", strides=2, activation="relu"), layers.Dropout(rate=0.2), layers.Conv1D(filters=16, kernel_size=7, padding="same", strides=2, activation="relu"), layers.Conv1DTranspose(filters=16, kernel_size=7, padding="same", strides=2, activation="relu"), layers.Dropout(rate=0.2), layers.Conv1DTranspose(filters=32, kernel_size=7, padding="same", strides=2,
# coding: utf-8 # In[ ]: import matplotlib.pyplot as plt import gym import numpy as np from tensorflow.keras import models, layers, optimizers env = gym.make('CartPole-v0') STATE_DIM, ACTION_DIM = 4, 2 model = models.Sequential([ layers.Dense(100, input_dim=STATE_DIM, activation='relu'), layers.Dropout(0.1), layers.Dense(ACTION_DIM, activation="softmax") ]) model.compile(loss='mean_squared_error', optimizer=optimizers.Adam(0.001)) def choose_action(s): """预测动作""" prob = model.predict(np.array([s]))[0] return np.random.choice(len(prob), p=prob) def discount_rewards(rewards, gamma=0.95): """计算衰减reward的累加期望,并中心化和标准化处理""" prior = 0
x = layers.Conv2D(32, 3, activation="relu")(inputs) x = layers.Conv2D(64, 3, activation="relu")(x) block_1_output = layers.MaxPooling2D(3)(x) x = layers.Conv2D(64, 3, activation="relu", padding="same")(block_1_output) x = layers.Conv2D(64, 3, activation="relu", padding="same")(x) block_2_output = layers.add([x, block_1_output]) x = layers.Conv2D(64, 3, activation="relu", padding="same")(block_2_output) x = layers.Conv2D(64, 3, activation="relu", padding="same")(x) block_3_output = layers.add([x, block_2_output]) x = layers.Conv2D(64, 3, activation="relu")(block_3_output) x = layers.GlobalAveragePooling2D()(x) x = layers.Dense(256, activation="relu")(x) x = layers.Dropout(0.5)(x) outputs = layers.Dense(10)(x) model = keras.Model(inputs, outputs, name="toy_resnet") model.summary() """ Plot the model: """ keras.utils.plot_model(model, "mini_resnet.png", show_shapes=True) """ Now train the model: """
def Generator(): # Encoder: input = tf.keras.Input(shape=[L_node, W_node, Channel]) # input: (None, 256, 256, 1) e1 = layers.Conv2D(64, 5, 2, 'same')(input) # e1: (None, 128, 128, 64) e2 = layers.LeakyReLU(0.2)(e1) # e2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(e2) e2_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(e2) e2_1 = layers.BatchNormalization()(e2_1) e2_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(e2) e2_2 = layers.BatchNormalization()(e2_2) e2_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(e2) e2_3 = layers.BatchNormalization()(e2_3) e2 = layers.concatenate([e2_1, e2_2, e2_3], 3) e2 = layers.Conv2D(128, 5, 2, 'same')(e2) # Downsampling e2 = layers.BatchNormalization()(e2) # e2: (None, 64, 64, 128) e3 = layers.LeakyReLU(0.2)(e2) # e3 = layers.Conv2D(128, 3, 1, 'same', activation='relu')(e3) e3_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(e3) e3_1 = layers.BatchNormalization()(e3_1) e3_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(e3) e3_2 = layers.BatchNormalization()(e3_2) e3_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(e3) e3_3 = layers.BatchNormalization()(e3_3) e3 = layers.concatenate([e3_1, e3_2, e3_3], 3) e3 = layers.Conv2D(256, 5, 2, 'same')(e3) # Downsampling e3 = layers.BatchNormalization()(e3) # e3: (None, 32, 32, 256) e4 = layers.LeakyReLU(0.2)(e3) # e4 = layers.Conv2D(256, 3, 1, 'same', activation='relu')(e4) e4_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(e4) e4_1 = layers.BatchNormalization()(e4_1) e4_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(e4) e4_2 = layers.BatchNormalization()(e4_2) e4_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(e4) e4_3 = layers.BatchNormalization()(e4_3) e4 = layers.concatenate([e4_1, e4_2, e4_3], 3) e4 = layers.Conv2D(512, 5, 2, 'same')(e4) # Downsampling e4 = layers.BatchNormalization()(e4) # e4: (None, 16, 16, 512) e5 = layers.LeakyReLU(0.2)(e4) # e5 = layers.Conv2D(512, 3, 1, 'same', activation='relu')(e5) e5_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(e5) e5_1 = layers.BatchNormalization()(e5_1) e5_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(e5) e5_2 = layers.BatchNormalization()(e5_2) e5_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(e5) e5_3 = layers.BatchNormalization()(e5_3) e5 = layers.concatenate([e5_1, e5_2, e5_3], 3) e5 = layers.Conv2D(512, 5, 2, 'same')(e5) # Downsampling e5 = layers.BatchNormalization()(e5) # e5: (None, 8, 8, 512) e6 = layers.LeakyReLU(0.2)(e5) # e6 = layers.Conv2D(512, 3, 1, 'same', activation='relu')(e6) e6_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(e6) e6_1 = layers.BatchNormalization()(e6_1) e6_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(e6) e6_2 = layers.BatchNormalization()(e6_2) e6_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(e6) e6_3 = layers.BatchNormalization()(e6_3) e6 = layers.concatenate([e6_1, e6_2, e6_3], 3) e6 = layers.Conv2D(512, 5, 2, 'same')(e6) # Dowmsampling e6 = layers.BatchNormalization()(e6) # e6: (None, 4, 4, 512) e7 = layers.LeakyReLU(0.2)(e6) # e7 = layers.Conv2D(512, 3, 1, 'same', activation='relu')(e7) e7_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(e7) e7_1 = layers.BatchNormalization()(e7_1) e7_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(e7) e7_2 = layers.BatchNormalization()(e7_2) e7_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(e7) e7_3 = layers.BatchNormalization()(e7_3) e7 = layers.concatenate([e7_1, e7_2, e7_3], 3) e7 = layers.Conv2D(512, 5, 2, 'same')(e7) # Downsampling e7 = layers.BatchNormalization()(e7) # e7: (None, 2, 2, 512) e8 = layers.LeakyReLU(0.2)(e7) # e8 = layers.Conv2D(512, 3, 1, 'same', activation='relu')(e8) e8_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(e8) e8_1 = layers.BatchNormalization()(e8_1) e8_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(e8) e8_2 = layers.BatchNormalization()(e8_2) e8_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(e8) e8_3 = layers.BatchNormalization()(e8_3) e8 = layers.concatenate([e8_1, e8_2, e8_3], 3) e8 = layers.Conv2D(512, 5, 2, 'same')(e8) # Downsampling e8 = layers.BatchNormalization()(e8) # e8: (None, 1, 1, 512) # Decoder: d1 = layers.Activation('relu')(e8) # d1 = layers.Conv2D(512, 3, 1, 'same', activation='relu')(d1) d1_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(d1) d1_1 = layers.BatchNormalization()(d1_1) d1_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(d1) d1_2 = layers.BatchNormalization()(d1_2) d1_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(d1) d1_3 = layers.BatchNormalization()(d1_3) d1 = layers.concatenate([d1_1, d1_2, d1_3], 3) d1 = layers.Conv2DTranspose(512, 5, 2, 'same')(d1) # Upsampling d1 = layers.BatchNormalization()(d1) d1 = layers.Dropout(0.5)(d1) d1 = layers.concatenate([d1, e7], 3) # d1: (None, 2, 2, 512*2) d2 = layers.Activation('relu')(d1) # d2 = layers.Conv2D(512, 3, 1, 'same', activation='relu')(d2) d2_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(d2) d2_1 = layers.BatchNormalization()(d2_1) d2_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(d2) d2_2 = layers.BatchNormalization()(d2_2) d2_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(d2) d2_3 = layers.BatchNormalization()(d2_3) d2 = layers.concatenate([d2_1, d2_2, d2_3], 3) d2 = layers.Conv2DTranspose(512, 5, 2, 'same')(d2) # Upsampling d2 = layers.BatchNormalization()(d2) d2 = layers.Dropout(0.5)(d2) d2 = layers.concatenate([d2, e6], 3) # d2: (None, 4, 4, 512*2) d3 = layers.Activation('relu')(d2) # d3 = layers.Conv2D(512, 3, 1, 'same', activation='relu')(d3) d3_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(d3) d3_1 = layers.BatchNormalization()(d3_1) d3_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(d3) d3_2 = layers.BatchNormalization()(d3_2) d3_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(d3) d3_3 = layers.BatchNormalization()(d3_3) d3 = layers.concatenate([d3_1, d3_2, d3_3], 3) d3 = layers.Conv2DTranspose(512, 5, 2, 'same')(d3) # Upsampling d3 = layers.BatchNormalization()(d3) d3 = layers.Dropout(0.5)(d3) d3 = layers.concatenate([d3, e5], 3) # d3: (None, 8, 8, 512*2) d4 = layers.Activation('relu')(d3) # d4 = layers.Conv2D(512, 3, 1, 'same', activation='relu')(d4) d4_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(d4) d4_1 = layers.BatchNormalization()(d4_1) d4_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(d4) d4_2 = layers.BatchNormalization()(d4_2) d4_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(d4) d4_3 = layers.BatchNormalization()(d4_3) d4 = layers.concatenate([d4_1, d4_2, d4_3], 3) d4 = layers.Conv2DTranspose(512, 5, 2, 'same')(d4) # Upsampling d4 = layers.BatchNormalization()(d4) d4 = layers.Dropout(0.5)(d4) d4 = layers.concatenate([d4, e4], 3) # d4: (None, 16, 16, 512*2) d5 = layers.Activation('relu')(d4) # d5 = layers.Conv2D(512, 3, 1, 'same', activation='relu')(d5) d5_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(d5) d5_1 = layers.BatchNormalization()(d5_1) d5_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(d5) d5_2 = layers.BatchNormalization()(d5_2) d5_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(d5) d5_3 = layers.BatchNormalization()(d5_3) d5 = layers.concatenate([d5_1, d5_2, d5_3], 3) d5 = layers.Conv2DTranspose(256, 5, 2, 'same')(d5) # Upsampling d5 = layers.BatchNormalization()(d5) d5 = layers.Dropout(0.5)(d5) d5 = layers.concatenate([d5, e3], 3) # d5: (None, 32, 32, 256*2) d6 = layers.Activation('relu')(d5) # d6 = layers.Conv2D(256, 3, 1, 'same', activation='relu')(d6) d6_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(d6) d6_1 = layers.BatchNormalization()(d6_1) d6_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(d6) d6_2 = layers.BatchNormalization()(d6_2) d6_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(d6) d6_3 = layers.BatchNormalization()(d6_3) d6 = layers.concatenate([d6_1, d6_2, d6_3], 3) d6 = layers.Conv2DTranspose(128, 5, 2, 'same')(d6) # Upsampling d6 = layers.BatchNormalization()(d6) d6 = layers.Dropout(0.5)(d6) d6 = layers.concatenate([d6, e2], 3) # d6: (None, 64, 64, 128*2) d7 = layers.Activation('relu')(d6) # d7 = layers.Conv2D(128, 3, 1, 'same', activation='relu')(d7) d7_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(d7) d7_1 = layers.BatchNormalization()(d7_1) d7_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(d7) d7_2 = layers.BatchNormalization()(d7_2) d7_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(d7) d7_3 = layers.BatchNormalization()(d7_3) d7 = layers.concatenate([d7_1, d7_2, d7_3], 3) d7 = layers.Conv2DTranspose(64, 5, 2, 'same')(d7) # Upsampling d7 = layers.BatchNormalization()(d7) d7 = layers.Dropout(0.5)(d7) d7 = layers.concatenate([d7, e1], 3) # d7: (None, 128, 128, 64*2) d8 = layers.Activation('relu')(d7) # d8 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(d8) d8_1 = layers.Conv2D(128, 1, 1, 'same', activation='relu')(d8) d8_1 = layers.BatchNormalization()(d8_1) d8_2 = layers.Conv2D(64, 3, 1, 'same', activation='relu')(d8) d8_2 = layers.BatchNormalization()(d8_2) d8_3 = layers.Conv2D(32, 5, 1, 'same', activation='relu')(d8) d8_3 = layers.BatchNormalization()(d8_3) d8 = layers.concatenate([d8_1, d8_2, d8_3], 3) d8 = layers.Conv2DTranspose(1, 5, 2, 'same')(d8) # Upsampling d8 = layers.Activation('tanh')(d8) # d8: (None, 256, 256, 1) output = d8 model = tf.keras.Model(input, output) return model
""" model = models.Sequential() #64 filtre, relu aktivasyon fonksiyonu, 3x3'lük filtreler. Padding yok ve stride = 1 model.add( layers.Conv2D(64, (3, 3), activation='relu', input_shape=(200, 200, 3))) model.add(layers.MaxPooling2D((2, 2))) model.add(layers.Conv2D(64, (3, 3), activation='relu')) model.add(layers.MaxPooling2D((2, 2))) model.add(layers.Conv2D(128, (3, 3), activation='relu')) model.add(layers.MaxPooling2D((2, 2))) model.add(layers.Conv2D(128, (3, 3), activation='relu')) model.add(layers.MaxPooling2D((2, 2))) model.add(layers.Flatten()) model.add(layers.Dense(512, activation='relu')) #overfittingin önlenmesi için model.add(layers.Dropout(0.2)) model.add(layers.Dense(512, activation='relu')) model.add(layers.Dropout(0.2)) #Çıkışın alındığı yer. 10 sınıf olduğu için 10 adet nöron model.add(layers.Dense(10, activation='sigmoid')) #Bu fonksiyon modeling özetini çıkartır model.summary() #loss ve optimizasyon fonksiyonlari model kurulur. model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) #model burada eğitilir. Eğitim setinin %10'nunu validation için ayırdıö loss = model.fit(x_train,
early_stop = EarlyStopping('val_loss', patience=30) reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1, patience=10, verbose=1) callbacks = [early_stop, reduce_lr] ################################################## base_model = Xception(input_shape=(299, 299, 3), include_top=False, weights='imagenet') base_model.trainable = False inputs = Input(shape=(299, 299, 3)) x = base_model(inputs, training=False) x = layers.GlobalAveragePooling2D()(x) x = layers.Dropout(0.2)(x) outputs = layers.Dense(150, activation='softmax')(x) model = Model(inputs, outputs) model.summary() # Train train_datagen = ImageDataGenerator( # featurewise_center=True, # featurewise_std_normalization=True, rotation_range=20, width_shift_range=0.2, height_shift_range=0.2, rescale=1. / 255, zoom_range=0.2,
def __init__(self, channels, init_block_channels, stem1_blocks_channels, in_channels=3, in_size=(331, 331), classes=1000, data_format="channels_last", **kwargs): super(PNASNet, self).__init__(**kwargs) self.in_size = in_size self.classes = classes self.data_format = data_format self.features = nasnet_dual_path_sequential(return_two=False, first_ordinals=2, last_ordinals=2, name="features") self.features.add( NASNetInitBlock(in_channels=in_channels, out_channels=init_block_channels, data_format=data_format, name="init_block")) in_channels = init_block_channels self.features.add( Stem1Unit(in_channels=in_channels, out_channels=stem1_blocks_channels, data_format=data_format, name="stem1_unit")) prev_in_channels = in_channels in_channels = stem1_blocks_channels for i, channels_per_stage in enumerate(channels): stage = nasnet_dual_path_sequential(name="stage{}".format(i + 1)) for j, out_channels in enumerate(channels_per_stage): reduction = (j == 0) extra_padding = (j == 0) and (i not in [0, 2]) match_prev_layer_dimensions = (j == 1) or ((j == 0) and (i == 0)) stage.add( PnasUnit( in_channels=in_channels, prev_in_channels=prev_in_channels, out_channels=out_channels, reduction=reduction, extra_padding=extra_padding, match_prev_layer_dimensions=match_prev_layer_dimensions, data_format=data_format, name="unit{}".format(j + 1))) prev_in_channels = in_channels in_channels = out_channels self.features.add(stage) self.features.add(nn.ReLU(name="activ")) self.features.add( nn.AveragePooling2D(pool_size=11, strides=1, data_format=data_format, name="final_pool")) self.output1 = tf.keras.Sequential(name="output1") self.output1.add(nn.Dropout(rate=0.5, name="dropout")) self.output1.add( nn.Dense(units=classes, input_dim=in_channels, name="fc"))
(['shots_attempted_total'], [SimpleImputer(), StandardScaler()])], df_out=False) Z_train = mapper.fit_transform(X_train) Z_test = mapper.transform(X_test) # build keras model for goals from tensorflow.keras import layers from tensorflow.keras import Input from tensorflow.keras import models from tensorflow.keras.models import Model model = models.Sequential() model.add(layers.Dense(30, activation='selu', input_dim=Z_train.shape[1])) model.add(layers.Dense(10, activation='selu')) model.add(layers.Dropout(0.25)) model.add(layers.Dense(5, activation='selu')) model.add(layers.Dense(1)) model.compile(optimizer='Nadam', loss='mse', metrics=['mae']) model.fit(Z_train, y_train_goals, epochs=100, batch_size=10) import pydot from tensorflow.keras.utils import plot_model plot_model(model) y_hat_test_goals = model.predict(Z_test) r2_score(y_test_goals, y_hat_test_goals) mean_absolute_error(y_test_goals, y_hat_test_goals) plt.figure(figsize=(8, 8)) plt.scatter(y_test_goals, y_hat_test_goals, alpha=1 / 4)
def dense_block(units, dropout_rate, inputs): x = layers.Dense(units, activation="relu")(inputs) x = layers.BatchNormalization()(x) outputs = layers.Dropout(dropout_rate)(x) return outputs
strides=(2, 1))(outputs) if BATCHNORM: outputs = tf.keras.layers.BatchNormalization()(outputs) outputs = layers.Activation(tf.nn.relu)(outputs) outputs = layers.Conv2D(NUM_CHANNELS, KERNEL_SIZE, padding="same", strides=(2, 2))(outputs) if BATCHNORM: outputs = tf.keras.layers.BatchNormalization()(outputs) outputs = layers.Activation(tf.nn.relu)(outputs) outputs = layers.Flatten()(outputs) outputs_flat = layers.Flatten()(inputs) outputs_flat = layers.Dense(1512)(outputs_flat) if BATCHNORM: outputs = tf.keras.layers.BatchNormalization()(outputs) outputs = layers.Activation(tf.nn.relu)(outputs) outputs_flat = layers.Dropout(DROPOUT_RATE)(outputs_flat) outputs_flat = layers.Dense(1256)(outputs_flat) if BATCHNORM: outputs = tf.keras.layers.BatchNormalization()(outputs) outputs = layers.Activation(tf.nn.relu)(outputs) outputs_flat = layers.Dropout(DROPOUT_RATE)(outputs_flat) outputs = layers.Concatenate(axis=1)([outputs_flat, outputs]) outputs = layers.Dense(1024)(outputs) outputs = layers.Dropout(DROPOUT_RATE)(outputs) if BATCHNORM: outputs = tf.keras.layers.BatchNormalization()(outputs) outputs = layers.Activation(tf.nn.relu)(outputs) outputs = layers.Dense(512)(outputs) outputs = layers.Dropout(DROPOUT_RATE)(outputs) if BATCHNORM: outputs = tf.keras.layers.BatchNormalization()(outputs) outputs = layers.Activation(tf.nn.relu)(outputs) pi = layers.Dense(pi_output_count, name="policy")(outputs)
dimension = 128 vocabulary_size = len(tk.word_index) inputs = layers.Input(shape=(input_size, )) embedding_layer = TokenAndPositionEmbedding(input_size, vocabulary_size + 1, embed_dim) X = embedding_layer(inputs) transformer_block = TransformerBlock(embed_dim, num_heads, ff_dim) #transformer_block2 = TransformerBlock(embed_dim, num_heads, ff_dim) X = transformer_block(X) #X = transformer_block2(X) X = layers.GlobalAvgPool1D()(X) #X = Flatten()(X) X = layers.Dense(128, activation='relu')(X) X = layers.Dropout(0.5)(X) X = layers.Dense(64, activation='relu')(X) X = layers.Dropout(0.5)(X) X = layers.Dense(32, activation='relu')(X) X = layers.Dropout(0.5)(X) outputs = layers.Dense(1)(X) model = Model(inputs=inputs, outputs=outputs) model.compile(optimizer='adam', loss='mse', metrics=['mse']) # Adam, categorical_crossentropy model.summary() # log_dir = "logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1) # # model.fit(np_data, y_data, epochs=10, batch_size= 64, validation_split=0.3, callbacks=[tensorboard_callback])
def build_model(self, phenotype): model = models.Sequential() filter_size = 32 nconvs = 0 optimizer = None model.add( layers.InputLayer(input_shape=(28, 28, params['DATASET_NUM_SHAPE']))) nblocks = int(phenotype[0]) for n in range(nblocks): for block in phenotype.split(','): if 'Conv' in block: if nconvs == 2: filter_size *= 2 nconvs = 0 model.add( layers.Conv2D(filter_size, (3, 3), activation='relu', padding='same')) if 'BNorm' in block: model.add(layers.BatchNormalization()) nconvs += 1 if 'MaxPool' in block: model.add(layers.MaxPooling2D(pool_size=(2, 2))) if 'Dropout' in block: model.add(layers.Dropout(0.25)) for block in phenotype.split(','): if 'Flatten' in block: model.add(layers.Flatten()) if 'Fc' in block: nfc, neurons = re.findall('\d+', block) for n in range(int(nfc)): model.add(layers.Dense(int(neurons))) model.add(layers.Activation('relu')) if 'Dropout' in block: model.add(layers.Dropout(0.5)) if 'Lr' in block: args = re.findall('\d+\.\d+', block) optimizer = optimizers.Adam(learning_rate=float(args[0])) model.add( layers.Dense(params['DATASET_NUM_CLASSES'], activation='softmax')) model.summary() # F1 Score metric function def f1_score(y_true, y_pred): true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) possible_positives = K.sum(K.round(K.clip(y_true, 0, 1))) predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1))) precision = true_positives / (predicted_positives + K.epsilon()) recall = true_positives / (possible_positives + K.epsilon()) f1_val = 2 * (precision * recall) / (precision + recall + K.epsilon()) return f1_val model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy', f1_score]) return model
def _create_keras_temp_model(self): # Each input sample consists of a bag of x`MAX_CONTEXTS` tuples (source_terminal, path, target_terminal). # The valid mask indicates for each context whether it actually exists or it is just a padding. path_source_token_input = keras.Input((self.config.MAX_CONTEXTS,), dtype=tf.int32) path_input = keras.Input((self.config.MAX_CONTEXTS,), dtype=tf.int32) path_target_token_input = keras.Input((self.config.MAX_CONTEXTS,), dtype=tf.int32) context_valid_mask = keras.Input((self.config.MAX_CONTEXTS,)) # Input paths are indexes, we embed these here. paths_embedded = layers.Embedding( self.vocabs.path_vocab.size, self.config.PATH_EMBEDDINGS_SIZE, name='path_embedding')(path_input) # Input terminals are indexes, we embed these here. token_embedding_shared_layer = layers.Embedding( self.vocabs.token_vocab.size, self.config.TOKEN_EMBEDDINGS_SIZE, name='token_embedding') path_source_token_embedded = token_embedding_shared_layer(path_source_token_input) path_target_token_embedded = token_embedding_shared_layer(path_target_token_input) # `Context` is a concatenation of the 2 terminals & path embedding. # Each context is a vector of size 3 * EMBEDDINGS_SIZE. context_embedded = layers.Concatenate()([path_source_token_embedded, paths_embedded, path_target_token_embedded]) context_embedded = layers.Dropout(1 - self.config.DROPOUT_KEEP_RATE)(context_embedded) # Lets get dense: Apply a dense layer for each context vector (using same weights for all of the context). context_after_dense = layers.TimeDistributed( Dense(self.config.CODE_VECTOR_SIZE, use_bias=False, activation='tanh'))(context_embedded) # The final code vectors are received by applying attention to the "densed" context vectors. code_vectors, attention_weights = AttentionLayer(name='attention')( [context_after_dense, context_valid_mask]) # "Decode": Now we use another dense layer to get the target word embedding from each code vector. target_index = layers.Dense( self.vocabs.target_vocab.size, use_bias=False, activation='softmax', name='target_index')(code_vectors) print(target_index) print('PPPP'*80) # Wrap the layers into a Keras model, using our subtoken-metrics and the CE loss. inputs = [path_source_token_input, path_input, path_target_token_input, context_valid_mask] self.keras_train_model = keras.Model(inputs=inputs, outputs=target_index) # Actual target word predictions (as strings). Used as a second output layer. # Used for predict() and for the evaluation metrics calculations. topk_predicted_words, topk_predicted_words_scores = TopKWordPredictionsLayer( self.config.TOP_K_WORDS_CONSIDERED_DURING_PREDICTION, self.vocabs.target_vocab.get_index_to_word_lookup_table(), name='target_string')(target_index) # topk_predicted_words, topk_predicted_words_scores = TopKWordPredictionsLayer( # self.config.TOP_K_WORDS_CONSIDERED_DURING_PREDICTION, # self.vocabs.target_vocab.get_index_to_word_lookup_table(), # name='target_string')(target_index) # We use another dedicated Keras model for evaluation. # The evaluation model outputs the `topk_predicted_words` as a 2nd output. # The separation between train and eval models is for efficiency. # self.keras_eval_model = keras.Model( inputs=inputs, outputs=[target_index, topk_predicted_words], name="code2vec-keras-model") # model_new = load_model('keke.h5', custom_objects={'AttentionLayer' : AttentionLayer}) # # # a = model_new.get_layer(-1) # # a = model_new.get_layer('target_index')#.get_weights()(code_vectors) # # a = getattr(model_new, 'input_1') # # a = model_new.__getitem__ # print(tf.keras.backend.get_value(model_new.layers[9])) # # a = tf.keras.backend.get_value(model_new.output) # # print(model_new['input_1']) # print('-'*80) # self.keras_eval_model = keras.Model( # inputs=inputs, outputs=[a, topk_predicted_words], name="code2vec-keras-model") # self.keras_eval_model = model_new # self.keras_eval_model.summary() # We use another dedicated Keras function to produce predictions. # It have additional outputs than the original model. # It is based on the trained layers of the original model and uses their weights. predict_outputs = tuple(KerasPredictionModelOutput( target_index=target_index, code_vectors=code_vectors, attention_weights=attention_weights, topk_predicted_words=topk_predicted_words, topk_predicted_words_scores=topk_predicted_words_scores)) self.keras_model_predict_function = K.function(inputs=inputs, outputs=predict_outputs)
history = model.fit(X_train, y_train, epochs=20, verbose=1) # 绘制不同层数的网络决策边界曲线 preds = model.predict_classes(np.c_[XX.ravel(), YY.ravel()]) title = "网络层数({})".format(n) make_plot(X_train, y_train, title, XX, YY, preds) # Dropout的影响 for n in range(5): # 构建5种不同数量Dropout层的网络 model = Sequential() model.add(layers.Dense(8, input_dim=2, activation='relu')) counter = 0 for _ in range(n): model.add(layers.Dense(64, activation='relu')) if counter < n: # 添加n个Dropout层 counter += 1 model.add(layers.Dropout(rate=0.5)) model.add(layers.Dense(1, activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) history = model.fit(X_train, y_train, epochs=20, verbose=1) preds = model.predict_classes(np.c_[XX.ravel(), YY.ravel()]) title = "Dropout({})".format(n) make_plot(X_train, y_train, title, XX, YY, preds) def build_model_with_regularizeation(_lambda): # 创建带正则化项的神经网络 model = Sequential() model.add(layers.Dense(8, input_dim=2, activation='relu')) model.add(
# convert class vectors to binary class matrices y_train = keras.utils.to_categorical(y_train, num_classes) y_test = keras.utils.to_categorical(y_test, num_classes) model = keras.Sequential( [ keras.Input(shape=input_shape), layers.Conv2D(32, kernel_size=(3, 3), activation="relu"), layers.MaxPooling2D(pool_size=(2, 2)), layers.Conv2D(64, kernel_size=(3, 3), activation="relu"), layers.MaxPooling2D(pool_size=(2, 2)), layers.Flatten(), layers.Dropout(0.5), layers.Dense(num_classes, activation="softmax"), ] ) model.summary() batch_size = 128 epochs = 15 model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"]) model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_split=0.1) score = model.evaluate(x_test, y_test, verbose=0)
def create_model(model_type='state_estimator', model_opt='best_noise_opt'): ''' inputs: model_type: str specifying either 'state_estimator' or 'quality_control' type machine learning model. model_opt: str specifying dataset the model parameters were optimized on. Valid options for 'state_estimator' model_type: 'noiseless_opt' or 'best_noise_opt'. Valid options for 'quality_control' type: 'uniform_noise_dist_opt'. ''' valid_model_types = ['state_estimator','quality_control'] if model_type not in valid_model_types: raise ValueError( 'model_type not recognized: ', model_type, ' Valid values: ', valid_model_types) valid_model_opts = { 'state_estimator': ['noiseless_opt', 'best_noise_opt'], 'quality_control': ['uniform_noise_dist_opt']} if model_opt not in valid_model_opts[model_type]: raise ValueError( 'model_opt not recognized: ', model_opt, ' Valid values: ', valid_model_opts[model_type]) if model_type=='state_estimator' and model_opt=='best_noise_opt': lr = 1.21e-3 k_size = [[7, 7], [7, 7]] cnn_maxpool = False cnn_stack = 2 n_cnn = 2 # these lists should be length n_cnn n_filters = [[22, 22], [35, 35]] drop_rates = [[0.655,0.655], [0.194, 0.194]] layer_norm = False ave_pool = True activation='relu' dense_n = 0 elif model_type=='state_estimator' and model_opt == 'noiseless_opt': lr = 3.45e-3 k_size = [[5], [5], [5]] cnn_maxpool=False cnn_stack = 1 n_cnn = 3 n_filters = [[23], [7], [18]] drop_rates = [[0.12], [0.28], [0.30]] layer_norm = True ave_pool = True activation = 'relu' dense_n = 0 elif model_type=='quality_control' and model_opt=='uniform_noise_dist_opt': lr = 2.65e-4 k_size = [[7, 3]] cnn_maxpool = True cnn_stack = 2 n_cnn = 1 n_filters = [[184, 249]] drop_rates = [[0.05, 0.0]] layer_norm = True ave_pool = True activation='swish' dense_n = 1 dense_dropout = [0.6] dense_units = [161] # set stride to 2 if not using maxpool as size reduction if cnn_maxpool: cnn_stride=1 else: cnn_stride=2 # input layer inputs = tf_layers.Input(shape=(config.SUB_SIZE,config.SUB_SIZE,1)) x = inputs for i in range(n_cnn): for j in range(cnn_stack): if j==cnn_stack-1: stride = cnn_stride else: stride=1 x = tf_layers.Conv2D( filters=n_filters[i][j], kernel_size=k_size[i][j], padding='same', strides=stride)(x) x = tf_layers.Dropout(rate=drop_rates[i][j])(x) if layer_norm: x = tf_layers.LayerNormalization()(x) x = tf_layers.Activation(activation)(x) if cnn_maxpool: x = tf_layers.MaxPooling2D(pool_size=(2,2), strides=2)(x) if ave_pool: x = tf_layers.GlobalAvgPool2D()(x) x = tf_layers.Flatten()(x) for i in range(dense_n): x = tf_layers.Dense(units=dense_units[i],activation=activation)(x) x = tf_layers.Dropout(rate=dense_dropout[i])(x) if model_type=='state_estimator': outputs = tf_layers.Dense( units=config.NUM_STATES, activation='softmax')(x) model = tf_Model(inputs, outputs, name='device_state_estimator_'+model_opt) model.compile( optimizer=tf_Adam(learning_rate=lr), loss='categorical_crossentropy', metrics=['accuracy']) elif model_type=='quality_control': outputs = tf_layers.Dense( units=config.NUM_QUALITY_CLASSES, activation='softmax')(x) model = tf_Model( inputs=inputs, outputs=outputs, name='data_quality_control_'+model_opt) model.compile( optimizer=tf_Adam(learning_rate=lr), loss='categorical_crossentropy', metrics=['accuracy']) return model
for spectrogram, _ in spectrogram_ds.take(1): input_shape = spectrogram.shape print('Input shape:', input_shape) num_labels = len(commands) norm_layer = preprocessing.Normalization() norm_layer.adapt(spectrogram_ds.map(lambda x, _: x)) model = models.Sequential([ layers.Input(shape=input_shape), #preprocessing.Resizing(32, 32), norm_layer, layers.Conv2D(32, 3, activation='relu'), layers.Conv2D(64, 3, activation='relu'), layers.MaxPooling2D(), layers.Dropout(0.25), layers.Flatten(), layers.Dense(128, activation='relu'), layers.Dropout(0.5), layers.Dense(num_labels), ]) model.summary() model.compile( optimizer=tf.keras.optimizers.Adam(), loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy'], ) EPOCHS = 50
def build_model(self, phenotype): # To free memory on google colab. if K.backend() == 'tensorflow': K.clear_session() nconv, npool, nfc, nfcneuron = [int(i) for i in re.findall('\d+', phenotype.split('lr-')[0])] has_dropout = 'dropout' in phenotype has_batch_normalization = 'bnorm' in phenotype has_pool = 'pool' in phenotype learning_rate = float(phenotype.split('lr-')[1]) # number of filters filter_size = 32 model = models.Sequential() try: # Pooling for i in range(npool): # Convolutions for j in range(nconv): model.add(layers.Conv2D(filter_size, (3, 3), activation='relu', padding='same', input_shape=(32, 32, 3))) # Duplicate number of filters for each two convolutions if (((i + j) % 2) == 1): filter_size = filter_size * 2 # Add batch normalization if has_batch_normalization: model.add(layers.BatchNormalization()) # Add pooling if has_pool: model.add(layers.MaxPooling2D(pool_size=(2, 2))) # Add dropout if has_dropout: model.add(layers.Dropout(0.25)) model.add(layers.Flatten()) # fully connected for i in range(nfc): model.add(layers.Dense(nfcneuron)) model.add(layers.Activation('relu')) if has_dropout: model.add(layers.Dropout(0.5)) model.add(layers.Dense(10, activation='softmax')) # model.summary() except Exception as ex: # Some NN topologies are invalid print(ex) return None opt = optimizers.Adam(lr=learning_rate) # F1 Score metric function def f1_score(y_true, y_pred): true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) possible_positives = K.sum(K.round(K.clip(y_true, 0, 1))) predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1))) precision = true_positives / (predicted_positives + K.epsilon()) recall = true_positives / (possible_positives + K.epsilon()) f1_val = 2 * (precision * recall) / (precision + recall + K.epsilon()) return f1_val model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy', f1_score]) return model