コード例 #1
0
ファイル: demo.py プロジェクト: Joxis/AdaptiveReID
def init_model(backbone_model_name,
               freeze_backbone_for_N_epochs,
               input_shape,
               region_num,
               attribute_name_to_label_encoder_dict,
               kernel_regularization_factor,
               bias_regularization_factor,
               gamma_regularization_factor,
               beta_regularization_factor,
               use_adaptive_l1_l2_regularizer,
               min_value_in_clipping,
               max_value_in_clipping,
               share_last_block=False):
    def _add_objective_module(input_tensor):
        # Add GlobalAveragePooling2D
        if len(K.int_shape(input_tensor)) == 4:
            global_average_pooling_tensor = GlobalAveragePooling2D()(
                input_tensor)
        else:
            global_average_pooling_tensor = input_tensor
        if min_value_in_clipping is not None and max_value_in_clipping is not None:
            global_average_pooling_tensor = Lambda(
                lambda x: K.clip(x,
                                 min_value=min_value_in_clipping,
                                 max_value=max_value_in_clipping))(
                                     global_average_pooling_tensor)

        # https://arxiv.org/abs/1801.07698v1 Section 3.2.2 Output setting
        # https://arxiv.org/abs/1807.11042
        classification_input_tensor = global_average_pooling_tensor
        classification_embedding_tensor = BatchNormalization(
            scale=True, epsilon=2e-5)(classification_input_tensor)

        # Add categorical crossentropy loss
        assert len(attribute_name_to_label_encoder_dict) == 1
        # label_encoder = attribute_name_to_label_encoder_dict["identity_ID"]
        # class_num = len(label_encoder.classes_)
        # TODO: hardcoded for Market1501 model
        class_num = 751
        classification_output_tensor = Dense(
            units=class_num,
            use_bias=False,
            kernel_initializer=RandomNormal(
                mean=0.0, stddev=0.001))(classification_embedding_tensor)
        classification_output_tensor = Activation("softmax")(
            classification_output_tensor)

        # Add miscellaneous loss
        miscellaneous_input_tensor = global_average_pooling_tensor
        miscellaneous_embedding_tensor = miscellaneous_input_tensor
        miscellaneous_output_tensor = miscellaneous_input_tensor

        return classification_output_tensor, classification_embedding_tensor, miscellaneous_output_tensor, miscellaneous_embedding_tensor

    def _apply_concatenation(tensor_list):
        if len(tensor_list) == 1:
            return tensor_list[0]
        else:
            return Concatenate()(tensor_list)

    def _triplet_hermans_loss(y_true,
                              y_pred,
                              metric="euclidean",
                              margin="soft"):
        # Create the loss in two steps:
        # 1. Compute all pairwise distances according to the specified metric.
        # 2. For each anchor along the first dimension, compute its loss.
        dists = cdist(y_pred, y_pred, metric=metric)
        loss = batch_hard(dists=dists,
                          pids=tf.argmax(y_true, axis=-1),
                          margin=margin)
        return loss

    # Initiation
    classification_output_tensor_list = []
    classification_embedding_tensor_list = []
    miscellaneous_output_tensor_list = []
    miscellaneous_embedding_tensor_list = []

    # Initiate the early blocks
    model_instantiation = getattr(applications, backbone_model_name, None)
    assert model_instantiation is not None, "Backbone {} is not supported.".format(
        backbone_model_name)
    submodel_list, preprocess_input = model_instantiation(
        input_shape=input_shape)
    vanilla_input_tensor = Input(shape=K.int_shape(submodel_list[0].input)[1:])
    intermediate_output_tensor = vanilla_input_tensor
    for submodel in submodel_list[:-1]:
        if freeze_backbone_for_N_epochs > 0:
            submodel.trainable = False
        intermediate_output_tensor = submodel(intermediate_output_tensor)

    # Initiate the last blocks
    last_block = submodel_list[-1]
    last_block_for_global_branch_model = replicate_model(
        last_block, name="last_block_for_global_branch")
    if freeze_backbone_for_N_epochs > 0:
        last_block_for_global_branch_model.trainable = False
    if share_last_block:
        last_block_for_regional_branch_model = last_block_for_global_branch_model
    else:
        last_block_for_regional_branch_model = replicate_model(
            last_block, name="last_block_for_regional_branch")
        if freeze_backbone_for_N_epochs > 0:
            last_block_for_regional_branch_model.trainable = False

    # Add the global branch
    classification_output_tensor, classification_embedding_tensor, miscellaneous_output_tensor, miscellaneous_embedding_tensor = _add_objective_module(
        last_block_for_global_branch_model(intermediate_output_tensor))
    classification_output_tensor_list.append(classification_output_tensor)
    classification_embedding_tensor_list.append(
        classification_embedding_tensor)
    miscellaneous_output_tensor_list.append(miscellaneous_output_tensor)
    miscellaneous_embedding_tensor_list.append(miscellaneous_embedding_tensor)

    # Add the regional branch
    if region_num > 0:
        # Process each region
        regional_branch_output_tensor = last_block_for_regional_branch_model(
            intermediate_output_tensor)
        total_height = K.int_shape(regional_branch_output_tensor)[1]
        region_size = total_height // region_num
        for region_index in np.arange(region_num):
            # Get a slice of feature maps
            start_index = region_index * region_size
            end_index = (region_index + 1) * region_size
            if region_index == region_num - 1:
                end_index = total_height
            sliced_regional_branch_output_tensor = Lambda(
                lambda x, start_index=start_index, end_index=end_index:
                x[:, start_index:end_index])(regional_branch_output_tensor)

            # Downsampling
            sliced_regional_branch_output_tensor = Conv2D(
                filters=K.int_shape(sliced_regional_branch_output_tensor)[-1]
                // region_num,
                kernel_size=3,
                padding="same")(sliced_regional_branch_output_tensor)
            sliced_regional_branch_output_tensor = Activation("relu")(
                sliced_regional_branch_output_tensor)

            # Add the regional branch
            classification_output_tensor, classification_embedding_tensor, miscellaneous_output_tensor, miscellaneous_embedding_tensor = _add_objective_module(
                sliced_regional_branch_output_tensor)
            classification_output_tensor_list.append(
                classification_output_tensor)
            classification_embedding_tensor_list.append(
                classification_embedding_tensor)
            miscellaneous_output_tensor_list.append(
                miscellaneous_output_tensor)
            miscellaneous_embedding_tensor_list.append(
                miscellaneous_embedding_tensor)

    # Define the merged model
    embedding_tensor_list = [
        _apply_concatenation(miscellaneous_embedding_tensor_list)
    ]
    embedding_size_list = [
        K.int_shape(embedding_tensor)[1]
        for embedding_tensor in embedding_tensor_list
    ]
    merged_embedding_tensor = _apply_concatenation(embedding_tensor_list)
    merged_model = Model(inputs=[vanilla_input_tensor],
                         outputs=classification_output_tensor_list +
                         miscellaneous_output_tensor_list +
                         [merged_embedding_tensor])
    merged_model = specify_regularizers(merged_model,
                                        kernel_regularization_factor,
                                        bias_regularization_factor,
                                        gamma_regularization_factor,
                                        beta_regularization_factor,
                                        use_adaptive_l1_l2_regularizer)

    # Define the models for training/inference
    training_model = Model(inputs=[merged_model.input],
                           outputs=merged_model.output[:-1],
                           name="training_model")
    inference_model = Model(inputs=[merged_model.input],
                            outputs=[merged_model.output[-1]],
                            name="inference_model")
    inference_model.embedding_size_list = embedding_size_list

    # Compile the model
    classification_loss_function_list = [
        "categorical_crossentropy"
    ] * len(classification_output_tensor_list)
    triplet_hermans_loss_function = lambda y_true, y_pred: 1.0 * _triplet_hermans_loss(
        y_true, y_pred)
    miscellaneous_loss_function_list = [
        triplet_hermans_loss_function
    ] * len(miscellaneous_output_tensor_list)
    training_model.compile_kwargs = {
        "optimizer":
        Adam(),
        "loss":
        classification_loss_function_list + miscellaneous_loss_function_list
    }
    training_model.compile(**training_model.compile_kwargs)

    # Print the summary of the models
    # summarize_model(training_model)
    # summarize_model(inference_model)

    return training_model, inference_model, preprocess_input
コード例 #2
0
ファイル: solution.py プロジェクト: nixingyang/FlipReID
def init_model(backbone_model_name, freeze_backbone_for_N_epochs, input_shape,
               region_num, attribute_name_to_label_encoder_dict,
               kernel_regularization_factor, bias_regularization_factor,
               gamma_regularization_factor, beta_regularization_factor,
               pooling_mode, min_value, max_value, use_horizontal_flipping):
    def _add_pooling_module(input_tensor):
        # Add a global pooling layer
        output_tensor = input_tensor
        if len(K.int_shape(output_tensor)) == 4:
            if pooling_mode == "Average":
                output_tensor = GlobalAveragePooling2D()(output_tensor)
            elif pooling_mode == "Max":
                output_tensor = GlobalMaxPooling2D()(output_tensor)
            elif pooling_mode == "GeM":
                output_tensor = GlobalGeMPooling2D()(output_tensor)
            else:
                assert False, "{} is an invalid argument!".format(pooling_mode)

        # Add the clipping operation
        if min_value is not None and max_value is not None:
            output_tensor = Lambda(lambda x: K.clip(
                x, min_value=min_value, max_value=max_value))(output_tensor)

        return output_tensor

    def _add_classification_module(input_tensor):
        # Add a batch normalization layer
        output_tensor = input_tensor
        output_tensor = BatchNormalization(epsilon=2e-5)(output_tensor)

        # Add a dense layer with softmax activation
        label_encoder = attribute_name_to_label_encoder_dict["identity_ID"]
        class_num = len(label_encoder.classes_)
        output_tensor = Dense(units=class_num,
                              use_bias=False,
                              kernel_initializer=RandomNormal(
                                  mean=0.0, stddev=0.001))(output_tensor)
        output_tensor = Activation("softmax")(output_tensor)

        return output_tensor

    def _triplet_hermans_loss(y_true,
                              y_pred,
                              metric="euclidean",
                              margin="soft"):
        # Create the loss in two steps:
        # 1. Compute all pairwise distances according to the specified metric.
        # 2. For each anchor along the first dimension, compute its loss.
        dists = cdist(y_pred, y_pred, metric=metric)
        loss = batch_hard(dists=dists,
                          pids=tf.argmax(y_true, axis=-1),
                          margin=margin)
        return loss

    # Initiation
    miscellaneous_output_tensor_list = []

    # Initiate the early blocks
    applications_instance = Applications()
    model_name_to_model_function = applications_instance.get_model_name_to_model_function(
    )
    assert backbone_model_name in model_name_to_model_function.keys(
    ), "Backbone {} is not supported.".format(backbone_model_name)
    model_function = model_name_to_model_function[backbone_model_name]
    blocks = applications_instance.get_model_in_blocks(
        model_function=model_function, include_top=False)
    vanilla_input_tensor = Input(shape=input_shape)
    intermediate_output_tensor = vanilla_input_tensor
    for block in blocks[:-1]:
        block = Applications.wrap_block(block, intermediate_output_tensor)
        intermediate_output_tensor = block(intermediate_output_tensor)

    # Initiate the last blocks
    last_block = Applications.wrap_block(blocks[-1],
                                         intermediate_output_tensor)
    last_block_for_global_branch_model = replicate_model(
        model=last_block, suffix="global_branch")
    last_block_for_regional_branch_model = replicate_model(
        model=last_block, suffix="regional_branch")

    # Add the global branch
    miscellaneous_output_tensor = _add_pooling_module(
        input_tensor=last_block_for_global_branch_model(
            intermediate_output_tensor))
    miscellaneous_output_tensor_list.append(miscellaneous_output_tensor)

    # Add the regional branch
    if region_num > 0:
        # Process each region
        regional_branch_output_tensor = last_block_for_regional_branch_model(
            intermediate_output_tensor)
        total_height = K.int_shape(regional_branch_output_tensor)[1]
        region_size = total_height // region_num
        for region_index in np.arange(region_num):
            # Get a slice of feature maps
            start_index = region_index * region_size
            end_index = (region_index + 1) * region_size
            if region_index == region_num - 1:
                end_index = total_height
            sliced_regional_branch_output_tensor = Lambda(
                lambda x, start_index=start_index, end_index=end_index:
                x[:, start_index:end_index])(regional_branch_output_tensor)

            # Downsampling
            sliced_regional_branch_output_tensor = Conv2D(
                filters=K.int_shape(sliced_regional_branch_output_tensor)[-1]
                // region_num,
                kernel_size=3,
                padding="same")(sliced_regional_branch_output_tensor)
            sliced_regional_branch_output_tensor = Activation("relu")(
                sliced_regional_branch_output_tensor)

            # Add the regional branch
            miscellaneous_output_tensor = _add_pooling_module(
                input_tensor=sliced_regional_branch_output_tensor)
            miscellaneous_output_tensor_list.append(
                miscellaneous_output_tensor)

    # Define the model used in inference
    inference_model = Model(inputs=[vanilla_input_tensor],
                            outputs=miscellaneous_output_tensor_list,
                            name="inference_model")
    specify_regularizers(inference_model, kernel_regularization_factor,
                         bias_regularization_factor,
                         gamma_regularization_factor,
                         beta_regularization_factor)

    # Define the model used in classification
    classification_input_tensor_list = [
        Input(shape=K.int_shape(item)[1:])
        for item in miscellaneous_output_tensor_list
    ]
    classification_output_tensor_list = []
    for classification_input_tensor in classification_input_tensor_list:
        classification_output_tensor = _add_classification_module(
            input_tensor=classification_input_tensor)
        classification_output_tensor_list.append(classification_output_tensor)
    classification_model = Model(inputs=classification_input_tensor_list,
                                 outputs=classification_output_tensor_list,
                                 name="classification_model")
    specify_regularizers(classification_model, kernel_regularization_factor,
                         bias_regularization_factor,
                         gamma_regularization_factor,
                         beta_regularization_factor)

    # Define the model used in training
    expand = lambda x: x if isinstance(x, list) else [x]
    vanilla_input_tensor = Input(shape=K.int_shape(inference_model.input)[1:])
    vanilla_feature_tensor_list = expand(inference_model(vanilla_input_tensor))
    if use_horizontal_flipping:
        flipped_input_tensor = tf.image.flip_left_right(vanilla_input_tensor)
        flipped_feature_tensor_list = expand(
            inference_model(flipped_input_tensor))
        merged_feature_tensor_list = [
            sum(item_tuple) / 2 for item_tuple in zip(
                vanilla_feature_tensor_list, flipped_feature_tensor_list)
        ]
    else:
        merged_feature_tensor_list = vanilla_feature_tensor_list
    miscellaneous_output_tensor_list = merged_feature_tensor_list
    classification_output_tensor_list = expand(
        classification_model(merged_feature_tensor_list))
    training_model = Model(inputs=[vanilla_input_tensor],
                           outputs=miscellaneous_output_tensor_list +
                           classification_output_tensor_list,
                           name="training_model")

    # Add the flipping loss
    if use_horizontal_flipping:
        flipping_loss_list = [
            K.mean(mean_squared_error(*item_tuple)) for item_tuple in zip(
                vanilla_feature_tensor_list, flipped_feature_tensor_list)
        ]
        flipping_loss = sum(flipping_loss_list)
        training_model.add_metric(flipping_loss,
                                  name="flipping",
                                  aggregation="mean")
        training_model.add_loss(1.0 * flipping_loss)

    # Compile the model
    triplet_hermans_loss_function = lambda y_true, y_pred: 1.0 * _triplet_hermans_loss(
        y_true, y_pred)
    miscellaneous_loss_function_list = [
        triplet_hermans_loss_function
    ] * len(miscellaneous_output_tensor_list)
    categorical_crossentropy_loss_function = lambda y_true, y_pred: 1.0 * categorical_crossentropy(
        y_true, y_pred, from_logits=False, label_smoothing=0.0)
    classification_loss_function_list = [
        categorical_crossentropy_loss_function
    ] * len(classification_output_tensor_list)
    training_model.compile_kwargs = {
        "optimizer":
        Adam(),
        "loss":
        miscellaneous_loss_function_list + classification_loss_function_list
    }
    if freeze_backbone_for_N_epochs > 0:
        specify_trainable(model=training_model,
                          trainable=False,
                          keywords=[block.name for block in blocks])
    training_model.compile(**training_model.compile_kwargs)

    # Print the summary of the training model
    summarize_model(training_model)

    return training_model, inference_model