def attention_3d_block(inputs, seq_len=21): # input_dim = int(inputs.shape[2]) a = Permute((2, 1))(inputs) a = Dense(seq_len, activation='softmax')(a) a_probs = Permute((2, 1), name='attention_vec')(a) # output_attention_mul = merge([inputs, a_probs], name='attention_mul', mode='mul') output_attention_mul = multiply([inputs, a_probs], name='attention_mul') return output_attention_mul
def __attention_3d_block(self, _lstm_output, _time_steps) -> Layer: """https://github.com/philipperemy/keras-attention-mechanism/blob/master/attention_lstm.py """ att = Permute((2, 1))(_lstm_output) att = Reshape((_lstm_output.shape[2].value, _time_steps))(att) att = Dense(_time_steps, activation='softmax')(att) att_probs = Permute((2, 1), name='attention_vec')(att) return Multiply(name='attention_mul')([_lstm_output, att_probs])
def layer(input_tensor): h, w, c = int_shape(input_tensor)[1:] H = h * factor W = w * factor x = Conv2DBlock(c * factor**2, (1, 1), padding='same', name='duc_{}'.format(factor))(input_tensor) x = Permute((3, 1, 2))(x) x = Reshape((c, factor, factor, h, w))(x) x = Permute((1, 4, 2, 5, 3))(x) x = Reshape((c, H, W))(x) x = Permute((2, 3, 1))(x) return x
def attention_3d_block(inputs, SINGLE_ATTENTION_VECTOR=False): # inputs.shape = (batch_size, time_steps, input_dim) input_dim = int(inputs.shape[2]) TIME_STEPS = int(inputs.shape[1]) a = Permute((2, 1))(inputs) a = Reshape( (input_dim, TIME_STEPS) )(a) # this line is not useful. It's just to know which dimension is what. a = Dense(TIME_STEPS, activation='softmax')(a) if SINGLE_ATTENTION_VECTOR: a = Lambda(lambda x: K.mean(x, axis=1), name='dim_reduction')(a) a = RepeatVector(input_dim)(a) a_probs = Permute((2, 1), name='attention_vec')(a) output_attention_mul = concatenate([inputs, a_probs], name='attention_mul') return output_attention_mul
def vgg_face_blank(): # Model initialization mdl = Sequential() # First layer is a dummy-permutation = Identity to specify input shape mdl.add(Permute((1, 2, 3), input_shape=(224, 224, 3))) # WARNING : 0 is the sample dim # Model body for l in convblock(64, 1, bits=2): mdl.add(l) for l in convblock(128, 2, bits=2): mdl.add(l) for l in convblock(256, 3, bits=3): mdl.add(l) for l in convblock(512, 4, bits=3): mdl.add(l) for l in convblock(512, 5, bits=3): mdl.add(l) # Model head mdl.add(Convolution2D(4096, kernel_size=(7, 7), activation='relu', name='fc6')) mdl.add(Dropout(0.5)) mdl.add(Convolution2D(4096, kernel_size=(1, 1), activation='relu', name='fc7')) mdl.add(Dropout(0.5)) mdl.add(Convolution2D(2622, kernel_size=(1, 1), activation='relu', name='fc8')) mdl.add(Flatten()) mdl.add(Activation('softmax')) return mdl
def construct_model(windowSize,numChannels): # construct sequential model model = Sequential() # permute input so that it is as in EEG Net paper model.add(Permute((3,2,1), input_shape=(windowSize,numChannels,1))) # layer1 model.add(Conv2D(16, kernel_size=(numChannels, 1), padding='valid', strides=(1, 1), data_format='channels_first', activation='relu')) model.add(BatchNormalization(axis=1, scale=False, center=False)) #model.add(Permute((2,1,3))) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2),strides=(2, 2),padding='same')) # layer2 model.add(Conv2D(8,kernel_size=(1, 64),data_format='channels_first',padding='same')) model.add(BatchNormalization(axis=1,scale=False, center=False)) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2),strides=(2, 2),padding='same')) model.add(Dropout(0.5)) # layer3 model.add(Conv2D(4,kernel_size=(5, 5),data_format='channels_first',padding='same')) model.add(BatchNormalization(axis=1,scale=False,center=False)) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2), data_format='channels_first',padding='same')) model.add(Dropout(0.5)) # layer4 model.add(Flatten()) model.add(Dense(1024, activation='relu')) model.add(Dropout(0.5)) # layer5 model.add(Dense(2, activation='softmax')) #print(model.summary()) return model
def squeeze_excite_block(input, ratio=16): ''' Create a channel-wise squeeze-excite block Args: input: input tensor filters: number of output filters Returns: a keras tensor References - [Squeeze and Excitation Networks](https://arxiv.org/abs/1709.01507) ''' init = input channel_axis = 1 if K.image_data_format() == "channels_first" else -1 filters = K.int_shape(init)[channel_axis] se_shape = (1, 1, filters) se = GlobalAveragePooling2D()(init) se = Reshape(se_shape)(se) se = Dense(filters // ratio, activation='relu', kernel_initializer='he_normal', use_bias=False)(se) se = Dense(filters, activation='sigmoid', kernel_initializer='he_normal', use_bias=False)(se) if K.image_data_format() == 'channels_first': se = Permute((3, 1, 2))(se) x = multiply([init, se]) return x
def default_classification_model(num_classes, num_anchors, pyramid_feature_size=256, prior_probability=0.01, classification_feature_size=256, name='classification_submodel'): """Creates the default regression submodel. Args: num_classes: Number of classes to predict a score for at each feature level. num_anchors: Number of anchors to predict classification scores for at each feature level. pyramid_feature_size: The number of filters to expect from the feature pyramid levels. classification_feature_size: The number of filters to use in the layers in the classification submodel. name: The name of the submodel. Returns: A keras.models.Model that predicts classes for each anchor. """ options = { 'kernel_size': 3, 'strides': 1, 'padding': 'same', } if K.image_data_format() == 'channels_first': inputs = Input(shape=(pyramid_feature_size, None, None, None)) else: inputs = Input(shape=(None, None, None, pyramid_feature_size)) outputs = inputs for i in range(4): outputs = Conv3D(filters=classification_feature_size, activation='relu', name='pyramid_classification_{}'.format(i), kernel_initializer=RandomNormal(mean=0.0, stddev=0.01, seed=None), bias_initializer='zeros', **options)(outputs) outputs = Conv3D( filters=num_classes * num_anchors, kernel_initializer=RandomNormal(mean=0.0, stddev=0.01, seed=None), bias_initializer=PriorProbability(probability=prior_probability), name='pyramid_classification', **options)(outputs) # reshape output and apply sigmoid if K.image_data_format() == 'channels_first': outputs = Permute((2, 3, 1), name='pyramid_classification_permute')(outputs) outputs = Reshape((-1, num_classes), name='pyramid_classification_reshape')(outputs) outputs = Activation('sigmoid', name='pyramid_classification_sigmoid')(outputs) return Model(inputs=inputs, outputs=outputs, name=name)
def EEGNet_old(nb_classes, Chans = 64, Samples = 128, regRate = 0.0001, dropoutRate = 0.25, kernels = [(2, 32), (8, 4)], strides = (2, 4)): """ Keras Implementation of EEGNet_v1 (https://arxiv.org/abs/1611.08024v2) This model is the original EEGNet model proposed on arxiv https://arxiv.org/abs/1611.08024v2 with a few modifications: we use striding instead of max-pooling as this helped slightly in classification performance while also providing a computational speed-up. Note that we no longer recommend the use of this architecture, as the new version of EEGNet performs much better overall and has nicer properties. Inputs: nb_classes : total number of final categories Chans, Samples : number of EEG channels and samples, respectively regRate : regularization rate for L1 and L2 regularizations dropoutRate : dropout fraction kernels : the 2nd and 3rd layer kernel dimensions (default is the [2, 32] x [8, 4] configuration) strides : the stride size (note that this replaces the max-pool used in the original paper) """ # start the model input_main = Input((1, Chans, Samples)) layer1 = Conv2D(16, (Chans, 1), input_shape=(1, Chans, Samples), kernel_regularizer = l1_l2(l1=regRate, l2=regRate))(input_main) layer1 = BatchNormalization(axis=1)(layer1) layer1 = Activation('elu')(layer1) layer1 = Dropout(dropoutRate)(layer1) permute_dims = 2, 1, 3 permute1 = Permute(permute_dims)(layer1) layer2 = Conv2D(4, kernels[0], padding = 'same', kernel_regularizer=l1_l2(l1=0.0, l2=regRate), strides = strides)(permute1) layer2 = BatchNormalization(axis=1)(layer2) layer2 = Activation('elu')(layer2) layer2 = Dropout(dropoutRate)(layer2) layer3 = Conv2D(4, kernels[1], padding = 'same', kernel_regularizer=l1_l2(l1=0.0, l2=regRate), strides = strides)(layer2) layer3 = BatchNormalization(axis=1)(layer3) layer3 = Activation('elu')(layer3) layer3 = Dropout(dropoutRate)(layer3) flatten = Flatten(name = 'flatten')(layer3) dense = Dense(nb_classes, name = 'dense')(flatten) softmax = Activation('softmax', name = 'softmax')(dense) return Model(inputs=input_main, outputs=softmax)
def mlstm_fcn(config: Union[str, SoccerSageConfig] = SoccerSageConfig()) -> Model: '''Load and train the MLSTM-FCN model. Source: https://github.com/titu1994/MLSTM-FCN/blob/master/eeg_model.py Args: config: The configuration data. Returns: Model: A model in TensorFlow. ''' if isinstance(config, str): config = SoccerSageConfig.from_yaml(config) n_classes = config.num_results if config.classifier_type == 'total_goals': n_classes = config.num_goals ip = Input(shape=(config.time_steps, config.feature_size)) x = Permute((2, 1))(ip) #x = Masking(mask_value=config.masking_value)(x) x = LSTM(64)(ip) x = Dropout(0.8)(x) y = Conv1D(128, 8, padding='same', kernel_initializer='he_uniform')(ip) y = BatchNormalization()(y) y = Activation('relu')(y) y = squeeze_excite_block(y) y = Conv1D(256, 5, padding='same', kernel_initializer='he_uniform')(y) y = BatchNormalization()(y) y = Activation('relu')(y) y = squeeze_excite_block(y) y = Conv1D(128, 3, padding='same', kernel_initializer='he_uniform')(y) y = BatchNormalization()(y) y = Activation('relu')(y) y = GlobalAveragePooling1D()(y) x = concatenate([x, y]) out = Dense(n_classes, activation='softmax')(x) classifier = Model(ip, out) classifier.compile( loss='categorical_crossentropy', optimizer=tf.keras.optimizers.Adam(lr=config.learning_rate), metrics=['accuracy', ranked_probability_score]) if config.pretrained_classifier: files = SoccerSageFiles(config) classifier.load_weights(files.model_weights) return classifier
def get_cnn1d_tx_selu(output_size, img_height, img_width, show=True): model_input = Input(shape=(img_height * img_width, ), name='Main_input') x = Reshape((img_height, img_width), name='Reshape_1')(model_input) x = Permute((2, 1), name='Permute_1')(x) x = Conv1D(filters=256, kernel_size=48, strides=1, padding='same', activation='selu', name='Conv1D_selu_1')(x) x = Conv1D(filters=128, kernel_size=48, strides=1, padding='same', activation='selu', name='Conv1D_selu_2')(x) x = Conv1D(filters=64, kernel_size=48, strides=1, padding='same', activation='selu', name='Conv1D_selu_3')(x) x = Conv1D(filters=32, kernel_size=48, strides=1, padding='same', activation='selu', name='Conv1D_selu_4')(x) x = Conv1D(filters=16, kernel_size=48, strides=1, padding='same', activation='selu', name='Conv1D_selu_5')(x) x = Conv1D(filters=8, kernel_size=48, strides=1, padding='same', activation='selu', name='Conv1D_selu_6')(x) x = Flatten(name='Flatten_1')(x) x = Dense(512, activation='selu', name='Dense_selu_1')(x) x = Dense(512, activation='selu', name='Dense_selu_2')(x) x = Dense(512, activation='selu', name='Dense_selu_3')(x) cnn1d_output = Dense(output_size, activation='linear', name='Output_Dense_linear')(x) cnn1d = Model(inputs=model_input, outputs=cnn1d_output, name='CNN1D_Tx_selu') if show: print('CNN1D Tx summary:') cnn1d.summary() print() return cnn1d
def T_trans(T, T_F, H, W): T_in = Input(shape=(T + 7, H, W)) T_in_p = Permute((2, 3, 1))(T_in) T_mid = Conv2D(filters=T_F, kernel_size=(1, 1), padding="same")(T_in_p) T_act = Activation('relu')(T_mid) T_fin = Conv2D(filters=1, kernel_size=(1, 1), padding="same")(T_act) T_mul = Activation('relu')(T_fin) T_model = Model(inputs=T_in, outputs=T_mul) return T_model
def default_n_linear(num_outputs, input_shape=(120, 160, 3), roi_crop=(0, 0)): drop = 0.1 #we now expect that cropping done elsewhere. we will adjust our expeected image size here: input_shape = adjust_input_shape(input_shape, roi_crop) img_in = Input(shape=input_shape, name='img_in') x = img_in x = Convolution2D(24, (5, 5), strides=(2, 2), activation='relu', name="conv2d_1")(x) x = Dropout(drop)(x) x = Convolution2D(32, (5, 5), strides=(2, 2), activation='relu', name="conv2d_2")(x) x = Dropout(drop)(x) x = Convolution2D(64, (5, 5), strides=(2, 2), activation='relu', name="conv2d_3")(x) x = Dropout(drop)(x) x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu', name="conv2d_4")(x) x = Dropout(drop)(x) x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu', name="conv2d_5")(x) x = Dropout(drop)(x) # x = Flatten(name='flattened')(x) a, b, c, d = x.shape # returns dimension a = b * c * d x = Permute([1, 2, 3])(x) x = Reshape((int(a), ))(x) # convert dim -> int x = Dense(100, activation='relu')(x) x = Dropout(drop)(x) x = Dense(50, activation='relu')(x) x = Dropout(drop)(x) outputs = [] for i in range(num_outputs): outputs.append( Dense(1, activation='linear', name='n_outputs' + str(i))(x)) model = Model(inputs=[img_in], outputs=outputs) return model
def horizontal_axis_convolution(input, filters, l2): shared_squash = Conv3D(filters, [4, 1, 1], kernel_regularizer=l2) squash_x = shared_squash(input) box_x = spread_plane(squash_x, filters, [1, 2, 3, 4]) # x, y, z, f -> x, y, z, f rotate = Permute([2, 1, 3, 4])(input) squash_y = shared_squash(rotate) box_y = spread_plane(squash_y, filters, [2, 1, 3, 4]) # y, x, z, f -> x, y, z, f return box_x, box_y
def build_cgcnn_layer(self, atom_fea, bond_fea, nbr_list): total_fea = Lambda(_concat_nbrs, output_shape=_concat_nbrs_output_shape)( [atom_fea, bond_fea, nbr_list]) # total_fea shape (None, N, M, 2 * atom_fea_len + bond_fea_len) nbr_core = Dense(self.atom_fea_len)(total_fea) nbr_filter = Dense(1)(total_fea) if self.use_bn: nbr_core = BatchNormalization(axis=-1)(nbr_core) nbr_filter = Permute((1, 3, 2))(nbr_filter) nbr_filter = Activation('softmax')(nbr_filter) nbr_filter = Permute((1, 3, 2))(nbr_filter) # nbr_filter = keras.activations.softmax(nbr_filter, axis=-2) nbr_core = Activation('relu')(nbr_core) nbr_sumed = Lambda(lambda x: tf.reduce_mean(x[0] * x[1], axis=2))( [nbr_filter, nbr_core]) if self.use_bn: nbr_sumed = BatchNormalization(axis=-1)(nbr_sumed) out = Activation('relu')(Add()([atom_fea, nbr_sumed])) return out
def reg_mlstm_fcn(config: Union[ str, SoccerSageConfig] = SoccerSageConfig()) -> Model: '''Load and train the regression MLSTM-FCN model. Source: https://github.com/titu1994/MLSTM-FCN/blob/master/eeg_model.py Args: config: The configuration data. Returns: Model: A model in TensorFlow. ''' if isinstance(config, str): config = SoccerSageConfig.from_yaml(config) ip = Input(shape=(config.time_steps, config.feature_size)) x = Permute((2, 1))(ip) # x = Masking(mask_value=config.masking_value)(x) x = LSTM(8)(x) x = Dropout(0.8)(x) y = Conv1D(128, 8, padding='same', kernel_initializer='he_uniform')(ip) y = BatchNormalization()(y) y = Activation('relu')(y) y = squeeze_excite_block(y) y = Conv1D(256, 5, padding='same', kernel_initializer='he_uniform')(y) y = BatchNormalization()(y) y = Activation('relu')(y) y = squeeze_excite_block(y) y = Conv1D(128, 3, padding='same', kernel_initializer='he_uniform')(y) y = BatchNormalization()(y) y = Activation('relu')(y) y = GlobalAveragePooling1D()(y) x = concatenate([x, y]) out = Dense(1, activation='linear')(x) classifier = Model(ip, out) classifier.compile( loss='mse', optimizer=tf.keras.optimizers.Adam(lr=config.learning_rate), metrics=['mse', 'mae']) if config.pretrained_classifier: files = SoccerSageFiles(config) classifier.load_weights(files.regressor_weights) return classifier
def vertical_plane_convolution(input, plane_filters, l2): reduce = Conv3D(plane_filters, 1, kernel_regularizer=l2)(input) shared_squash = Conv3D(plane_filters, [4, 1, 4], kernel_regularizer=l2) squash_xz = shared_squash(reduce) box_xz = spread_axis(squash_xz, plane_filters, [1, 3, 2, 4]) # x, z, y, f -> x, y, z, f rotate = Permute([2, 1, 3, 4])(reduce) squash_yz = shared_squash(rotate) box_yz = spread_axis(squash_yz, plane_filters, [3, 1, 2, 4]) # y, z, x, f -> x, y, z, f return box_xz, box_yz
def default_regression_model(num_values, num_anchors, pyramid_feature_size=256, regression_feature_size=256, name='regression_submodel'): """Creates the default regression submodel. Args: num_values (int): Number of values to regress. num_anchors (int): Number of anchors to regress for each feature level. pyramid_feature_size (int): The number of filters to expect from the feature pyramid levels. regression_feature_size (int): The number of filters to use in the layers in the regression submodel. name (str): The name of the submodel. Returns: tensorflow.keras.Model: A model that predicts regression values for each anchor. """ # All new conv layers except the final one in the # RetinaNet (classification) subnets are initialized # with bias b = 0 and a Gaussian weight fill with stddev = 0.01. options = { 'kernel_size': 3, 'strides': 1, 'padding': 'same', 'kernel_initializer': RandomNormal(mean=0.0, stddev=0.01, seed=None), 'bias_initializer': 'zeros' } if K.image_data_format() == 'channels_first': inputs = Input(shape=(pyramid_feature_size, None, None)) else: inputs = Input(shape=(None, None, pyramid_feature_size)) outputs = inputs for i in range(4): outputs = Conv2D(filters=regression_feature_size, activation='relu', name='pyramid_regression_{}'.format(i), **options)(outputs) outputs = Conv2D(num_anchors * num_values, name='pyramid_regression', **options)(outputs) if K.image_data_format() == 'channels_first': outputs = Permute((2, 3, 1), name='pyramid_regression_permute')(outputs) outputs = Reshape((-1, num_values), name='pyramid_regression_reshape')(outputs) return Model(inputs=inputs, outputs=outputs, name=name)
def attention_3d_block(inputs): # inputs.shape = (batch_size, time_steps, input_dim) input_dim = int(inputs.shape[2]) a = inputs AveragePooling = pooling.GlobalAveragePooling1D( data_format='channels_last')(a) den1 = Dense(input_dim, activation='relu')(AveragePooling) den2 = Dense(input_dim, activation='hard_sigmoid')(den1) if SINGLE_ATTENTION_VECTOR: a = Lambda(lambda x: K.mean(x, axis=1), name='dim_reduction')(den2) a = RepeatVector(input_dim)(a) a_probs = Permute((1, 2), name='attention_vec')(a) output_attention_mul = merge.multiply([inputs, a_probs], name='attention_mul') # output_attention_mul = merge([inputs, a_probs], name='attention_mul', mode='mul') # 旧版本 return output_attention_mul
def default_bhv(num_outputs, num_bvh_inputs, input_shape): ''' Notes: this model depends on concatenate which failed on keras < 2.0.8 ''' img_in = Input(shape=input_shape, name='img_in') bvh_in = Input(shape=(num_bvh_inputs, ), name="behavior_in") x = img_in #x = Cropping2D(cropping=((60,0), (0,0)))(x) #trim 60 pixels off top x = Convolution2D(24, (5, 5), strides=(2, 2), activation='relu')(x) x = Convolution2D(32, (5, 5), strides=(2, 2), activation='relu')(x) x = Convolution2D(64, (5, 5), strides=(2, 2), activation='relu')(x) x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu')(x) x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu')(x) # x = Flatten(name='flattened')(x) a, b, c, d = x.shape # returns dimension a = b * c * d x = Permute([1, 2, 3])(x) x = Reshape((int(a), ))(x) # convert dim -> int x = Dense(100, activation='relu')(x) x = Dropout(.1)(x) y = bvh_in y = Dense(num_bvh_inputs * 2, activation='relu')(y) y = Dense(num_bvh_inputs * 2, activation='relu')(y) y = Dense(num_bvh_inputs * 2, activation='relu')(y) z = concatenate([x, y]) z = Dense(100, activation='relu')(z) z = Dropout(.1)(z) z = Dense(50, activation='relu')(z) z = Dropout(.1)(z) #categorical output of the angle angle_out = Dense(15, activation='softmax', name='angle_out')( z ) # Connect every input with every output and output 15 hidden units. Use Softmax to give percentage. 15 categories and find best one based off percentage 0.0-1.0 #continous output of throttle throttle_out = Dense(20, activation='softmax', name='throttle_out')( z) # Reduce to 1 number, Positive number only model = Model(inputs=[img_in, bvh_in], outputs=[angle_out, throttle_out]) return model
def interest_evolution(concat_behavior, deep_input_item, user_behavior_length, gru_type="GRU", use_neg=False, neg_concat_behavior=None, att_hidden_size=(64, 16), att_activation='sigmoid', att_weight_normalization=False, ): if gru_type not in ["GRU", "AIGRU", "AGRU", "AUGRU"]: raise ValueError("gru_type error ") aux_loss_1 = None embedding_size = None rnn_outputs = DynamicGRU(embedding_size, return_sequence=True, name="gru1")([concat_behavior, user_behavior_length]) print(concat_behavior) print(neg_concat_behavior) if gru_type == "AUGRU" and use_neg: aux_loss_1 = auxiliary_loss(rnn_outputs[:, :-1, :], concat_behavior[:, 1:, :], neg_concat_behavior[:, 1:, :], tf.subtract(user_behavior_length, 1), stag="gru") # [:, 1:] if gru_type == "GRU": rnn_outputs2 = DynamicGRU(embedding_size, return_sequence=True, name="gru2")([rnn_outputs, user_behavior_length]) # attention_score = AttentionSequencePoolingLayer(hidden_size=att_hidden_size, activation=att_activation, weight_normalization=att_weight_normalization, return_score=True)([ # deep_input_item, rnn_outputs2, user_behavior_length]) # outputs = Lambda(lambda x: tf.matmul(x[0], x[1]))( # [attention_score, rnn_outputs2]) # hist = outputs hist = AttentionSequencePoolingLayer(att_hidden_units=att_hidden_size, att_activation=att_activation, weight_normalization=att_weight_normalization, return_score=False)([ deep_input_item, rnn_outputs2, user_behavior_length]) else: # AIGRU AGRU AUGRU scores = AttentionSequencePoolingLayer(att_hidden_units=att_hidden_size, att_activation=att_activation, weight_normalization=att_weight_normalization, return_score=True)([ deep_input_item, rnn_outputs, user_behavior_length]) if gru_type == "AIGRU": hist = multiply([rnn_outputs, Permute([2, 1])(scores)]) final_state2 = DynamicGRU(embedding_size, gru_type="GRU", return_sequence=False, name='gru2')( [hist, user_behavior_length]) else: # AGRU AUGRU final_state2 = DynamicGRU(embedding_size, gru_type=gru_type, return_sequence=False, name='gru2')([rnn_outputs, user_behavior_length, Permute([2, 1])(scores)]) hist = final_state2 return hist, aux_loss_1
def default_avcnetCV(input_shape=(120, 160, 3), roi_crop=(0, 0)): #we now expect that cropping done elsewhere. we will adjust our expeected image size here: input_shape = adjust_input_shape(input_shape, roi_crop) opt = keras.optimizers.Adam() cfg = dk.load_config() drop = 0.2 # Input img_in = Input( shape=input_shape, name='img_in' ) # First layer, input layer, Shape comes from camera.py resolution, RGB x = img_in x1 = Conv2D(24, (5, 5), padding='same')(x) x1 = Activation('relu')(x1) x1 = MaxPooling2D(pool_size=(2, 2), strides=[2, 2])(x1) x1 = Dropout(drop)(x1) x2 = Conv2D(48, (5, 5), padding='same')(x1) x2 = Activation('relu')(x2) x2 = MaxPooling2D(pool_size=(2, 2), strides=[2, 2])(x2) x2 = Dropout(drop)(x2) x3 = Conv2D(96, (5, 5), padding='same')(x2) x3 = Activation('relu')(x3) x3 = MaxPooling2D(pool_size=(2, 2), strides=[2, 2])(x3) x3 = Dropout(drop)(x3) # x4 = Flatten()(x3) # OpenCV/dnn does not support flatten a, b, c, d = x3.shape # returns dimension a = b * c * d x4 = Permute([1, 2, 3])(x3) x4 = Reshape((int(a), ))(x4) # convert dim -> int x4 = Dense(500)(x4) x4 = Activation('relu')(x4) x4 = Dropout(drop)(x4) angle_out = Dense(4, activation='softmax', name='angle_out')(x4) model = Model(inputs=[img_in], outputs=[angle_out]) return model
def self_attn_block(inp, n_c, squeeze_factor=8): """ GAN Self Attention Block Code borrows from https://github.com/taki0112/Self-Attention-GAN-Tensorflow """ msg = "Input channels must be >= {}, recieved nc={}".format( squeeze_factor, n_c) assert n_c // squeeze_factor > 0, msg var_x = inp shape_x = var_x.get_shape().as_list() var_f = Conv2D( n_c // squeeze_factor, 1, kernel_regularizer=regularizers.l2(GAN22_REGULARIZER))(var_x) var_g = Conv2D( n_c // squeeze_factor, 1, kernel_regularizer=regularizers.l2(GAN22_REGULARIZER))(var_x) var_h = Conv2D( n_c, 1, kernel_regularizer=regularizers.l2(GAN22_REGULARIZER))(var_x) shape_f = var_f.get_shape().as_list() shape_g = var_g.get_shape().as_list() shape_h = var_h.get_shape().as_list() flat_f = Reshape((-1, shape_f[-1]))(var_f) flat_g = Reshape((-1, shape_g[-1]))(var_g) flat_h = Reshape((-1, shape_h[-1]))(var_h) var_s = Lambda(lambda var_x: K.batch_dot(var_x[0], Permute((2, 1))(var_x[1])))( [flat_g, flat_f]) beta = Softmax(axis=-1)(var_s) var_o = Lambda(lambda var_x: K.batch_dot(var_x[0], var_x[1]))( [beta, flat_h]) var_o = Reshape(shape_x[1:])(var_o) var_o = Scale()(var_o) out = add([var_o, inp]) return out
def get_cnn1d_tx(output_size, img_height, img_width, weight_decay=1e-4, show=True): # value with same Tx would use in same channel model_input = Input(shape=(img_height * img_width, ), name='Main_input') x = Reshape((img_height, img_width), name='Reshape_1')(model_input) x = Permute((2, 1), name='Permute_1')(x) x = Conv1D(filters=256, kernel_size=3, strides=1, padding='same', activation='selu', kernel_initializer='lecun_normal', bias_initializer='zeros', kernel_regularizer=l2(weight_decay), name='Conv1D_selu_1')(x) x = Conv1D(filters=128, kernel_size=3, strides=1, padding='same', activation='selu', kernel_initializer='lecun_normal', bias_initializer='zeros', kernel_regularizer=l2(weight_decay), name='Conv1D_selu_2')(x) x = Conv1D(filters=64, kernel_size=3, strides=1, padding='same', activation='selu', kernel_initializer='lecun_normal', bias_initializer='zeros', kernel_regularizer=l2(weight_decay), name='Conv1D_selu_3')(x) x = Conv1D(filters=32, kernel_size=3, strides=1, padding='same', activation='selu', kernel_initializer='lecun_normal', bias_initializer='zeros', kernel_regularizer=l2(weight_decay), name='Conv1D_selu_4')(x) x = Conv1D(filters=16, kernel_size=3, strides=1, padding='same', activation='selu', kernel_initializer='lecun_normal', bias_initializer='zeros', kernel_regularizer=l2(weight_decay), name='Conv1D_selu_5')(x) x = Flatten(name='Flatten_1')(x) x = Dense(512, activation='selu', kernel_initializer='lecun_normal', bias_initializer='zeros', kernel_regularizer=l2(weight_decay), name='Dense_selu_1')(x) x = Dense(512, activation='selu', kernel_initializer='lecun_normal', bias_initializer='zeros', kernel_regularizer=l2(weight_decay), name='Dense_selu_2')(x) cnn1d_output = Dense(output_size, activation='linear', name='Output_Dense_linear')(x) cnn1d = Model(inputs=model_input, outputs=cnn1d_output, name='CNN1D_Tx') if show: print('CNN1D_Tx summary:') cnn1d.summary() print() return cnn1d
def Unet(inputs, weights, n_classes, input_height, input_width): x = tf.reshape(inputs, shape=[-1, input_height, input_width, 3]) w, b, s = get_weights_biases_scale(weights, 'conv2d_1/kernel:0', 'conv2d_1/bias:0') conv1 = conv_2d(x, w, b, s, activation='relu') w, b, s = get_weights_biases_scale(weights, 'conv2d_2/kernel:0', 'conv2d_2/bias:0') conv1 = conv_2d(conv1, w, b, s, activation='relu') pool1 = maxpool_2d(conv1) w, b, s = get_weights_biases_scale(weights, 'conv2d_3/kernel:0', 'conv2d_3/bias:0') conv2 = conv_2d(pool1, w, b, s, activation='relu') w, b, s = get_weights_biases_scale(weights, 'conv2d_4/kernel:0', 'conv2d_4/bias:0') conv2 = conv_2d(conv2, w, b, s, activation='relu') pool2 = maxpool_2d(conv2) w, b, s = get_weights_biases_scale(weights, 'conv2d_5/kernel:0', 'conv2d_5/bias:0') conv3 = conv_2d(pool2, w, b, s, activation='relu') w, b, s = get_weights_biases_scale(weights, 'conv2d_6/kernel:0', 'conv2d_6/bias:0') conv3 = conv_2d(conv3, w, b, s, activation='relu') pool3 = maxpool_2d(conv3) w, b, s = get_weights_biases_scale(weights, 'conv2d_7/kernel:0', 'conv2d_7/bias:0') conv4 = conv_2d(pool3, w, b, s, activation='relu') w, b, s = get_weights_biases_scale(weights, 'conv2d_8/kernel:0', 'conv2d_8/bias:0') conv4 = conv_2d(conv4, w, b, s, activation='relu') pool4 = maxpool_2d(conv4) w, b, s = get_weights_biases_scale(weights, 'conv2d_9/kernel:0', 'conv2d_9/bias:0') conv5 = conv_2d(pool4, w, b, s, activation='relu') w, b, s = get_weights_biases_scale(weights, 'conv2d_10/kernel:0', 'conv2d_10/bias:0') conv5 = conv_2d(conv5, w, b, s, activation='relu') pool5 = maxpool_2d(conv5) up6 = UpSampling2D(size=(2, 2))(pool5) up6 = tf.concat([up6, conv5], axis=-1) w, b, s = get_weights_biases_scale(weights, 'conv2d_11/kernel:0', 'conv2d_11/bias:0') conv6 = conv_2d(up6, w, b, s, activation='relu') w, b, s = get_weights_biases_scale(weights, 'conv2d_12/kernel:0', 'conv2d_12/bias:0') conv6 = conv_2d(conv6, w, b, s, activation='relu') up7 = UpSampling2D(size=(2, 2))(conv6) up7 = tf.concat([up7, conv4], axis=-1) w, b, s = get_weights_biases_scale(weights, 'conv2d_13/kernel:0', 'conv2d_13/bias:0') conv7 = conv_2d(up7, w, b, s, activation='relu') w, b, s = get_weights_biases_scale(weights, 'conv2d_14/kernel:0', 'conv2d_14/bias:0') conv7 = conv_2d(conv7, w, b, s, activation='relu') up8 = UpSampling2D(size=(2, 2))(conv7) up8 = tf.concat([up8, conv3], axis=-1) w, b, s = get_weights_biases_scale(weights, 'conv2d_15/kernel:0', 'conv2d_15/bias:0') conv8 = conv_2d(up8, w, b, s, activation='relu') w, b, s = get_weights_biases_scale(weights, 'conv2d_16/kernel:0', 'conv2d_16/bias:0') conv8 = conv_2d(conv8, w, b, s, activation='relu') up9 = UpSampling2D(size=(2, 2))(conv8) up9 = tf.concat([up9, conv2], axis=-1) w, b, s = get_weights_biases_scale(weights, 'conv2d_17/kernel:0', 'conv2d_17/bias:0') conv9 = conv_2d(up9, w, b, s, activation='relu') w, b, s = get_weights_biases_scale(weights, 'conv2d_18/kernel:0', 'conv2d_18/bias:0') conv9 = conv_2d(conv9, w, b, s, activation='relu') up10 = UpSampling2D(size=(2, 2))(conv9) up10 = tf.concat([up10, conv1], axis=-1) w, b, s = get_weights_biases_scale(weights, 'conv2d_19/kernel:0', 'conv2d_19/bias:0') conv10 = conv_2d(up10, w, b, s, activation='relu') w, b, s = get_weights_biases_scale(weights, 'conv2d_20/kernel:0', 'conv2d_20/bias:0') conv10 = conv_2d(conv10, w, b, s, activation='relu') w, b, s = get_weights_biases_scale(weights, 'conv2d_21/kernel:0', 'conv2d_21/bias:0') conv11 = conv_2d(conv10, w, b, s, activation='relu') conv11 = tf.reshape(conv11, shape=[n_classes, input_height * input_width]) conv11 = Permute((2, 1))(conv11) return conv11
def spread_axis(input, filters, permute_dims): gather = Reshape((FOUR * filters,))(input) repeat = RepeatVector(FOUR * FOUR)(gather) spread = Reshape((FOUR, FOUR, FOUR, filters))(repeat) permute = Permute(permute_dims)(spread) return permute
def build_model(self): """Helper method for creating the model""" vocab = set() for story, q, answer in self.train_stories + self.test_stories: vocab |= set(story + q + [answer]) vocab = sorted(vocab) # Reserve 0 for masking via pad_sequences vocab_size = len(vocab) + 1 story_maxlen = max( len(x) for x, _, _ in self.train_stories + self.test_stories) query_maxlen = max( len(x) for _, x, _ in self.train_stories + self.test_stories) word_idx = {c: i + 1 for i, c in enumerate(vocab)} self.inputs_train, self.queries_train, self.answers_train = ( vectorize_stories(word_idx, story_maxlen, query_maxlen, self.train_stories)) self.inputs_test, self.queries_test, self.answers_test = ( vectorize_stories(word_idx, story_maxlen, query_maxlen, self.test_stories)) # placeholders input_sequence = Input((story_maxlen, )) question = Input((query_maxlen, )) # encoders # embed the input sequence into a sequence of vectors input_encoder_m = Sequential() input_encoder_m.add(Embedding(input_dim=vocab_size, output_dim=64)) input_encoder_m.add(Dropout(self.config.get("dropout", 0.3))) # output: (samples, story_maxlen, embedding_dim) # embed the input into a sequence of vectors of size query_maxlen input_encoder_c = Sequential() input_encoder_c.add( Embedding(input_dim=vocab_size, output_dim=query_maxlen)) input_encoder_c.add(Dropout(self.config.get("dropout", 0.3))) # output: (samples, story_maxlen, query_maxlen) # embed the question into a sequence of vectors question_encoder = Sequential() question_encoder.add( Embedding(input_dim=vocab_size, output_dim=64, input_length=query_maxlen)) question_encoder.add(Dropout(self.config.get("dropout", 0.3))) # output: (samples, query_maxlen, embedding_dim) # encode input sequence and questions (which are indices) # to sequences of dense vectors input_encoded_m = input_encoder_m(input_sequence) input_encoded_c = input_encoder_c(input_sequence) question_encoded = question_encoder(question) # compute a "match" between the first input vector sequence # and the question vector sequence # shape: `(samples, story_maxlen, query_maxlen)` match = dot([input_encoded_m, question_encoded], axes=(2, 2)) match = Activation("softmax")(match) # add the match matrix with the second input vector sequence response = add([match, input_encoded_c ]) # (samples, story_maxlen, query_maxlen) response = Permute( (2, 1))(response) # (samples, query_maxlen, story_maxlen) # concatenate the match matrix with the question vector sequence answer = concatenate([response, question_encoded]) # the original paper uses a matrix multiplication. # we choose to use a RNN instead. answer = LSTM(32)(answer) # (samples, 32) # one regularization layer -- more would probably be needed. answer = Dropout(self.config.get("dropout", 0.3))(answer) answer = Dense(vocab_size)(answer) # (samples, vocab_size) # we output a probability distribution over the vocabulary answer = Activation("softmax")(answer) # build the final model model = Model([input_sequence, question], answer) return model
def default_classification_model(num_classes, num_anchors, pyramid_feature_size=256, prior_probability=0.01, classification_feature_size=256, frames_per_batch=1, name='classification_submodel'): """Creates the default regression submodel. Args: num_classes (int): Number of classes to predict a score for at each feature level. num_anchors (int): Number of anchors to predict classification scores for at each feature level. pyramid_feature_size (int): The number of filters to expect from the feature pyramid levels. prior_probability (float): the prior probability classification_feature_size (int): The number of filters to use in the layers in the classification submodel. frames_per_batch (int): Size of z axis in generated batches. If equal to 1, assumes 2D data. name (str): The name of the submodel. Returns: tensorflow.keras.Model: A model that predicts classes for each anchor. """ time_distributed = frames_per_batch > 1 options = { 'kernel_size': (3, 3, 3) if time_distributed else 3, 'strides': 1, 'padding': 'same', } shape = [None] * (4 if time_distributed else 3) if K.image_data_format() == 'channels_first': shape[0] = pyramid_feature_size else: shape[-1] = pyramid_feature_size inputs = Input(shape=shape) outputs = inputs conv = Conv3D if time_distributed else Conv2D for i in range(4): outputs = conv(filters=classification_feature_size, activation='relu', name='pyramid_classification_{}'.format(i), kernel_initializer=RandomNormal(mean=0.0, stddev=0.01, seed=None), bias_initializer='zeros', **options)(outputs) outputs = conv( filters=num_classes * num_anchors, kernel_initializer=RandomNormal(mean=0.0, stddev=0.01, seed=None), bias_initializer=PriorProbability(probability=prior_probability), name='pyramid_classification', **options)(outputs) # reshape output and apply sigmoid if K.image_data_format() == 'channels_first': rank = 4 if time_distributed else 3 perm = tuple(list(range(2, rank + 1)) + [1]) outputs = Permute(perm, name='pyramid_classification_permute')(outputs) new_shape = (frames_per_batch, -1, num_classes) if not time_distributed: new_shape = new_shape[1:] outputs = Reshape(new_shape, name='pyramid_classification_reshape')(outputs) outputs = Activation('sigmoid', name='pyramid_classification_sigmoid')(outputs) return Model(inputs=inputs, outputs=outputs, name=name)
def default_regression_model(num_values, num_anchors, pyramid_feature_size=256, regression_feature_size=256, frames_per_batch=1, name='regression_submodel'): """Creates the default regression submodel. Args: num_values (int): Number of values to regress. num_anchors (int): Number of anchors to regress for each feature level. pyramid_feature_size (int): The number of filters to expect from the feature pyramid levels. regression_feature_size (int): The number of filters to use in the layers in the regression submodel. frames_per_batch (int): Size of z axis in generated batches. If equal to 1, assumes 2D data. name (str): The name of the submodel. Returns: tensorflow.keras.Model: A model that predicts regression values for each anchor. """ # All new conv layers except the final one in the # RetinaNet (classification) subnets are initialized # with bias b = 0 and a Gaussian weight fill with stddev = 0.01. time_distributed = frames_per_batch > 1 options = { 'kernel_size': (3, 3, 3) if time_distributed else 3, 'strides': 1, 'padding': 'same', 'kernel_initializer': RandomNormal(mean=0.0, stddev=0.01, seed=None), 'bias_initializer': 'zeros' } shape = [None] * (4 if time_distributed else 3) if K.image_data_format() == 'channels_first': shape[0] = pyramid_feature_size else: shape[-1] = pyramid_feature_size inputs = Input(shape=shape) outputs = inputs conv = Conv3D if time_distributed else Conv2D for i in range(4): outputs = conv(filters=regression_feature_size, activation='relu', name='pyramid_regression_{}'.format(i), **options)(outputs) outputs = conv(num_anchors * num_values, name='pyramid_regression', **options)(outputs) if K.image_data_format() == 'channels_first': rank = 4 if time_distributed else 3 perm = tuple(list(range(2, rank + 1)) + [1]) outputs = Permute(perm, name='pyramid_regression_permute')(outputs) new_shape = (frames_per_batch, -1, num_values) if not time_distributed: new_shape = new_shape[1:] outputs = Reshape(new_shape, name='pyramid_regression_reshape')(outputs) return Model(inputs=inputs, outputs=outputs, name=name)
sequence_input = Input(shape=(data.shape[1], ), dtype='int32') # (Batch size, embedded_sequences = embedding_layer(sequence_input) x = Bidirectional(LSTM(UNITS, return_sequences=True, dropout=DROP, activity_regularizer=k.regularizers.l2(REG)), merge_mode='concat')( embedded_sequences) # (batch_size, timesteps, units) a = TimeDistributed(Dense(UNITS, activity_regularizer=k.regularizers.l2(REG)))(x) attention = TimeDistributed(Dense(1, activation='tanh', name='timeDense'))( a) # (batch_size, timesteps, 1) attention = Flatten()(attention) # (batch size, timesteps) attention = Activation('softmax')(attention) # (batch, timesteps) attention = RepeatVector(UNITS * 2)(attention) # (batch, units, timesteps) attention = Permute([2, 1])(attention) #(batch, timesteps, units) rejoined = multiply([x, attention]) # rejoined = k.backend.sum(rejoined, axis=-2 , keepdims=False)(rejoined) # x = LSTM(UNITS, return_sequences=True, dropout=DROP, activity_regularizer=k.regularizers.l2(REG))(rejoined) # (batch_size, timesteps, units) # x = TimeDistributed(Dense(UNITS, activation='relu', activity_regularizer=k.regularizers.l2(REG)))(x) # attention = TimeDistributed(Dense(1, activation='tanh', name='timeDense'))(x) # (batch_size, timesteps, 1) # attention = Flatten()(attention) # (batch size, timesteps) # attention = Activation('softmax')(attention) # (batch, timesteps) # attention = RepeatVector(UNITS)(attention) # (batch, units, timesteps) # attention = Permute([2,1])(attention) #(batch, timesteps, units) # rejoined = multiply([x, attention]) interm = LSTM(UNITS, activity_regularizer=k.regularizers.l2(REG), dropout=DROP)(rejoined) interm = Dense(UNITS, activation='relu',