def create_cnn_model1(): model = Sequential() # model.add(layers.Flatten(input_shape=(3000, 1))) model.add(layers.Input(shape=(3000, 1))) model.add( layers.Convolution1D(16, kernel_size=5, activation=activations.relu, padding="valid")) model.add( layers.Convolution1D(16, kernel_size=5, activation=activations.relu, padding="valid")) model.add(layers.MaxPool1D(pool_size=2)) model.add( layers.Convolution1D(32, kernel_size=3, activation=activations.relu, padding="valid")) model.add( layers.Convolution1D(32, kernel_size=3, activation=activations.relu, padding="valid")) # model.add(layers.Convolution1D(32, kernel_size=3, activation=activations.relu, padding="valid")) model.add(layers.MaxPool1D(pool_size=2)) model.add(layers.SpatialDropout1D(rate=0.01)) model.add( layers.Convolution1D(32, kernel_size=3, activation=activations.relu, padding="valid")) model.add( layers.Convolution1D(32, kernel_size=3, activation=activations.relu, padding="valid")) model.add(layers.MaxPool1D(pool_size=2)) model.add(layers.SpatialDropout1D(rate=0.01)) model.add( layers.Convolution1D(256, kernel_size=3, activation=activations.relu, padding="valid")) model.add( layers.Convolution1D(256, kernel_size=3, activation=activations.relu, padding="valid")) model.add(layers.GlobalMaxPool1D()) # model.add(layers.Flatten()) model.add(Dense(64, activation='relu')) model.add(Dense(32, activation='relu')) model.add(Dense(5, activation=activations.softmax)) model.compile(optimizer=optimizers.Adam(0.001), loss=losses.sparse_categorical_crossentropy, metrics=['acc']) # model.summary() return model
def prediction_network(fc): fc = layers.Conv1D(16, [7], padding='same', activation=activation)(fc) fc = layers.SpatialDropout1D(rate=0.3)(fc) fc = layers.Conv1D(8, [7], padding='same', activation=activation)(fc) fc = layers.SpatialDropout1D(rate=0.3)(fc) fc = layers.Conv1D(1, [7], padding='same')(fc) return fc
def cnn_base(): model = Sequential(layers=[ layers.Convolution1D(16, kernel_size=5, activation='relu', padding='valid', input_shape=(3000, 1)), layers.Convolution1D( 16, kernel_size=5, activation='relu', padding='valid'), layers.MaxPool1D(pool_size=2), layers.SpatialDropout1D(rate=0.01), layers.Convolution1D( 32, kernel_size=3, activation='relu', padding='valid'), layers.Convolution1D( 32, kernel_size=3, activation='relu', padding='valid'), layers.MaxPool1D(pool_size=2), layers.SpatialDropout1D(rate=0.01), layers.Convolution1D( 32, kernel_size=3, activation='relu', padding='valid'), layers.Convolution1D( 32, kernel_size=3, activation='relu', padding='valid'), layers.MaxPool1D(pool_size=2), layers.Convolution1D( 256, kernel_size=3, activation='relu', padding='valid'), layers.Convolution1D( 256, kernel_size=3, activation='relu', padding='valid'), layers.GlobalMaxPool1D(), layers.Dropout(rate=0.01), layers.Dense(64, activation='relu'), ]) model.compile(optimizer=optimizers.Adam(0.001), loss=losses.sparse_categorical_crossentropy, metrics=['acc']) #,class_model='categorical' return model
def __init__(self, embed_dim, num_heads, ff_dim, rate=0.1): super(TransformerBlock, self).__init__() self.att = MultiHeadSelfAttention(embed_dim, num_heads) self.ffn = keras.Sequential([ layers.Dense(ff_dim, activation="relu"), layers.Dense(embed_dim), ]) self.layernorm1 = layers.LayerNormalization(epsilon=1e-6) self.layernorm2 = layers.LayerNormalization(epsilon=1e-6) self.dropout1 = layers.SpatialDropout1D(rate) self.dropout2 = layers.SpatialDropout1D(rate)
def __init__(self, dilation_rate, nb_filters, kernel_size, padding, dropout_rate=0.0, conv_regularization=0.08): """ Defines the residual block for TCN :param x: The previous layer in the model :param dilation_rate: The dilation rate for this residual block :param nb_filters: The number of convolutional filters to use in this block :param kernel_size: The size of the convolutional kernel :param padding: The padding used in the convolutional layers, 'same' or 'causal'. :param dropout_rate: Float between 0 and 1. Fraction of the input units to drop. :param conv_regularization: L2 regularization coefficient. :return: A tuple where the first element is the residual model layer, and the second is the skip connection. """ super(ResidualBlock, self).__init__() self.dilation_rate = dilation_rate self.nb_filters = nb_filters self.kernel_size = kernel_size self.padding = padding self.dropout_rate = dropout_rate self.conv_regularization = conv_regularization self.con1 = layers.SeparableConv1D( filters=self.nb_filters, kernel_size=self.kernel_size, dilation_rate=self.dilation_rate, padding=self.padding, kernel_regularizer=regularizers.l2(conv_regularization), kernel_initializer=initializers.RandomNormal(mean=0.0, stddev=0.01)) self.dropout1 = layers.SpatialDropout1D(self.dropout_rate) self.con2 = layers.SeparableConv1D( filters=self.nb_filters, kernel_size=self.kernel_size, dilation_rate=self.dilation_rate, padding=self.padding, kernel_regularizer=regularizers.l2(conv_regularization), kernel_initializer=initializers.RandomNormal(mean=0.0, stddev=0.01)) self.dropout2 = layers.SpatialDropout1D(self.dropout_rate) self.conv_matching = layers.Conv1D( filters=nb_filters, kernel_size=1, padding='same', kernel_initializer=initializers.RandomNormal(mean=0.0, stddev=0.01))
def get_baseline_convolutional_encoder(filters, embedding_dimension, input_shape=None, dropout=0.05): encoder = Sequential() # Initial conv if input_shape is None: # In this case we are using the encoder as part of a siamese network and the input shape will be determined # automatically based on the input shape of the siamese network encoder.add( layers.Conv1D(filters, 32, padding='same', activation='relu')) else: # In this case we are using the encoder to build a classifier network and the input shape must be defined encoder.add( layers.Conv1D(filters, 32, padding='same', activation='relu', input_shape=input_shape)) encoder.add(layers.BatchNormalization()) encoder.add(layers.SpatialDropout1D(dropout)) encoder.add(layers.MaxPool1D(4, 4)) # Further convs encoder.add( layers.Conv1D(2 * filters, 3, padding='same', activation='relu')) encoder.add(layers.BatchNormalization()) encoder.add(layers.SpatialDropout1D(dropout)) encoder.add(layers.MaxPool1D()) encoder.add( layers.Conv1D(3 * filters, 3, padding='same', activation='relu')) encoder.add(layers.BatchNormalization()) encoder.add(layers.SpatialDropout1D(dropout)) encoder.add(layers.MaxPool1D()) encoder.add( layers.Conv1D(4 * filters, 3, padding='same', activation='relu')) encoder.add(layers.BatchNormalization()) encoder.add(layers.SpatialDropout1D(dropout)) encoder.add(layers.MaxPool1D()) encoder.add(layers.GlobalMaxPool1D()) encoder.add(layers.Dense(embedding_dimension)) return encoder
def create_model(data, catcols): inputs = [] outputs = [] for c in catcols: num_unique_values = int(data[c].nunique()) embed_dim = int(min(np.ceil((num_unique_values)/2), 50)) inp = layers.Input(shape=(1, )) out = layers.Embedding(num_unique_values + 1, embed_dim, name=c)(inp) out = layers.SpatialDropout1D(0.3)(out) out = layers.Reshape(target_shape=(embed_dim, ))(out) inputs.append(inp) outputs.append(out) x = layers.Concatenate()(outputs) x = layers.BatchNormalization()(x) x = layers.Dense(300, activation='relu')(x) x = layers.Dropout(0.3)(x) x = layers.BatchNormalization()(x) x = layers.Dense(300, activation='relu')(x) x = layers.Dropout(0.3)(x) x = layers.BatchNormalization()(x) y = layers.Dense(2, activation='softmax')(x) model = Model(inputs=inputs, outputs=y) return model
def build_transactions_rnn(transactions_cat_features, embedding_projections, product_col_name='product', rnn_units=128, classifier_units=32, optimizer=None): if not optimizer: optimizer = keras.optimizers.Adam(lr=1e-3) inputs = [] cat_embeds = [] for feature_name in transactions_cat_features: inp = L.Input(shape=(None, ), dtype='uint32', name=f'input_{feature_name}') inputs.append(inp) source_size, projection = embedding_projections[feature_name] emb = L.Embedding(source_size + 1, projection, trainable=True, mask_zero=False, name=f'embedding_{feature_name}')(inp) cat_embeds.append(emb) # product feature inp = L.Input(shape=(1, ), dtype='uint32', name=f'input_product') inputs.append(inp) source_size, projection = embedding_projections['product'] product_emb = L.Embedding(source_size + 1, projection, trainable=True, mask_zero=False, name=f'embedding_product')(inp) product_emb_reshape = L.Reshape((projection, ))(product_emb) concated_cat_embeds = L.concatenate(cat_embeds) dropout_embeds = L.SpatialDropout1D(0.05)(concated_cat_embeds) sequences = L.Bidirectional(L.GRU(units=rnn_units, return_sequences=True))(dropout_embeds) pooled_avg_sequences = L.GlobalAveragePooling1D()(sequences) pooled_max_sequences = L.GlobalMaxPooling1D()(sequences) #add dropout=0.5 concated = L.concatenate( [pooled_avg_sequences, pooled_max_sequences, product_emb_reshape]) dense_intermediate = L.Dense(classifier_units, activation='relu', kernel_regularizer=keras.regularizers.L1L2( 1e-7, 1e-5))(concated) proba = L.Dense(1, activation='sigmoid')(dense_intermediate) model = Model(inputs=inputs, outputs=proba) model.compile(loss='binary_crossentropy', optimizer=optimizer) return model
def create_keras_embedding_model(data, cat_cols, N_uniq=50, N_dense=300, drop_rate=0.3): inputs = [] outputs = [] for col in cat_cols: n_unique = int(data[col].nunique()) embed_dim = int(min(np.ceil(n_unique / 2), N_uniq)) inp = layers.Input(shape=(1, )) out = layers.Embedding(n_unique + 1, embed_dim, name=col)(inp) out = layers.SpatialDropout1D(drop_rate)(out) out = layers.Reshape(target_shape=(embed_dim, ))(out) inputs.append(inp) outputs.append(out) x = layers.Concatenate()(outputs) x = layers.BatchNormalization()(x) x = layers.Dense(N_dense, activation='relu')(x) x = layers.Dropout(drop_rate)(x) x = layers.BatchNormalization()(x) x = layers.Dense(N_dense, activation='relu')(x) x = layers.Dropout(drop_rate)(x) x = layers.BatchNormalization()(x) y = layers.Dense(2, activation='softmax')(x) model = Model(inputs=inputs, outputs=y) return model
def get_stweet_model(): path = "/content/drive/My Drive/Datasets/" embedding_matrix = pickle.load( open(path + 'embedding_matrix_tweet.pl', 'rb')) embedding_layer = tf.keras.layers.Embedding(290575, 300, weights=[embedding_matrix], input_length=30, trainable=False) sequence_input = layers.Input(shape=(30, ), dtype='int32') embedding_sequences = embedding_layer(sequence_input) x = layers.SpatialDropout1D(0.2)(embedding_sequences) x = layers.Conv1D(64, 5, activation='relu')(x) x = layers.Bidirectional( layers.LSTM(64, dropout=0.2, recurrent_dropout=0.2))(x) x = layers.Dense(512, activation='relu')(x) x = layers.Dropout(0.5)(x) x = layers.Dense(512, activation='relu')(x) outputs = layers.Dense(2, activation='softmax')(x) model = tf.keras.Model(sequence_input, outputs) model.compile(optimizer='adam', loss='CategoricalCrossentropy', metrics=['accuracy']) return model
def model_rnn(kwargs): K.clear_session() nn_input = input_[kwargs["input_layer"]](**kwargs["input_params"]) x = embeddings[kwargs["emb_layer"]](**kwargs["emb_params"])(nn_input) x = layers.SpatialDropout1D(0.1)(x) x = layers.Bidirectional(layers.GRU(32, dropout=0.3, recurrent_dropout=0.3, return_sequences=True))(x) x = layers.Bidirectional(layers.GRU(16, dropout=0.4, recurrent_dropout=0.4, return_sequences=True))(x) avg_pool = layers.GlobalAveragePooling1D()(x) max_pool = layers.GlobalMaxPooling1D()(x) x = layers.concatenate([avg_pool, max_pool]) x = layers.Dropout(0.1)(x) x = layers.Dense(100)(x) x = layers.BatchNormalization()(x) x = layers.Activation("relu")(x) x = layers.Dropout(0.3)(x) nn_pred = layers.Dense(kwargs["out_units"], activation=kwargs["out_activation"])(x) model = Model(inputs=nn_input, outputs=nn_pred) model.compile( loss=kwargs["loss"], optimizer=kwargs["optimizer"], metrics=["accuracy"] ) sess = K.get_session() init = tf.global_variables_initializer() sess.run(init) return model
def create_cnn(total_words=1000, embedded_dimension=300, embedding_matrix=None, input_length=100, optimizer='adam'): # Add an Input Layer input_layer = layers.Input((input_length, )) # Add the word embedding Layer embedding_layer = layers.Embedding(total_words + 1, embedded_dimension, weights=[embedding_matrix], trainable=False)(input_layer) embedding_layer = layers.SpatialDropout1D(0.5)(embedding_layer) # Add the convolutional Layer conv_layer = layers.Convolution1D(100, 3, activation="relu")(embedding_layer) # Add the pooling Layer pooling_layer = layers.GlobalMaxPool1D()(conv_layer) # Add the output Layers output_layer1 = layers.Dense(50, activation="relu")(pooling_layer) output_layer1 = layers.Dropout(0.6)(output_layer1) output_layer2 = layers.Dense(1, activation="sigmoid")(output_layer1) # Compile the model model = models.Model(inputs=input_layer, outputs=output_layer2) model.compile(optimizer=optimizer, loss='binary_crossentropy') return model
def __init__(self, cfg, verbose=False): super(ConvRec, self).__init__(cfg, verbose) self.conv1d = layers.Conv1D(filters=cfg.model_conv1d_filters, kernel_size=3, padding='same', activation='relu', use_bias=(not cfg.model_bn_in), name='conv1d') self.dropout = layers.SpatialDropout1D(cfg.model_dropout, name='do1d') RNN = layers.GRU if cfg.model_rec_type == 'gru' else layers.LSTM ret_seqs = [True] * cfg.model_rec_layers ret_seqs[-1] = False self.recs = [] for i, ret_seq in enumerate(ret_seqs): if cfg.model_rec_bi: rec = layers.Bidirectional(RNN(units=cfg.model_rec_size, return_sequences=ret_seq, name=f'rec{i}'), merge_mode=cfg.model_rec_bi_merge) else: rec = RNN(units=cfg.model_rec_size, return_sequences=ret_seq, name=f'rec{i}') self.recs.append(rec) setattr(self, f'rec{i}', rec)
def __init__(self, cfg, num_classes, verbose=False): super(FullConv, self).__init__() self.verbose = verbose div = 1 self.pad = layers.ZeroPadding2D(padding=(1, 0), name='zpad2d') self.conv2d = layers.Conv2D(filters=cfg.conv2d_filters, kernel_size=(3, cfg.reps_size), padding='valid', activation='relu', name='conv2d') self.dropout = layers.SpatialDropout1D(cfg.dropout, name='do1d') self.conv1d1 = layers.Conv1D(filters=cfg.conv1d_filters, kernel_size=3, padding='same', activation='relu', name='conv1d1') self.conv1d2 = layers.Conv1D(filters=cfg.conv1d_filters, kernel_size=3, padding='same', activation='relu', name='conv1d2') self.conv1d3 = layers.Conv1D(filters=cfg.conv1d_filters, kernel_size=3, padding='same', activation='relu', name='conv1d3') self.pool1d = layers.MaxPool1D(pool_size=2, name='mpool1d') self.flat = layers.Flatten(name='flat') self.fc = layers.Dense(num_classes, activation='softmax', name='fc')
def create_cnn(model_params): """ This function creates a Deep Convolutional Network based on model_params dictionary :param model_params: dict of model_params :return: keras model """ # Add an Input Layer, expected vectors of 0 and 1, with one vector for each word input_layer = layers.Input((70, )) # Add the word embedding Layer embedding_layer = layers.Embedding(len(word_index) + 1, 300, weights=[embedding_matrix], trainable=True)(input_layer) embedding_layer = layers.SpatialDropout1D( model_params['spatial_dropout'])(embedding_layer) # Add the convolutional Layer for ly in range(model_params['num_conv_blocks']): if ly == 0: conv_layer = layers.Convolution1D( model_params['num_conv_filters'], model_params['filter_size'], activation=model_params['activation_func'])(embedding_layer) else: conv_layer = layers.Convolution1D( model_params['num_conv_filters'] * ly * 2, model_params['filter_size'], activation=model_params['activation_func'])(conv_layer) # Add the pooling Layer pooling_layer = layers.GlobalMaxPool1D()(conv_layer) # Add the output Layers for ly in range(model_params['num_dense_layers']): if ly == 0: output_layer1 = layers.Dense( model_params['num_dense_neurons'], activation=model_params['activation_func'])(pooling_layer) output_layer1 = layers.Dropout( model_params['dense_dropout'])(output_layer1) else: output_layer1 = layers.Dense( model_params['num_dense_neurons'], activation=model_params['activation_func'])(output_layer1) output_layer1 = layers.Dropout( model_params['dense_dropout'])(output_layer1) output_layer2 = layers.Dense(1, activation="sigmoid")(output_layer1) # Compile the model model = models.Model(inputs=input_layer, outputs=output_layer2) model.compile(optimizer=optimizers.Adam( lr=model_params['learning_rate'], decay=model_params['learning_rate'] / model_params['epochs']), loss='binary_crossentropy', metrics=['accuracy']) return model
def LSTM(sequence_length, embedding_layer): return keras.Sequential([ keras.Input(shape=(sequence_length, )), embedding_layer, layers.SpatialDropout1D(0.3), layers.LSTM(100, return_sequences=True), layers.GlobalMaxPool1D(), layers.Dense(50, activation="relu"), layers.Dropout(0.2), layers.Dense(6, activation="sigmoid") ])
def __init__(self, cfg, verbose=False): super(Conv2D, self).__init__(cfg, verbose) self.pad = layers.ZeroPadding2D(padding=(1, 0), name='zpad2d') self.conv2d = layers.Conv2D(filters=cfg.model_conv2d_filters, kernel_size=(3, 512), padding='valid', activation='relu', use_bias=(not cfg.model_bn_in), name='conv2d') self.dropout = layers.SpatialDropout1D(cfg.model_dropout, name='do1d') self.gap = layers.GlobalAveragePooling1D(name='gap')
def BGRU_CNN(sequence_length, embedding_layer): return keras.Sequential([ keras.Input(shape=(sequence_length, )), embedding_layer, layers.SpatialDropout1D(0.3), layers.Bidirectional(layers.GRU(100, return_sequences=True)), layers.Conv1D(100, 3, activation="relu"), layers.GlobalMaxPool1D(), layers.Dense(50, activation="relu"), layers.Dropout(0.2), layers.Dense(6, activation="sigmoid") ])
def cnn_v2(lr=0.005): model = Sequential(layers=[ layers.Convolution1D(128, kernel_size=50, strides=25, activation='relu', padding='valid', input_shape=(3000, 1)), # layers.Convolution1D(128, kernel_size=50, strides=25, activation='relu', padding='valid'), layers.MaxPool1D(pool_size=8, strides=8), layers.SpatialDropout1D(rate=0.1), layers.Convolution1D( 128, kernel_size=8, strides=1, activation='relu', padding='valid'), layers.Convolution1D( 128, kernel_size=8, strides=1, activation='relu', padding='valid'), layers.Convolution1D( 128, kernel_size=8, strides=1, activation='relu', padding='valid'), layers.MaxPool1D(4, 4), layers.SpatialDropout1D(rate=0.5), layers.Flatten(), # layers.Convolution1D(32, kernel_size=3, activation='relu', padding='valid'), # layers.Convolution1D(32, kernel_size=3, activation='relu', padding='valid'), # layers.MaxPool1D(pool_size=2), # layers.SpatialDropout1D(rate=0.1), # layers.Convolution1D(32, kernel_size=3, activation='relu', padding='valid'), # layers.Convolution1D(32, kernel_size=3, activation='relu', padding='valid'), # layers.MaxPool1D(pool_size=2), # layers.Convolution1D(256, kernel_size=3, activation='relu', padding='valid'), # layers.Convolution1D(256, kernel_size=3, activation='relu', padding='valid'), # layers.GlobalMaxPool1D(), # layers.Dropout(rate=0.01), layers.Dense(64, activation='relu'), layers.Dropout(rate=0.1), # layers.Dense(64, activation='relu'), layers.Dropout(rate=0.5), layers.Dense(5, activation='softmax') ]) model.compile(optimizer=optimizers.Adam(lr), loss=losses.sparse_categorical_crossentropy, metrics=['accuracy']) #,class_model='categorical' return model
def keras_model_fn_cpu(model_config, vocab_size, embedding_size, embeddings): """ CPU version of Stacked Bi-LSTM and Bi-GRU with Two Fasttext """ ## hyperparams model_name = model_config['model_name'] num_class = model_config['num_class'] lstm_hs = model_config['lstm_hs'] gru_hs = model_config['gru_hs'] learning_rate = model_config['learning_rate'] with tf.device('/cpu:0'): ## build model inputs = ks.Input(shape=(None, ), dtype='int32', name='inputs') embedded_sequences_ft1 = layers.Embedding(vocab_size, embedding_size, trainable=False, mask_zero=False)(inputs) embedded_sequences_ft2 = layers.Embedding(vocab_size, embedding_size, trainable=False, mask_zero=False)(inputs) concat_embed = layers.concatenate( [embedded_sequences_ft1, embedded_sequences_ft2]) concat_embed = layers.SpatialDropout1D(0.5)(concat_embed) x = layers.Bidirectional( layers.LSTM(lstm_hs, recurrent_activation='sigmoid', return_sequences=True))(concat_embed) x, x_h, x_c = layers.Bidirectional( layers.GRU(gru_hs, reset_after=True, recurrent_activation='sigmoid', return_sequences=True, return_state=True))(x) x_1 = layers.GlobalMaxPool1D()(x) x_2 = layers.GlobalAvgPool1D()(x) x_out = layers.concatenate([x_1, x_2, x_h]) x_out = layers.BatchNormalization()(x_out) outputs = layers.Dense(num_class, activation='softmax', name='outputs')(x_out) # outputs model = ks.Model(inputs, outputs, name=model_name) ## compile model.compile(loss='categorical_crossentropy', optimizer=ks.optimizers.Adam(lr=learning_rate, clipnorm=.25, beta_1=0.7, beta_2=0.99), metrics=[ 'categorical_accuracy', ks.metrics.TopKCategoricalAccuracy(k=3) ]) return model
def __init__(self, cfg, num_classes, verbose=False): super(Conv2D, self).__init__() self.verbose = verbose self.pad = layers.ZeroPadding2D(padding=(1, 0), name='zpad2d') self.conv2d = layers.Conv2D(filters=cfg.conv2d_filters, kernel_size=(3, cfg.reps_size), padding='valid', activation='relu', name='conv2d') self.dropout = layers.SpatialDropout1D(cfg.dropout, name='do1d') self.gap = layers.GlobalAveragePooling1D(name='gap') self.fc = layers.Dense(num_classes, activation='softmax', name='fc')
def __init__(self, cfg, verbose=False): super(FullConv, self).__init__(cfg.batchnorm, verbose) div = 1 self.pad = layers.ZeroPadding2D(padding=(1, 0), name='zpad2d') self.conv2d = layers.Conv2D( filters=cfg.conv2d_filters, kernel_size=(3, cfg.reps_size), padding='valid', activation='relu', name='conv2d') self.dropout1 = layers.SpatialDropout1D(cfg.dropout, name='do1d1') self.conv1d1 = layers.Conv1D( filters=cfg.conv2d_filters//div, kernel_size=3, padding='same', activation='relu', name='conv1d1') self.dropout2 = layers.SpatialDropout1D(cfg.dropout, name='do1d2') self.conv1d2 = layers.Conv1D( filters=cfg.conv2d_filters//div, kernel_size=3, padding='same', activation='relu', name='conv1d2') self.dropout3 = layers.SpatialDropout1D(cfg.dropout, name='do1d3') self.conv1d3 = layers.Conv1D( filters=cfg.conv2d_filters//div, kernel_size=3, padding='same', activation='relu', name='conv1d3') self.dropout4 = layers.SpatialDropout1D(cfg.dropout, name='do1d4') self.pool1d = layers.MaxPool1D(pool_size=2, name='mpool1d') self.flat = layers.Flatten(name='flat')
def __init__(self, filters: int, kernel_size: int, dilation_rate: int, dropout_rate: float, activation: str, **kwargs): super(ResidualBlock, self).__init__(**kwargs) self.filters = filters self.causal_conv_1 = layers.Conv1D(filters=self.filters, kernel_size=kernel_size, dilation_rate=dilation_rate, padding='causal') self.weight_norm_1 = layers.LayerNormalization() self.dropout_1 = layers.SpatialDropout1D(rate=dropout_rate) self.activation_1 = layers.Activation(activation) self.causal_conv_2 = layers.Conv1D(filters=self.filters, kernel_size=kernel_size, dilation_rate=dilation_rate, padding='causal') self.weight_norm_2 = layers.LayerNormalization() self.dropout_2 = layers.SpatialDropout1D(rate=dropout_rate) self.activation_2 = layers.Activation(activation) self.activation_3 = layers.Activation(activation)
def build_model_bpps_Attn(embed_size, seq_len=107, pred_len=68, dropout=0.5, sp_dropout=0.2, embed_dim=200, hidden_dim=256, n_layers=3): input_s = L.Input(shape=(seq_len, 3)) input_bpps = L.Input(shape=(seq_len, 2)) inputs = L.Concatenate(axis=2)([input_s, input_bpps]) embed = L.Embedding(input_dim=embed_size, output_dim=embed_dim)(inputs) reshaped = tf.reshape(embed, shape=(-1, embed.shape[1], embed.shape[2] * embed.shape[3])) hidden = L.SpatialDropout1D(sp_dropout)(reshaped) conv1 = L.Conv1D(128, 64, 1, padding="same", activation=None)(hidden) h1 = L.LayerNormalization()(conv1) h1 = L.LeakyReLU()(h1) conv2 = L.Conv1D(128, 32, 1, padding="same", activation=None)(hidden) h2 = L.LayerNormalization()(conv2) h2 = L.LeakyReLU()(h2) conv3 = L.Conv1D(128, 16, 1, padding="same", activation=None)(hidden) h3 = L.LayerNormalization()(conv3) h3 = L.LeakyReLU()(h3) conv4 = L.Conv1D(128, 8, 1, padding="same", activation=None)(hidden) h4 = L.LayerNormalization()(conv4) h4 = L.LeakyReLU()(h4) hs = L.Concatenate()([h1, h2, h3, h4]) keys = L.Dropout(0.2)(hs) for x in range(n_layers): hidden = gru_layer(hidden_dim, dropout)(hidden) hidden = L.Attention(dropout=0.2)([hidden, keys]) # Since we are only making predictions on the first part of each sequence, # we have to truncate it truncated = hidden[:, :pred_len] out = L.Dense(5, activation='linear')(truncated) model = tf.keras.Model(inputs=[input_s, input_bpps], outputs=out) model.compile(tf.optimizers.Adam(), loss=MCRMSE) return model
def __init__(self, shape, num_lables, max_nb_words=50000, max_sequence_length=250, embedding_dim=100): self.max_sequence_length = max_sequence_length self.tokenizer = Tokenizer(num_words=max_nb_words, filters='!"#$%&()*+,-./:;<=>?@[\]^_`{|}~', lower=True) self.model = keras.Sequential() # Turns positive integers (indexes) into dense vectors of fixed size # NOTE: can only be used as the first layer in a model. self.model.add( layers.Embedding(max_nb_words, embedding_dim, input_length=shape)) # convolutional layer: applies a filter in order to create a feature map that # summarizes the presence of detected features in the input. # we use relu because it was suggested # kernel_size needs to be odd -> (1,3,5,7 are common) self.model.add(layers.Conv1D(64, kernel_size=3, activation="relu")) # Downsamples the input representation by taking the maximum value over the window defined by pool_size. self.model.add(layers.MaxPooling1D()) # The Dropout layer randomly sets input units to 0 # with a frequency rate at each step during training time, which helps prevent overfitting. self.model.add(layers.SpatialDropout1D(0.2)) # Adds a bidirectional LSTM layer. Allows us to read input twice (forwards and backwards) # We use concatenation as the merge mode because it was used in a paper self.model.add( layers.Bidirectional(layers.LSTM(100, dropout=0.2, recurrent_dropout=0.2), merge_mode='concat')) # performs activation calculation # model.add(layers.Dense(4, activation='softmax')) # For detecting all emotions self.model.add(layers.Dense( num_lables, activation='softmax')) # For detecting single emotions self.model.summary() # Configures the model for training. self.model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
def keras_dropout(layer, rate): """ Keras dropout layer. """ from keras import layers input_dim = len(layer.input.shape) if input_dim == 2: return layers.SpatialDropout1D(rate) elif input_dim == 3: return layers.SpatialDropout2D(rate) elif input_dim == 4: return layers.SpatialDropout3D(rate) else: return layers.Dropout(rate)
def __init__(self, cfg, verbose=False): super(Conv, self).__init__(cfg, verbose) self.convs = [] for i, filters in enumerate(cfg.model_conv_filters): use_bias = not cfg.model_bn_in if i == 0 else True conv = layers.Conv1D(filters=filters, kernel_size=3, padding='same', activation='relu', use_bias=use_bias, name=f'conv{i}') self.convs.append(conv) setattr(self, conv.name, conv) self.pool = layers.MaxPool1D(pool_size=2, name='pool') self.dropout = layers.SpatialDropout1D(cfg.model_dropout, name='dropout') self.gap = layers.GlobalAveragePooling1D(name='gap')
def create_model(data, catcols): print(len(catcols)) inputs = [] outputs = [] for c in catcols: print(c) num_unique_values = int(data[c].nunique()) embed_dim = int(min(num_unique_values, 70)) inp = layers.Input(shape=(1, )) print(inp.shape) print(num_unique_values) out = layers.Embedding(num_unique_values + 1, embed_dim, name=c)(inp) out = layers.SpatialDropout1D(0.3)(out) out = layers.Reshape(target_shape=(embed_dim, ))(out) print(out.shape) inputs.append(inp) outputs.append(out) x = layers.Concatenate()(outputs) x = layers.BatchNormalization()(x) x = layers.Dense(512, activation="relu")(x) x = layers.Dropout(0.5)(x) x = layers.BatchNormalization()(x) x = layers.Dense(512, activation="relu")(x) x = layers.Dropout(0.5)(x) x = layers.BatchNormalization()(x) x = layers.Dense(512, activation="relu")(x) x = layers.Dropout(0.4)(x) x = layers.BatchNormalization()(x) x = layers.Dense(512, activation="relu")(x) x = layers.Dropout(0.)(x) x = layers.BatchNormalization()(x) x = layers.Dense(512, activation="relu")(x) x = layers.Dropout(0.3)(x) x = layers.BatchNormalization()(x) y = layers.Dense(2, activation="softmax")(x) model = Model(inputs=inputs, outputs=y) return model
def __init__(self, units, num_heads=1, axis=2, residual=False, l2=None, dropout=0.1, **kwargs): # super(MultiHeadSelfAttention, self).__init__(**kwargs) assert units % num_heads == 0 self.num_heads = num_heads self.keys_dim = units // num_heads self.axis = axis self.residual = residual self.qs_layers = [] self.ks_layers = [] self.vs_layers = [] for _ in range(num_heads): self.qs_layers.append( layers.TimeDistributed( layers.Dense(self.keys_dim, use_bias=False, kernel_regularizer=regularizers.l2(l2)))) self.ks_layers.append( layers.TimeDistributed( layers.Dense(self.keys_dim, use_bias=False, kernel_regularizer=regularizers.l2(l2)))) self.vs_layers.append( layers.TimeDistributed( layers.Dense(self.keys_dim, use_bias=False, kernel_regularizer=regularizers.l2(l2)))) self.attention = ScaledDotProductAttention(dropout=dropout) self.dense = layers.TimeDistributed( layers.Dense(units, activation="relu", kernel_regularizer=regularizers.l2(l2))) self.dropout = layers.SpatialDropout1D(dropout) self.layernorm = LayerNormalization()
def get_ae_model(base, config): ## denoising auto encoder part ## node, adj -> middle feature -> node node = tf.keras.Input(shape=(None, X_node.shape[2]), name="node") adj = tf.keras.Input(shape=(None, None, As.shape[3]), name="adj") x = base([L.SpatialDropout1D(0.4)(node), adj]) x = forward(x, 128, rate=0.3) p = L.Dense(X_node.shape[2], "sigmoid")(x) # loss = - tf.reduce_mean(40 * node * tf.math.log(p + 1e-6) + (1 - node) * tf.math.log(1 - p + 1e-6)) loss = -tf.reduce_mean(node * tf.math.log(p + 1e-6) + (1 - node) * tf.math.log(1 - p + 1e-6)) model = tf.keras.Model(inputs=[node, adj], outputs=[loss]) opt = get_optimizer() model.compile(optimizer=opt, loss=lambda t, y: y) return model