def create_cnn_model1():
model = Sequential()
# model.add(layers.Flatten(input_shape=(3000, 1)))
model.add(layers.Input(shape=(3000, 1)))
model.add(
layers.Convolution1D(16,
kernel_size=5,
activation=activations.relu,
padding="valid"))
model.add(
layers.Convolution1D(16,
kernel_size=5,
activation=activations.relu,
padding="valid"))
model.add(layers.MaxPool1D(pool_size=2))
model.add(
layers.Convolution1D(32,
kernel_size=3,
activation=activations.relu,
padding="valid"))
model.add(
layers.Convolution1D(32,
kernel_size=3,
activation=activations.relu,
padding="valid"))
# model.add(layers.Convolution1D(32, kernel_size=3, activation=activations.relu, padding="valid"))
model.add(layers.MaxPool1D(pool_size=2))
model.add(layers.SpatialDropout1D(rate=0.01))
model.add(
layers.Convolution1D(32,
kernel_size=3,
activation=activations.relu,
padding="valid"))
model.add(
layers.Convolution1D(32,
kernel_size=3,
activation=activations.relu,
padding="valid"))
model.add(layers.MaxPool1D(pool_size=2))
model.add(layers.SpatialDropout1D(rate=0.01))
model.add(
layers.Convolution1D(256,
kernel_size=3,
activation=activations.relu,
padding="valid"))
model.add(
layers.Convolution1D(256,
kernel_size=3,
activation=activations.relu,
padding="valid"))
model.add(layers.GlobalMaxPool1D())
# model.add(layers.Flatten())
model.add(Dense(64, activation='relu'))
model.add(Dense(32, activation='relu'))
model.add(Dense(5, activation=activations.softmax))
model.compile(optimizer=optimizers.Adam(0.001),
loss=losses.sparse_categorical_crossentropy,
metrics=['acc'])
# model.summary()
return model
def prediction_network(fc):
fc = layers.Conv1D(16, [7], padding='same', activation=activation)(fc)
fc = layers.SpatialDropout1D(rate=0.3)(fc)
fc = layers.Conv1D(8, [7], padding='same', activation=activation)(fc)
fc = layers.SpatialDropout1D(rate=0.3)(fc)
fc = layers.Conv1D(1, [7], padding='same')(fc)
return fc
def cnn_base():
model = Sequential(layers=[
layers.Convolution1D(16,
kernel_size=5,
activation='relu',
padding='valid',
input_shape=(3000, 1)),
layers.Convolution1D(
16, kernel_size=5, activation='relu', padding='valid'),
layers.MaxPool1D(pool_size=2),
layers.SpatialDropout1D(rate=0.01),
layers.Convolution1D(
32, kernel_size=3, activation='relu', padding='valid'),
layers.Convolution1D(
32, kernel_size=3, activation='relu', padding='valid'),
layers.MaxPool1D(pool_size=2),
layers.SpatialDropout1D(rate=0.01),
layers.Convolution1D(
32, kernel_size=3, activation='relu', padding='valid'),
layers.Convolution1D(
32, kernel_size=3, activation='relu', padding='valid'),
layers.MaxPool1D(pool_size=2),
layers.Convolution1D(
256, kernel_size=3, activation='relu', padding='valid'),
layers.Convolution1D(
256, kernel_size=3, activation='relu', padding='valid'),
layers.GlobalMaxPool1D(),
layers.Dropout(rate=0.01),
layers.Dense(64, activation='relu'),
])
model.compile(optimizer=optimizers.Adam(0.001),
loss=losses.sparse_categorical_crossentropy,
metrics=['acc']) #,class_model='categorical'
return model
def __init__(self, embed_dim, num_heads, ff_dim, rate=0.1):
super(TransformerBlock, self).__init__()
self.att = MultiHeadSelfAttention(embed_dim, num_heads)
self.ffn = keras.Sequential([
layers.Dense(ff_dim, activation="relu"),
layers.Dense(embed_dim),
])
self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)
self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)
self.dropout1 = layers.SpatialDropout1D(rate)
self.dropout2 = layers.SpatialDropout1D(rate)
def __init__(self,
dilation_rate,
nb_filters,
kernel_size,
padding,
dropout_rate=0.0,
conv_regularization=0.08):
"""
Defines the residual block for TCN
:param x: The previous layer in the model
:param dilation_rate: The dilation rate for this residual block
:param nb_filters: The number of convolutional filters to use in this block
:param kernel_size: The size of the convolutional kernel
:param padding: The padding used in the convolutional layers, 'same' or 'causal'.
:param dropout_rate: Float between 0 and 1. Fraction of the input units to drop.
:param conv_regularization: L2 regularization coefficient.
:return: A tuple where the first element is the residual model layer, and the second is the skip connection.
"""
super(ResidualBlock, self).__init__()
self.dilation_rate = dilation_rate
self.nb_filters = nb_filters
self.kernel_size = kernel_size
self.padding = padding
self.dropout_rate = dropout_rate
self.conv_regularization = conv_regularization
self.con1 = layers.SeparableConv1D(
filters=self.nb_filters,
kernel_size=self.kernel_size,
dilation_rate=self.dilation_rate,
padding=self.padding,
kernel_regularizer=regularizers.l2(conv_regularization),
kernel_initializer=initializers.RandomNormal(mean=0.0,
stddev=0.01))
self.dropout1 = layers.SpatialDropout1D(self.dropout_rate)
self.con2 = layers.SeparableConv1D(
filters=self.nb_filters,
kernel_size=self.kernel_size,
dilation_rate=self.dilation_rate,
padding=self.padding,
kernel_regularizer=regularizers.l2(conv_regularization),
kernel_initializer=initializers.RandomNormal(mean=0.0,
stddev=0.01))
self.dropout2 = layers.SpatialDropout1D(self.dropout_rate)
self.conv_matching = layers.Conv1D(
filters=nb_filters,
kernel_size=1,
padding='same',
kernel_initializer=initializers.RandomNormal(mean=0.0,
stddev=0.01))
def get_baseline_convolutional_encoder(filters,
embedding_dimension,
input_shape=None,
dropout=0.05):
encoder = Sequential()
# Initial conv
if input_shape is None:
# In this case we are using the encoder as part of a siamese network and the input shape will be determined
# automatically based on the input shape of the siamese network
encoder.add(
layers.Conv1D(filters, 32, padding='same', activation='relu'))
else:
# In this case we are using the encoder to build a classifier network and the input shape must be defined
encoder.add(
layers.Conv1D(filters,
32,
padding='same',
activation='relu',
input_shape=input_shape))
encoder.add(layers.BatchNormalization())
encoder.add(layers.SpatialDropout1D(dropout))
encoder.add(layers.MaxPool1D(4, 4))
# Further convs
encoder.add(
layers.Conv1D(2 * filters, 3, padding='same', activation='relu'))
encoder.add(layers.BatchNormalization())
encoder.add(layers.SpatialDropout1D(dropout))
encoder.add(layers.MaxPool1D())
encoder.add(
layers.Conv1D(3 * filters, 3, padding='same', activation='relu'))
encoder.add(layers.BatchNormalization())
encoder.add(layers.SpatialDropout1D(dropout))
encoder.add(layers.MaxPool1D())
encoder.add(
layers.Conv1D(4 * filters, 3, padding='same', activation='relu'))
encoder.add(layers.BatchNormalization())
encoder.add(layers.SpatialDropout1D(dropout))
encoder.add(layers.MaxPool1D())
encoder.add(layers.GlobalMaxPool1D())
encoder.add(layers.Dense(embedding_dimension))
return encoder
def create_model(data, catcols):
inputs = []
outputs = []
for c in catcols:
num_unique_values = int(data[c].nunique())
embed_dim = int(min(np.ceil((num_unique_values)/2), 50))
inp = layers.Input(shape=(1, ))
out = layers.Embedding(num_unique_values + 1, embed_dim, name=c)(inp)
out = layers.SpatialDropout1D(0.3)(out)
out = layers.Reshape(target_shape=(embed_dim, ))(out)
inputs.append(inp)
outputs.append(out)
x = layers.Concatenate()(outputs)
x = layers.BatchNormalization()(x)
x = layers.Dense(300, activation='relu')(x)
x = layers.Dropout(0.3)(x)
x = layers.BatchNormalization()(x)
x = layers.Dense(300, activation='relu')(x)
x = layers.Dropout(0.3)(x)
x = layers.BatchNormalization()(x)
y = layers.Dense(2, activation='softmax')(x)
model = Model(inputs=inputs, outputs=y)
return model
def build_transactions_rnn(transactions_cat_features,
embedding_projections,
product_col_name='product',
rnn_units=128,
classifier_units=32,
optimizer=None):
if not optimizer:
optimizer = keras.optimizers.Adam(lr=1e-3)
inputs = []
cat_embeds = []
for feature_name in transactions_cat_features:
inp = L.Input(shape=(None, ),
dtype='uint32',
name=f'input_{feature_name}')
inputs.append(inp)
source_size, projection = embedding_projections[feature_name]
emb = L.Embedding(source_size + 1,
projection,
trainable=True,
mask_zero=False,
name=f'embedding_{feature_name}')(inp)
cat_embeds.append(emb)
# product feature
inp = L.Input(shape=(1, ), dtype='uint32', name=f'input_product')
inputs.append(inp)
source_size, projection = embedding_projections['product']
product_emb = L.Embedding(source_size + 1,
projection,
trainable=True,
mask_zero=False,
name=f'embedding_product')(inp)
product_emb_reshape = L.Reshape((projection, ))(product_emb)
concated_cat_embeds = L.concatenate(cat_embeds)
dropout_embeds = L.SpatialDropout1D(0.05)(concated_cat_embeds)
sequences = L.Bidirectional(L.GRU(units=rnn_units,
return_sequences=True))(dropout_embeds)
pooled_avg_sequences = L.GlobalAveragePooling1D()(sequences)
pooled_max_sequences = L.GlobalMaxPooling1D()(sequences)
#add dropout=0.5
concated = L.concatenate(
[pooled_avg_sequences, pooled_max_sequences, product_emb_reshape])
dense_intermediate = L.Dense(classifier_units,
activation='relu',
kernel_regularizer=keras.regularizers.L1L2(
1e-7, 1e-5))(concated)
proba = L.Dense(1, activation='sigmoid')(dense_intermediate)
model = Model(inputs=inputs, outputs=proba)
model.compile(loss='binary_crossentropy', optimizer=optimizer)
return model
def create_keras_embedding_model(data,
cat_cols,
N_uniq=50,
N_dense=300,
drop_rate=0.3):
inputs = []
outputs = []
for col in cat_cols:
n_unique = int(data[col].nunique())
embed_dim = int(min(np.ceil(n_unique / 2), N_uniq))
inp = layers.Input(shape=(1, ))
out = layers.Embedding(n_unique + 1, embed_dim, name=col)(inp)
out = layers.SpatialDropout1D(drop_rate)(out)
out = layers.Reshape(target_shape=(embed_dim, ))(out)
inputs.append(inp)
outputs.append(out)
x = layers.Concatenate()(outputs)
x = layers.BatchNormalization()(x)
x = layers.Dense(N_dense, activation='relu')(x)
x = layers.Dropout(drop_rate)(x)
x = layers.BatchNormalization()(x)
x = layers.Dense(N_dense, activation='relu')(x)
x = layers.Dropout(drop_rate)(x)
x = layers.BatchNormalization()(x)
y = layers.Dense(2, activation='softmax')(x)
model = Model(inputs=inputs, outputs=y)
return model
def get_stweet_model():
path = "/content/drive/My Drive/Datasets/"
embedding_matrix = pickle.load(
open(path + 'embedding_matrix_tweet.pl', 'rb'))
embedding_layer = tf.keras.layers.Embedding(290575,
300,
weights=[embedding_matrix],
input_length=30,
trainable=False)
sequence_input = layers.Input(shape=(30, ), dtype='int32')
embedding_sequences = embedding_layer(sequence_input)
x = layers.SpatialDropout1D(0.2)(embedding_sequences)
x = layers.Conv1D(64, 5, activation='relu')(x)
x = layers.Bidirectional(
layers.LSTM(64, dropout=0.2, recurrent_dropout=0.2))(x)
x = layers.Dense(512, activation='relu')(x)
x = layers.Dropout(0.5)(x)
x = layers.Dense(512, activation='relu')(x)
outputs = layers.Dense(2, activation='softmax')(x)
model = tf.keras.Model(sequence_input, outputs)
model.compile(optimizer='adam',
loss='CategoricalCrossentropy',
metrics=['accuracy'])
return model
def model_rnn(kwargs):
K.clear_session()
nn_input = input_[kwargs["input_layer"]](**kwargs["input_params"])
x = embeddings[kwargs["emb_layer"]](**kwargs["emb_params"])(nn_input)
x = layers.SpatialDropout1D(0.1)(x)
x = layers.Bidirectional(layers.GRU(32, dropout=0.3, recurrent_dropout=0.3, return_sequences=True))(x)
x = layers.Bidirectional(layers.GRU(16, dropout=0.4, recurrent_dropout=0.4, return_sequences=True))(x)
avg_pool = layers.GlobalAveragePooling1D()(x)
max_pool = layers.GlobalMaxPooling1D()(x)
x = layers.concatenate([avg_pool, max_pool])
x = layers.Dropout(0.1)(x)
x = layers.Dense(100)(x)
x = layers.BatchNormalization()(x)
x = layers.Activation("relu")(x)
x = layers.Dropout(0.3)(x)
nn_pred = layers.Dense(kwargs["out_units"], activation=kwargs["out_activation"])(x)
model = Model(inputs=nn_input, outputs=nn_pred)
model.compile(
loss=kwargs["loss"],
optimizer=kwargs["optimizer"],
metrics=["accuracy"]
)
sess = K.get_session()
init = tf.global_variables_initializer()
sess.run(init)
return model
def create_cnn(total_words=1000,
embedded_dimension=300,
embedding_matrix=None,
input_length=100,
optimizer='adam'):
# Add an Input Layer
input_layer = layers.Input((input_length, ))
# Add the word embedding Layer
embedding_layer = layers.Embedding(total_words + 1,
embedded_dimension,
weights=[embedding_matrix],
trainable=False)(input_layer)
embedding_layer = layers.SpatialDropout1D(0.5)(embedding_layer)
# Add the convolutional Layer
conv_layer = layers.Convolution1D(100, 3,
activation="relu")(embedding_layer)
# Add the pooling Layer
pooling_layer = layers.GlobalMaxPool1D()(conv_layer)
# Add the output Layers
output_layer1 = layers.Dense(50, activation="relu")(pooling_layer)
output_layer1 = layers.Dropout(0.6)(output_layer1)
output_layer2 = layers.Dense(1, activation="sigmoid")(output_layer1)
# Compile the model
model = models.Model(inputs=input_layer, outputs=output_layer2)
model.compile(optimizer=optimizer, loss='binary_crossentropy')
return model
def __init__(self, cfg, verbose=False):
super(ConvRec, self).__init__(cfg, verbose)
self.conv1d = layers.Conv1D(filters=cfg.model_conv1d_filters,
kernel_size=3,
padding='same',
activation='relu',
use_bias=(not cfg.model_bn_in),
name='conv1d')
self.dropout = layers.SpatialDropout1D(cfg.model_dropout, name='do1d')
RNN = layers.GRU if cfg.model_rec_type == 'gru' else layers.LSTM
ret_seqs = [True] * cfg.model_rec_layers
ret_seqs[-1] = False
self.recs = []
for i, ret_seq in enumerate(ret_seqs):
if cfg.model_rec_bi:
rec = layers.Bidirectional(RNN(units=cfg.model_rec_size,
return_sequences=ret_seq,
name=f'rec{i}'),
merge_mode=cfg.model_rec_bi_merge)
else:
rec = RNN(units=cfg.model_rec_size,
return_sequences=ret_seq,
name=f'rec{i}')
self.recs.append(rec)
setattr(self, f'rec{i}', rec)
def __init__(self, cfg, num_classes, verbose=False):
super(FullConv, self).__init__()
self.verbose = verbose
div = 1
self.pad = layers.ZeroPadding2D(padding=(1, 0), name='zpad2d')
self.conv2d = layers.Conv2D(filters=cfg.conv2d_filters,
kernel_size=(3, cfg.reps_size),
padding='valid',
activation='relu',
name='conv2d')
self.dropout = layers.SpatialDropout1D(cfg.dropout, name='do1d')
self.conv1d1 = layers.Conv1D(filters=cfg.conv1d_filters,
kernel_size=3,
padding='same',
activation='relu',
name='conv1d1')
self.conv1d2 = layers.Conv1D(filters=cfg.conv1d_filters,
kernel_size=3,
padding='same',
activation='relu',
name='conv1d2')
self.conv1d3 = layers.Conv1D(filters=cfg.conv1d_filters,
kernel_size=3,
padding='same',
activation='relu',
name='conv1d3')
self.pool1d = layers.MaxPool1D(pool_size=2, name='mpool1d')
self.flat = layers.Flatten(name='flat')
self.fc = layers.Dense(num_classes, activation='softmax', name='fc')
def create_cnn(model_params):
"""
This function creates a Deep Convolutional Network based on model_params dictionary
:param model_params: dict of model_params
:return: keras model
"""
# Add an Input Layer, expected vectors of 0 and 1, with one vector for each word
input_layer = layers.Input((70, ))
# Add the word embedding Layer
embedding_layer = layers.Embedding(len(word_index) + 1,
300,
weights=[embedding_matrix],
trainable=True)(input_layer)
embedding_layer = layers.SpatialDropout1D(
model_params['spatial_dropout'])(embedding_layer)
# Add the convolutional Layer
for ly in range(model_params['num_conv_blocks']):
if ly == 0:
conv_layer = layers.Convolution1D(
model_params['num_conv_filters'],
model_params['filter_size'],
activation=model_params['activation_func'])(embedding_layer)
else:
conv_layer = layers.Convolution1D(
model_params['num_conv_filters'] * ly * 2,
model_params['filter_size'],
activation=model_params['activation_func'])(conv_layer)
# Add the pooling Layer
pooling_layer = layers.GlobalMaxPool1D()(conv_layer)
# Add the output Layers
for ly in range(model_params['num_dense_layers']):
if ly == 0:
output_layer1 = layers.Dense(
model_params['num_dense_neurons'],
activation=model_params['activation_func'])(pooling_layer)
output_layer1 = layers.Dropout(
model_params['dense_dropout'])(output_layer1)
else:
output_layer1 = layers.Dense(
model_params['num_dense_neurons'],
activation=model_params['activation_func'])(output_layer1)
output_layer1 = layers.Dropout(
model_params['dense_dropout'])(output_layer1)
output_layer2 = layers.Dense(1, activation="sigmoid")(output_layer1)
# Compile the model
model = models.Model(inputs=input_layer, outputs=output_layer2)
model.compile(optimizer=optimizers.Adam(
lr=model_params['learning_rate'],
decay=model_params['learning_rate'] / model_params['epochs']),
loss='binary_crossentropy',
metrics=['accuracy'])
return model
def LSTM(sequence_length, embedding_layer):
return keras.Sequential([
keras.Input(shape=(sequence_length, )), embedding_layer,
layers.SpatialDropout1D(0.3),
layers.LSTM(100, return_sequences=True),
layers.GlobalMaxPool1D(),
layers.Dense(50, activation="relu"),
layers.Dropout(0.2),
layers.Dense(6, activation="sigmoid")
])
def __init__(self, cfg, verbose=False):
super(Conv2D, self).__init__(cfg, verbose)
self.pad = layers.ZeroPadding2D(padding=(1, 0), name='zpad2d')
self.conv2d = layers.Conv2D(filters=cfg.model_conv2d_filters,
kernel_size=(3, 512),
padding='valid',
activation='relu',
use_bias=(not cfg.model_bn_in),
name='conv2d')
self.dropout = layers.SpatialDropout1D(cfg.model_dropout, name='do1d')
self.gap = layers.GlobalAveragePooling1D(name='gap')
def BGRU_CNN(sequence_length, embedding_layer):
return keras.Sequential([
keras.Input(shape=(sequence_length, )), embedding_layer,
layers.SpatialDropout1D(0.3),
layers.Bidirectional(layers.GRU(100, return_sequences=True)),
layers.Conv1D(100, 3, activation="relu"),
layers.GlobalMaxPool1D(),
layers.Dense(50, activation="relu"),
layers.Dropout(0.2),
layers.Dense(6, activation="sigmoid")
])
def cnn_v2(lr=0.005):
model = Sequential(layers=[
layers.Convolution1D(128,
kernel_size=50,
strides=25,
activation='relu',
padding='valid',
input_shape=(3000, 1)),
# layers.Convolution1D(128, kernel_size=50, strides=25, activation='relu', padding='valid'),
layers.MaxPool1D(pool_size=8, strides=8),
layers.SpatialDropout1D(rate=0.1),
layers.Convolution1D(
128, kernel_size=8, strides=1, activation='relu', padding='valid'),
layers.Convolution1D(
128, kernel_size=8, strides=1, activation='relu', padding='valid'),
layers.Convolution1D(
128, kernel_size=8, strides=1, activation='relu', padding='valid'),
layers.MaxPool1D(4, 4),
layers.SpatialDropout1D(rate=0.5),
layers.Flatten(),
# layers.Convolution1D(32, kernel_size=3, activation='relu', padding='valid'),
# layers.Convolution1D(32, kernel_size=3, activation='relu', padding='valid'),
# layers.MaxPool1D(pool_size=2),
# layers.SpatialDropout1D(rate=0.1),
# layers.Convolution1D(32, kernel_size=3, activation='relu', padding='valid'),
# layers.Convolution1D(32, kernel_size=3, activation='relu', padding='valid'),
# layers.MaxPool1D(pool_size=2),
# layers.Convolution1D(256, kernel_size=3, activation='relu', padding='valid'),
# layers.Convolution1D(256, kernel_size=3, activation='relu', padding='valid'),
# layers.GlobalMaxPool1D(),
# layers.Dropout(rate=0.01),
layers.Dense(64, activation='relu'),
layers.Dropout(rate=0.1),
# layers.Dense(64, activation='relu'),
layers.Dropout(rate=0.5),
layers.Dense(5, activation='softmax')
])
model.compile(optimizer=optimizers.Adam(lr),
loss=losses.sparse_categorical_crossentropy,
metrics=['accuracy']) #,class_model='categorical'
return model
def keras_model_fn_cpu(model_config, vocab_size, embedding_size, embeddings):
""" CPU version of Stacked Bi-LSTM and Bi-GRU with Two Fasttext
"""
## hyperparams
model_name = model_config['model_name']
num_class = model_config['num_class']
lstm_hs = model_config['lstm_hs']
gru_hs = model_config['gru_hs']
learning_rate = model_config['learning_rate']
with tf.device('/cpu:0'):
## build model
inputs = ks.Input(shape=(None, ), dtype='int32', name='inputs')
embedded_sequences_ft1 = layers.Embedding(vocab_size,
embedding_size,
trainable=False,
mask_zero=False)(inputs)
embedded_sequences_ft2 = layers.Embedding(vocab_size,
embedding_size,
trainable=False,
mask_zero=False)(inputs)
concat_embed = layers.concatenate(
[embedded_sequences_ft1, embedded_sequences_ft2])
concat_embed = layers.SpatialDropout1D(0.5)(concat_embed)
x = layers.Bidirectional(
layers.LSTM(lstm_hs,
recurrent_activation='sigmoid',
return_sequences=True))(concat_embed)
x, x_h, x_c = layers.Bidirectional(
layers.GRU(gru_hs,
reset_after=True,
recurrent_activation='sigmoid',
return_sequences=True,
return_state=True))(x)
x_1 = layers.GlobalMaxPool1D()(x)
x_2 = layers.GlobalAvgPool1D()(x)
x_out = layers.concatenate([x_1, x_2, x_h])
x_out = layers.BatchNormalization()(x_out)
outputs = layers.Dense(num_class, activation='softmax',
name='outputs')(x_out) # outputs
model = ks.Model(inputs, outputs, name=model_name)
## compile
model.compile(loss='categorical_crossentropy',
optimizer=ks.optimizers.Adam(lr=learning_rate,
clipnorm=.25,
beta_1=0.7,
beta_2=0.99),
metrics=[
'categorical_accuracy',
ks.metrics.TopKCategoricalAccuracy(k=3)
])
return model
def __init__(self, cfg, num_classes, verbose=False):
super(Conv2D, self).__init__()
self.verbose = verbose
self.pad = layers.ZeroPadding2D(padding=(1, 0), name='zpad2d')
self.conv2d = layers.Conv2D(filters=cfg.conv2d_filters,
kernel_size=(3, cfg.reps_size),
padding='valid',
activation='relu',
name='conv2d')
self.dropout = layers.SpatialDropout1D(cfg.dropout, name='do1d')
self.gap = layers.GlobalAveragePooling1D(name='gap')
self.fc = layers.Dense(num_classes, activation='softmax', name='fc')
def __init__(self, cfg, verbose=False):
super(FullConv, self).__init__(cfg.batchnorm, verbose)
div = 1
self.pad = layers.ZeroPadding2D(padding=(1, 0), name='zpad2d')
self.conv2d = layers.Conv2D(
filters=cfg.conv2d_filters, kernel_size=(3, cfg.reps_size),
padding='valid', activation='relu', name='conv2d')
self.dropout1 = layers.SpatialDropout1D(cfg.dropout, name='do1d1')
self.conv1d1 = layers.Conv1D(
filters=cfg.conv2d_filters//div, kernel_size=3,
padding='same', activation='relu', name='conv1d1')
self.dropout2 = layers.SpatialDropout1D(cfg.dropout, name='do1d2')
self.conv1d2 = layers.Conv1D(
filters=cfg.conv2d_filters//div, kernel_size=3,
padding='same', activation='relu', name='conv1d2')
self.dropout3 = layers.SpatialDropout1D(cfg.dropout, name='do1d3')
self.conv1d3 = layers.Conv1D(
filters=cfg.conv2d_filters//div, kernel_size=3,
padding='same', activation='relu', name='conv1d3')
self.dropout4 = layers.SpatialDropout1D(cfg.dropout, name='do1d4')
self.pool1d = layers.MaxPool1D(pool_size=2, name='mpool1d')
self.flat = layers.Flatten(name='flat')
def __init__(self, filters: int, kernel_size: int, dilation_rate: int,
dropout_rate: float, activation: str, **kwargs):
super(ResidualBlock, self).__init__(**kwargs)
self.filters = filters
self.causal_conv_1 = layers.Conv1D(filters=self.filters,
kernel_size=kernel_size,
dilation_rate=dilation_rate,
padding='causal')
self.weight_norm_1 = layers.LayerNormalization()
self.dropout_1 = layers.SpatialDropout1D(rate=dropout_rate)
self.activation_1 = layers.Activation(activation)
self.causal_conv_2 = layers.Conv1D(filters=self.filters,
kernel_size=kernel_size,
dilation_rate=dilation_rate,
padding='causal')
self.weight_norm_2 = layers.LayerNormalization()
self.dropout_2 = layers.SpatialDropout1D(rate=dropout_rate)
self.activation_2 = layers.Activation(activation)
self.activation_3 = layers.Activation(activation)
def build_model_bpps_Attn(embed_size,
seq_len=107,
pred_len=68,
dropout=0.5,
sp_dropout=0.2,
embed_dim=200,
hidden_dim=256,
n_layers=3):
input_s = L.Input(shape=(seq_len, 3))
input_bpps = L.Input(shape=(seq_len, 2))
inputs = L.Concatenate(axis=2)([input_s, input_bpps])
embed = L.Embedding(input_dim=embed_size, output_dim=embed_dim)(inputs)
reshaped = tf.reshape(embed,
shape=(-1, embed.shape[1],
embed.shape[2] * embed.shape[3]))
hidden = L.SpatialDropout1D(sp_dropout)(reshaped)
conv1 = L.Conv1D(128, 64, 1, padding="same", activation=None)(hidden)
h1 = L.LayerNormalization()(conv1)
h1 = L.LeakyReLU()(h1)
conv2 = L.Conv1D(128, 32, 1, padding="same", activation=None)(hidden)
h2 = L.LayerNormalization()(conv2)
h2 = L.LeakyReLU()(h2)
conv3 = L.Conv1D(128, 16, 1, padding="same", activation=None)(hidden)
h3 = L.LayerNormalization()(conv3)
h3 = L.LeakyReLU()(h3)
conv4 = L.Conv1D(128, 8, 1, padding="same", activation=None)(hidden)
h4 = L.LayerNormalization()(conv4)
h4 = L.LeakyReLU()(h4)
hs = L.Concatenate()([h1, h2, h3, h4])
keys = L.Dropout(0.2)(hs)
for x in range(n_layers):
hidden = gru_layer(hidden_dim, dropout)(hidden)
hidden = L.Attention(dropout=0.2)([hidden, keys])
# Since we are only making predictions on the first part of each sequence,
# we have to truncate it
truncated = hidden[:, :pred_len]
out = L.Dense(5, activation='linear')(truncated)
model = tf.keras.Model(inputs=[input_s, input_bpps], outputs=out)
model.compile(tf.optimizers.Adam(), loss=MCRMSE)
return model
def __init__(self,
shape,
num_lables,
max_nb_words=50000,
max_sequence_length=250,
embedding_dim=100):
self.max_sequence_length = max_sequence_length
self.tokenizer = Tokenizer(num_words=max_nb_words,
filters='!"#$%&()*+,-./:;<=>?@[\]^_`{|}~',
lower=True)
self.model = keras.Sequential()
# Turns positive integers (indexes) into dense vectors of fixed size
# NOTE: can only be used as the first layer in a model.
self.model.add(
layers.Embedding(max_nb_words, embedding_dim, input_length=shape))
# convolutional layer: applies a filter in order to create a feature map that
# summarizes the presence of detected features in the input.
# we use relu because it was suggested
# kernel_size needs to be odd -> (1,3,5,7 are common)
self.model.add(layers.Conv1D(64, kernel_size=3, activation="relu"))
# Downsamples the input representation by taking the maximum value over the window defined by pool_size.
self.model.add(layers.MaxPooling1D())
# The Dropout layer randomly sets input units to 0
# with a frequency rate at each step during training time, which helps prevent overfitting.
self.model.add(layers.SpatialDropout1D(0.2))
# Adds a bidirectional LSTM layer. Allows us to read input twice (forwards and backwards)
# We use concatenation as the merge mode because it was used in a paper
self.model.add(
layers.Bidirectional(layers.LSTM(100,
dropout=0.2,
recurrent_dropout=0.2),
merge_mode='concat'))
# performs activation calculation
# model.add(layers.Dense(4, activation='softmax')) # For detecting all emotions
self.model.add(layers.Dense(
num_lables, activation='softmax')) # For detecting single emotions
self.model.summary()
# Configures the model for training.
self.model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
def keras_dropout(layer, rate):
"""
Keras dropout layer.
"""
from keras import layers
input_dim = len(layer.input.shape)
if input_dim == 2:
return layers.SpatialDropout1D(rate)
elif input_dim == 3:
return layers.SpatialDropout2D(rate)
elif input_dim == 4:
return layers.SpatialDropout3D(rate)
else:
return layers.Dropout(rate)
def __init__(self, cfg, verbose=False):
super(Conv, self).__init__(cfg, verbose)
self.convs = []
for i, filters in enumerate(cfg.model_conv_filters):
use_bias = not cfg.model_bn_in if i == 0 else True
conv = layers.Conv1D(filters=filters,
kernel_size=3,
padding='same',
activation='relu',
use_bias=use_bias,
name=f'conv{i}')
self.convs.append(conv)
setattr(self, conv.name, conv)
self.pool = layers.MaxPool1D(pool_size=2, name='pool')
self.dropout = layers.SpatialDropout1D(cfg.model_dropout,
name='dropout')
self.gap = layers.GlobalAveragePooling1D(name='gap')
def create_model(data, catcols):
print(len(catcols))
inputs = []
outputs = []
for c in catcols:
print(c)
num_unique_values = int(data[c].nunique())
embed_dim = int(min(num_unique_values, 70))
inp = layers.Input(shape=(1, ))
print(inp.shape)
print(num_unique_values)
out = layers.Embedding(num_unique_values + 1, embed_dim, name=c)(inp)
out = layers.SpatialDropout1D(0.3)(out)
out = layers.Reshape(target_shape=(embed_dim, ))(out)
print(out.shape)
inputs.append(inp)
outputs.append(out)
x = layers.Concatenate()(outputs)
x = layers.BatchNormalization()(x)
x = layers.Dense(512, activation="relu")(x)
x = layers.Dropout(0.5)(x)
x = layers.BatchNormalization()(x)
x = layers.Dense(512, activation="relu")(x)
x = layers.Dropout(0.5)(x)
x = layers.BatchNormalization()(x)
x = layers.Dense(512, activation="relu")(x)
x = layers.Dropout(0.4)(x)
x = layers.BatchNormalization()(x)
x = layers.Dense(512, activation="relu")(x)
x = layers.Dropout(0.)(x)
x = layers.BatchNormalization()(x)
x = layers.Dense(512, activation="relu")(x)
x = layers.Dropout(0.3)(x)
x = layers.BatchNormalization()(x)
y = layers.Dense(2, activation="softmax")(x)
model = Model(inputs=inputs, outputs=y)
return model
def __init__(self,
units,
num_heads=1,
axis=2,
residual=False,
l2=None,
dropout=0.1,
**kwargs):
#
super(MultiHeadSelfAttention, self).__init__(**kwargs)
assert units % num_heads == 0
self.num_heads = num_heads
self.keys_dim = units // num_heads
self.axis = axis
self.residual = residual
self.qs_layers = []
self.ks_layers = []
self.vs_layers = []
for _ in range(num_heads):
self.qs_layers.append(
layers.TimeDistributed(
layers.Dense(self.keys_dim,
use_bias=False,
kernel_regularizer=regularizers.l2(l2))))
self.ks_layers.append(
layers.TimeDistributed(
layers.Dense(self.keys_dim,
use_bias=False,
kernel_regularizer=regularizers.l2(l2))))
self.vs_layers.append(
layers.TimeDistributed(
layers.Dense(self.keys_dim,
use_bias=False,
kernel_regularizer=regularizers.l2(l2))))
self.attention = ScaledDotProductAttention(dropout=dropout)
self.dense = layers.TimeDistributed(
layers.Dense(units,
activation="relu",
kernel_regularizer=regularizers.l2(l2)))
self.dropout = layers.SpatialDropout1D(dropout)
self.layernorm = LayerNormalization()
def get_ae_model(base, config):
## denoising auto encoder part
## node, adj -> middle feature -> node
node = tf.keras.Input(shape=(None, X_node.shape[2]), name="node")
adj = tf.keras.Input(shape=(None, None, As.shape[3]), name="adj")
x = base([L.SpatialDropout1D(0.4)(node), adj])
x = forward(x, 128, rate=0.3)
p = L.Dense(X_node.shape[2], "sigmoid")(x)
# loss = - tf.reduce_mean(40 * node * tf.math.log(p + 1e-6) + (1 - node) * tf.math.log(1 - p + 1e-6))
loss = -tf.reduce_mean(node * tf.math.log(p + 1e-6) +
(1 - node) * tf.math.log(1 - p + 1e-6))
model = tf.keras.Model(inputs=[node, adj], outputs=[loss])
opt = get_optimizer()
model.compile(optimizer=opt, loss=lambda t, y: y)
return model
