def train_model(df):
reviews = df['review']
labels = df['label']
num_words = len(
[word for sentence in reviews for word in sentence.split(' ')])
# getting the biggest sentence length for padding
max_num_words = max([len(sentence.split()) for sentence in reviews])
tokenizer = Tokenizer(num_words=max_num_words)
tokenizer.fit_on_texts(reviews)
sequences = tokenizer.texts_to_sequences(reviews)
word_index = tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))
data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)
labels = to_categorical(np.asarray(labels))
print('Shape of data tensor:', data.shape)
print('Shape of label tensor:', labels.shape)
# split the data into a training set and a validation set
indices = np.arange(data.shape[0])
np.random.shuffle(indices)
data = data[indices]
labels = labels[indices]
x_train, x_test, y_train, y_test = train_test_split(data,
labels,
test_size=0.2,
random_state=1)
x_train, x_val, y_train, y_val = train_test_split(x_train,
y_train,
test_size=0.2,
random_state=1)
# num_validation_samples = int(VALIDATION_SPLIT * data.shape[0])
#
# x_train = data[:-num_validation_samples]
# y_train = labels[:-num_validation_samples]
# x_val = data[-num_validation_samples:]
# y_val = labels[-num_validation_samples:]
print('Preparing embedding matrix.')
embeddings_index = get_embeddings_index()
# prepare embedding matrix
num_words = min(max_num_words, len(word_index) + 1)
# embedding_matrix = np.zeros((num_words, EMBEDDING_DIM))
# for word, i in word_index.items():
# if i >= max_num_words:
# continue
# embedding_vector = embeddings_index.get(word)
# if embedding_vector is not None:
# # words not found in embedding index will be all-zeros.
# embedding_matrix[i] = embedding_vector
# load pre-trained word embeddings into an Embedding layer
# note that we set trainable = False so as to keep the embeddings fixed
# embedding_layer = Embedding(num_words,
# EMBEDDING_DIM,
# embeddings_initializer=Constant(embedding_matrix),
# input_length=MAX_SEQUENCE_LENGTH,
# trainable=False)
#
embedding_layer = Embedding(len(word_index) + 1,
EMBEDDING_DIM,
input_length=MAX_SEQUENCE_LENGTH)
print('Training model.')
# train a 1D convnet with global maxpooling
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH, ), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
x = Conv1D(128, 5, activation='relu')(embedded_sequences)
x = MaxPooling1D(5)(x)
x = Conv1D(128, 5, activation='relu')(x)
x = MaxPooling1D(5)(x)
x = Conv1D(128, 5, activation='relu')(x)
x = GlobalMaxPooling1D()(x)
x = Dense(128, activation='relu')(x)
num_classes = len(set(df['label']))
num_classes = 6
preds = Dense(num_classes, activation='softmax')(x)
model = Model(sequence_input, preds)
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['acc'])
history = model.fit(x_train,
y_train,
batch_size=128,
epochs=5,
validation_data=(x_val, y_val))
model.save(filepath='sentiment.h5', overwrite=True, save_format=True)
# Evaluate the model on the test data using `evaluate`
print('\n# Evaluate on test data')
results = model.evaluate(x_test, y_test, batch_size=128)
print('test loss, test acc:', results)
return history
def sepcnn_model(blocks,
filters,
kernel_size,
embedding_dim,
dropout_rate,
pool_size,
input_shape,
num_classes,
num_features,
use_pretrained_embedding=False,
is_embedding_trainable=False,
embedding_matrix=None):
"""Creates an instance of a separable CNN model.
# Arguments
blocks: int, number of pairs of sepCNN and pooling blocks in the model.
filters: int, output dimension of the layers.
kernel_size: int, length of the convolution window.
embedding_dim: int, dimension of the embedding vectors.
dropout_rate: float, percentage of input to drop at Dropout layers.
pool_size: int, factor by which to downscale input at MaxPooling layer.
input_shape: tuple, shape of input to the model.
num_classes: int, number of output classes.
num_features: int, number of words (embedding input dimension).
use_pretrained_embedding: bool, true if pre-trained embedding is on.
is_embedding_trainable: bool, true if embedding layer is trainable.
embedding_matrix: dict, dictionary with embedding coefficients.
# Returns
A sepCNN model instance.
"""
op_units, op_activation = _get_last_layer_units_and_activation(num_classes)
model = models.Sequential()
# Add embedding layer. If pre-trained embedding is used add weights to the
# embeddings layer and set trainable to input is_embedding_trainable flag.
if use_pretrained_embedding:
model.add(Embedding(input_dim=num_features,
output_dim=embedding_dim,
input_length=input_shape[0],
weights=[embedding_matrix],
trainable=is_embedding_trainable))
else:
model.add(Embedding(input_dim=num_features,
output_dim=embedding_dim,
input_length=input_shape[0]))
for _ in range(blocks-1):
model.add(Dropout(rate=dropout_rate))
model.add(SeparableConv1D(filters=filters,
kernel_size=kernel_size,
activation='relu',
bias_initializer='random_uniform',
depthwise_initializer='random_uniform',
padding='same'))
model.add(SeparableConv1D(filters=filters,
kernel_size=kernel_size,
activation='relu',
bias_initializer='random_uniform',
depthwise_initializer='random_uniform',
padding='same'))
model.add(MaxPooling1D(pool_size=pool_size))
model.add(SeparableConv1D(filters=filters * 2,
kernel_size=kernel_size,
activation='relu',
bias_initializer='random_uniform',
depthwise_initializer='random_uniform',
padding='same'))
model.add(SeparableConv1D(filters=filters * 2,
kernel_size=kernel_size,
activation='relu',
bias_initializer='random_uniform',
depthwise_initializer='random_uniform',
padding='same'))
model.add(GlobalAveragePooling1D())
model.add(Dropout(rate=dropout_rate))
model.add(Dense(op_units, activation=op_activation))
return model
def get_resnet34_BN2(self,input_shape,nb_classes):
OUTPUT_CLASS = nb_classes # output classes
input1 = Input(shape=input_shape, name='input_ecg')
k = 1 # increment every 4th residual block
p = True # pool toggle every other residual block (end with 2^8)
convfilt = 64
encoder_confilt = 64 # encoder filters' num
convstr = 1
ksize = 16
poolsize = 2
poolstr = 2
drop = 0.5
# First convolutional block (conv,BN, relu)
lcount = 0
x = Conv1D(filters=convfilt,
kernel_size=ksize,
padding='same',
strides=convstr,
kernel_initializer='he_normal', name='layer' + str(lcount))(input1)
lcount += 1
x = BatchNormalization(axis=1, name='layer' + str(lcount))(x)
lcount += 1
x = Activation('relu')(x)
## Second convolutional block (conv, BN, relu, dropout, conv) with residual net
# Left branch (convolutions)
x1 = Conv1D(filters=convfilt,
kernel_size=ksize,
padding='same',
strides=convstr,
kernel_initializer='he_normal', name='layer' + str(lcount))(x)
lcount += 1
x1 = BatchNormalization(axis=1, name='layer' + str(lcount))(x1)
lcount += 1
x1 = Activation('relu')(x1)
x1 = Dropout(drop)(x1)
x1 = Conv1D(filters=convfilt,
kernel_size=ksize,
padding='same',
strides=convstr,
kernel_initializer='he_normal', name='layer' + str(lcount))(x1)
lcount += 1
x1 = MaxPooling1D(pool_size=poolsize,
strides=poolstr, padding='same')(x1)
# Right branch, shortcut branch pooling
x2 = MaxPooling1D(pool_size=poolsize,
strides=poolstr, padding='same')(x)
# Merge both branches
x = keras.layers.add([x1, x2])
del x1, x2
## Main loop
p = not p
for l in range(15):
if (l % 4 == 0) and (l > 0): # increment k on every fourth residual block
k += 1
# increase depth by 1x1 Convolution case dimension shall change
xshort = Conv1D(filters=convfilt * k, kernel_size=1, name='layer' + str(lcount))(x)
lcount += 1
else:
xshort = x
# Left branch (convolutions)
# notice the ordering of the operations has changed
x1 = BatchNormalization(axis=1, name='layer' + str(lcount))(x)
lcount += 1
x1 = Activation('relu')(x1)
x1 = Dropout(drop)(x1)
x1 = Conv1D(filters=convfilt * k,
kernel_size=ksize,
padding='same',
strides=convstr,
kernel_initializer='he_normal', name='layer' + str(lcount))(x1)
lcount += 1
x1 = BatchNormalization(axis=1, name='layer' + str(lcount))(x1)
lcount += 1
x1 = Activation('relu')(x1)
x1 = Dropout(drop)(x1)
x1 = Conv1D(filters=convfilt * k,
kernel_size=ksize,
padding='same',
strides=convstr,
kernel_initializer='he_normal', name='layer' + str(lcount))(x1)
lcount += 1
if p:
x1 = MaxPooling1D(pool_size=poolsize, strides=poolstr, padding='same')(x1)
# Right branch: shortcut connection
if p:
x2 = MaxPooling1D(pool_size=poolsize, strides=poolstr, padding='same')(xshort)
else:
x2 = xshort # pool or identity
# Merging branches
x = keras.layers.add([x1, x2])
# change parameters
p = not p # toggle pooling
# x = Conv1D(filters=convfilt * k, kernel_size=ksize, padding='same', strides=convstr, kernel_initializer='he_normal')(x)
# x_reg = Conv1D(filters=convfilt * k, kernel_size=1, padding='same', strides=convstr, kernel_initializer='he_normal')(x)
# Final bit
x = BatchNormalization(axis=1, name='layer' + str(lcount))(x)
lcount += 1
x = Activation('relu')(x)
x_ecg = Flatten()(x)
out1 = Dense(OUTPUT_CLASS, activation='softmax', name='main_output')(x_ecg)
model = Model(inputs=input1, outputs=out1)
return model
# Create the Embedding Layer
model = Sequential()
model.add(
Embedding(max_features,
embedding_dim,
embeddings_initializer=tf.keras.initializers.Constant(
embedding_matrix),
trainable=False))
model.add(Dropout(0.2))
# Build the Model
filters = 250
kernel_size = 3
hidden_dims = 250
model.add(Conv1D(filters, kernel_size, padding='valid'))
model.add(MaxPooling1D())
model.add(Conv1D(filters, 5, padding='valid', activation='relu'))
model.add(GlobalMaxPool1D())
model.add(Dense(hidden_dims, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(1, activation='sigmoid'))
model.summary()
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# Train Model
x_train, x_val, y_train, y_val = train_test_split(x_train_val,
y,
def CNNResBlockModelComplex(config):
def activation(activation_name, x):
if activation_name == 'leaky_relu':
return LeakyReLU(alpha=config.alpha)(x)
else:
return Activation(activation_name)(x)
def highway_layer(value, gate_bias=-3):
# https://towardsdatascience.com/review-highway-networks-gating-function-to-highway-image-classification-5a33833797b5
nonlocal i_hidden # to keep i_hidden "global" to all functions under CNNResBlockModel()
dim = K.int_shape(value)[-1]
# gate_bias_initializer = tensorflow.keras.initializers.Constant(gate_bias)
# gate = Dense(units=dim, bias_initializer=gate_bias_initializer)(value)
# gate = Activation("sigmoid")(gate)
# TODO (just for yellow color...) NOTE: to keep dimensions matched, convolution gate instead of regular sigmoid
# gate (T in paper)
gate = Conv2D(size_list[i_hidden + config.CNN_ResBlock_conv_per_block -
1],
kernel_size=filt_list[-1],
padding='same',
activation='sigmoid',
bias_initializer=tensorflow.keras.initializers.Constant(
gate_bias))(value)
# negated (C in paper)
negated_gate = Lambda(lambda x: 1.0 - x,
output_shape=(size_list[-1], ))(gate)
# use ResBlock as the Transformation
transformed = ResBlock(x=value)
transformed_gated = Multiply()([gate, transformed])
# UpSample value if needed
if value.shape.as_list()[-1] != negated_gate.shape.as_list()[-1]:
r = negated_gate.shape.as_list()[-1] / value.shape.as_list()[-1]
assert not (bool(r % 1))
value = tf.keras.layers.UpSampling3D(size=(1, 1, int(r)))(value)
identity_gated = Multiply()([negated_gate, value])
value = Add()([transformed_gated, identity_gated])
return value
def ResBlock(x):
for i in range(config.CNN_ResBlock_conv_per_block):
nonlocal i_hidden # to keep i_hidden "global" to all functions under CNNResBlockModel()
lamda_cnn = 0.0 if config.use_l2_in_cnn is False else lamda
x = Conv2D(size_list[i_hidden],
kernel_size=filt_list[i_hidden],
padding='same',
bias_regularizer=keras.regularizers.l2(lamda_cnn),
kernel_regularizer=keras.regularizers.l2(lamda_cnn))(x)
x = activation(activation_name, x)
x = BatchNormalization()(x)
i_hidden = i_hidden + 1
return x
if config.with_iq_matrices is False:
raise Exception(
'This model support only operation for IQ representation')
global background_implicit_inference
# parameters
lamda = config.Regularization_term
p_dropout = config.dropout
activation_name = config.activation
filt_dim2_list = config.Filter_shape_dim1 if config.Filter_shape_symmetric else config.Filter_shape_dim2
filt_list = [(x, y)
for x, y in zip(config.Filter_shape_dim1, filt_dim2_list)]
pool_list = [
(x, y) for x, y in zip(config.Pool_shape_dim1, config.Pool_shape_dim2)
]
size_list = config.hidden_size
dense_list = config.Dense_size
input_shape = config.model_input_dim
p_dropout_conv1d = config.CNN_ResBlock_dropout_conv1d
p_dropout_after_all_conv2d = config.dropout_after_all_conv2d
i_hidden = 0
# Input Layer
input_layer = Input(shape=input_shape)
assert len(size_list) == len(filt_list)
assert len(pool_list) == len(config.CNN_ResBlock_highway) == len(
config.CNN_ResBlock_dropout)
assert config.CNN_ResBlock_conv_per_block * len(
config.CNN_ResBlock_highway) == len(size_list)
assert len(config.Conv1D_size) == len(config.Conv1D_kernel)
x = input_layer
real_part = tf.expand_dims(x[:, :, :, 0], axis=-1)
imag_part = tf.expand_dims(x[:, :, :, 1], axis=-1)
real_part_output = Conv2D(size_list[0],
kernel_size=filt_list[0],
padding='same')(real_part)
imag_part_output = Conv2D(size_list[0],
kernel_size=filt_list[0],
padding='same')(imag_part)
real = tf.expand_dims(real_part_output, axis=-1)
imag = tf.expand_dims(imag_part_output, axis=-1)
filter_output = tf.concat([real, imag], axis=-1)
x = complex_activation()(filter_output)
# ResBlocks
for i in range(len(config.CNN_ResBlock_highway)):
if config.CNN_ResBlock_highway[i]:
# True = use Highway
x = highway_layer(value=x)
else:
# False = don't use Highway
x = ResBlock(x=x)
# MaxPool and Dropout
if config.CNN_ResBlock_dropout[i] != 0:
x = Dropout(rate=config.CNN_ResBlock_dropout[i])(x)
x = MaxPooling2D(pool_size=pool_list[i])(x)
# Flatten
x = Flatten()(x)
# Conv1D
if len(config.Conv1D_size) != 0:
x = tf.expand_dims(x, axis=-1)
for i in range(len(config.Conv1D_size)):
x = Conv1D(filters=config.Conv1D_size[i],
kernel_size=config.Conv1D_kernel[i])(x)
x = activation(activation_name, x)
x = BatchNormalization()(x)
if p_dropout_conv1d[i] != 0.0:
x = Dropout(rate=p_dropout_conv1d[1])(x)
# post-Conv1D
if len(config.Conv1D_size) != 0:
x = MaxPooling1D(pool_size=config.Conv1D_pool)(x)
# x = BatchNormalization()(x)
x = Flatten()(x)
# Dense
for i in range(len(dense_list)):
x = Dense(dense_list[i],
kernel_regularizer=keras.regularizers.l2(lamda))(x)
x = activation(activation_name, x)
if p_dropout_after_all_conv2d != 0 and len(config.Conv1D_size) == 0:
x = Dropout(rate=p_dropout_after_all_conv2d)(x)
x = BatchNormalization()(x)
x = Dropout(rate=p_dropout)(x)
# x = BatchNormalization()(x)
if config.learn_background:
x = Dense(3, activation='softmax')(x)
else:
x = Dense(1, activation='sigmoid')(x)
output_layer = x
model = Model(input_layer, output_layer)
if config.learn_background:
if config.background_implicit_inference:
background_implicit_inference = True
model = BlockBackgroundModel(input_layer, output_layer)
# else:
# model = Model(input_layer, output_layer)
# model.summary()
return model
nrsensi = nrtrain
balance = 1
# Call the trace preparation function to create the required training and validation set
train_x, train_y, val_x, val_y = traceconfig.prep_traces(
traceset, nrtrain, nrval, balance)
# Define the model, train the model and validate the model using the created data sets
model = Sequential([
Reshape((tracelength, 1), input_shape=(tracelength, )),
Conv1D(filters=filter,
kernel_size=kernel_mult * peakdist,
strides=peakdist // strides,
input_shape=(tracelength, 1),
activation='relu'),
MaxPooling1D(pool_size=pool),
Flatten(),
Dense(2, activation='sigmoid')
])
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
out = model.fit(train_x,
train_y,
validation_data=(val_x, val_y),
epochs=nrepochs,
batch_size=batchsize,
verbose=1)
# Dump the validation accuracy into an ASCII file
logging.basicConfig(filename='val_acc.log',
def exp_dim(global_feature, num_points):
return tf.tile(global_feature, [1, num_points, 1])
num_points = 29697590
k = 8
input_points = Input(shape=(num_points, 3))
x = Convolution1D(64, 1, activation='relu',
input_shape=(num_points, 3))(input_points)
x = BatchNormalization()(x)
x = Convolution1D(128, 1, activation='relu')(x)
x = BatchNormalization()(x)
x = Convolution1D(1024, 1, activation='relu')(x)
x = BatchNormalization()(x)
x = MaxPooling1D(pool_size=num_points)(x)
x = Dense(512, activation='relu')(x)
x = BatchNormalization()(x)
x = Dense(256, activation='relu')(x)
x = BatchNormalization()(x)
x = Dense(9,
weights=[
np.zeros([256, 9]),
np.array([1, 0, 0, 0, 1, 0, 0, 0, 1]).astype(np.float32)
])(x)
input_T = Reshape((3, 3))(x)
# forward net
g = Lambda(mat_mul, arguments={'B': input_T})(input_points)
g = Convolution1D(64, 1, input_shape=(num_points, 3), activation='relu')(g)
g = BatchNormalization()(g)
data_slice = data[0:1][:][:] print(data.shape) print(data_slice.shape) window_length = data.shape[1] #TODO: Normalize Data #Encoder input_window = Input(shape=(window_length,3)) x = Conv1D(64, 3, activation="relu", padding="same")(input_window) # Full Dimension x = BatchNormalization()(x) x = MaxPooling1D(3, padding="same")(x) x = Conv1D(32, 3, activation="relu", padding="same")(x) x = BatchNormalization()(x) x = MaxPooling1D(2, padding="same")(x) # 3 dims... I'm not super convinced this is actually 3 dimensions encoded = Conv1D(8, 3, activation="relu", padding="same")(x) encoder = Model(input_window, encoded) # 3 dimensions in the encoded layer x = Conv1D(8, 3, activation="relu", padding="same")(encoded) # Latent space x = BatchNormalization()(x) x = UpSampling1D(2)(x) # 6 dims x = Conv1D(64, 3, activation='relu', padding='same')(x) # 5 dims x = BatchNormalization()(x) x = UpSampling1D(3)(x) # 10 dims decoded = Conv1D(3, 3, activation='sigmoid', padding='same')(x) # 10 dims
def transformer(img_rows=None, img_cols=None, color_type=3, num_class=3):
"""
EQtransformer_model
:param img_rows: Height of the data.
:param img_cols: Width of the data.
:param color_type: Channel number of the data.
:param num_class: Output class number.
:return: EQtransformer model.
"""
inputs = Input(shape=(img_rows, img_cols, color_type))
new_dim = tf.squeeze(inputs, axis=1, name=None)
ff_dim = 64
num_head = 1
conv1 = Conv1D(8, 11, activation='relu', padding='same')(new_dim)
pool1 = MaxPooling1D(2)(conv1)
conv2 = Conv1D(16, 9, activation='relu', padding='same')(pool1)
pool2 = MaxPooling1D(2)(conv2)
conv3 = Conv1D(16, 7, activation='relu', padding='same')(pool2)
pool3 = MaxPooling1D(2)(conv3)
conv4 = Conv1D(32, 5, activation='relu', padding='same')(pool3)
pool4 = MaxPooling1D(2)(conv4)
conv5 = Conv1D(64, 5, activation='relu', padding='same')(pool4)
pool5 = MaxPooling1D(2)(conv5)
conv6 = Conv1D(64, 3, activation='relu', padding='same')(pool5)
pool6 = MaxPooling1D(2)(conv6)
resCNN = ResNet_build(64, 3)
res1 = resCNN(pool6)
bilstm1 = Bidirectional(LSTM(64, return_sequences=True))(res1)
bilstm1 = Dropout(0.1)(bilstm1)
bilstm1 = Conv1D(64, 1, activation='relu', padding='same')(bilstm1)
bilstm1 = LayerNormalization(epsilon=1e-6)(bilstm1)
bilstm1 = Activation('relu')(bilstm1)
bilstm2 = Bidirectional(LSTM(64, return_sequences=True))(bilstm1)
bilstm2 = Dropout(0.1)(bilstm2)
bilstm2 = Conv1D(64, 1, activation='relu', padding='same')(bilstm2)
bilstm2 = LayerNormalization(epsilon=1e-6)(bilstm2)
bilstm2 = Activation('relu')(bilstm2)
lstm1 = LSTM(64, return_sequences=True)(bilstm2)
transformer_block1 = TransformerBlockE(64, num_head, ff_dim)
transE = transformer_block1(lstm1)
transformer_block2 = TransformerBlockE(64, num_head, ff_dim)
transE = transformer_block2(transE)
up1 = UpSampling1D(size=2)(transE)
conv7 = Conv1D(96, 3, activation='relu', padding='same')(up1)
up2 = UpSampling1D(size=2)(conv7)
conv8 = Conv1D(96, 5, activation='relu', padding='same')(up2)
up3 = UpSampling1D(size=2)(conv8)
conv9 = Conv1D(32, 5, activation='relu', padding='same')(up3)
up4 = UpSampling1D(size=2)(conv9)
conv10 = Conv1D(16, 7, activation='relu', padding='same')(up4)
up5 = UpSampling1D(size=2)(conv10)
conv11 = Conv1D(16, 9, activation='relu', padding='same')(up5)
up6 = UpSampling1D(size=2)(conv11)
conv12 = Conv1D(8, 11, activation='relu', padding='same')(up6)
conv13 = Conv1D(1, 1, activation='sigmoid', padding='same')(conv12)
#############################################################################
lstm2 = LSTM(64, return_sequences=True)(transE)
transformer_block3 = TransformerBlockE(64, num_head, ff_dim)
transE_P = transformer_block3(lstm2)
up1_P = UpSampling1D(size=2)(transE_P)
conv7_P = Conv1D(96, 3, activation='relu', padding='same')(up1_P)
up2_P = UpSampling1D(size=2)(conv7_P)
conv8_P = Conv1D(96, 5, activation='relu', padding='same')(up2_P)
up3_P = UpSampling1D(size=2)(conv8_P)
conv9_P = Conv1D(32, 5, activation='relu', padding='same')(up3_P)
up4_P = UpSampling1D(size=2)(conv9_P)
conv10_P = Conv1D(16, 7, activation='relu', padding='same')(up4_P)
up5_P = UpSampling1D(size=2)(conv10_P)
conv11_P = Conv1D(16, 9, activation='relu', padding='same')(up5_P)
up6_P = UpSampling1D(size=2)(conv11_P)
conv12_P = Conv1D(8, 11, activation='relu', padding='same')(up6_P)
conv13_P = Conv1D(1, 1, activation='sigmoid', padding='same')(conv12_P)
#############################################################################
lstm3 = LSTM(64, return_sequences=True)(transE)
transformer_block4 = TransformerBlockE(64, num_head, ff_dim)
transE_S = transformer_block4(lstm3)
up1_S = UpSampling1D(size=2)(transE_S)
conv7_S = Conv1D(96, 3, activation='relu', padding='same')(up1_S)
up2_S = UpSampling1D(size=2)(conv7_S)
conv8_S = Conv1D(96, 5, activation='relu', padding='same')(up2_S)
up3_S = UpSampling1D(size=2)(conv8_S)
conv9_S = Conv1D(32, 5, activation='relu', padding='same')(up3_S)
up4_S = UpSampling1D(size=2)(conv9_S)
conv10_S = Conv1D(16, 7, activation='relu', padding='same')(up4_S)
up5_S = UpSampling1D(size=2)(conv10_S)
conv11_S = Conv1D(16, 9, activation='relu', padding='same')(up5_S)
up6_S = UpSampling1D(size=2)(conv11_S)
conv12_S = Conv1D(8, 11, activation='relu', padding='same')(up6_S)
conv13_S = Conv1D(1, 1, activation='sigmoid', padding='same')(conv12_S)
#############################################################################
output = concatenate([conv13, conv13_P, conv13_S], axis=2)
output = output[:, tf.newaxis, :, :]
model = Model(inputs=inputs, outputs=output)
return model
balanced = balanced_accuracy_score(y_test, y_p)
print('\nbalanced accuracy: %f\n' % (balanced))
model_4.save('model_04.h5')
# -------------------------------------------------------------------------------------------------------------------------------------------
# Conv NET 1
model_conv_1 = Sequential()
model_conv_1.add(Embedding(30000 + 2, 32, input_length=max_length))
# model.add(Dropout(0.2))
model_conv_1.add(Conv1D(64, 4, activation='relu'))
model_conv_1.add(MaxPooling1D(4))
model_conv_1.add(Flatten())
model_conv_1.add(Dense(1024, activation='relu'))
model_conv_1.add(Dense(1588, activation='softmax'))
opt = tf.keras.optimizers.Adam(lr=0.001, beta_1=0.99, beta_2=0.999)
model_conv_1.compile(loss='sparse_categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
# training
balanced = []
for inx in range(2, 3):
x_train = sequences[np.array(trn_1_inx[inx]), :]
def build_conv_model(depth_conv,
depth_dense,
filters,
kernel_size,
reg,
activation,
batch_norm,
dropout,
input_shape,
pooling=True,
pool_size=2,
padding='valid',
dilation_rate=1):
half_filters = int(filters / 2)
model = Sequential()
# conv + pooling layers
for i in range(depth_conv):
if i == (depth_conv - 1):
if i == 0:
model.add(
Conv1D(filters=half_filters,
kernel_size=kernel_size,
activation=activation,
kernel_regularizer=reg,
padding=padding,
dilation_rate=dilation_rate,
input_shape=input_shape))
else:
model.add(
Conv1D(filters=half_filters,
kernel_size=kernel_size,
activation=activation,
kernel_regularizer=reg,
padding=padding,
dilation_rate=dilation_rate))
else:
if i == 0:
model.add(
Conv1D(filters=filters,
kernel_size=kernel_size,
activation=activation,
kernel_regularizer=reg,
padding=padding,
dilation_rate=dilation_rate,
input_shape=input_shape))
else:
model.add(
Conv1D(filters=filters,
kernel_size=kernel_size,
activation=activation,
kernel_regularizer=reg,
padding=padding,
dilation_rate=dilation_rate))
if pooling:
model.add(MaxPooling1D(pool_size=pool_size))
model.add(Flatten())
# dense layers
for k in range(depth_dense):
if batch_norm:
model.add(BatchNormalization())
model.add(Dropout(dropout))
if k == (depth_dense - 1):
model.add(Dense(1, activation='sigmoid', kernel_regularizer=reg))
else:
model.add(Dense(256, activation=activation,
kernel_regularizer=reg))
return model
def get_test_model_exhaustive():
"""Returns a exhaustive test model."""
input_shapes = [(2, 3, 4, 5, 6), (2, 3, 4, 5, 6), (7, 8, 9, 10),
(7, 8, 9, 10), (11, 12, 13), (11, 12, 13), (14, 15),
(14, 15), (16, ),
(16, ), (2, ), (1, ), (2, ), (1, ), (1, 3), (1, 4),
(1, 1, 3), (1, 1, 4), (1, 1, 1, 3), (1, 1, 1, 4),
(1, 1, 1, 1, 3), (1, 1, 1, 1, 4), (26, 28, 3), (4, 4, 3),
(4, 4, 3), (4, ), (2, 3), (1, ), (1, ), (1, ), (2, 3),
(9, 16, 1), (1, 9, 16)]
inputs = [Input(shape=s) for s in input_shapes]
outputs = []
outputs.append(Conv1D(1, 3, padding='valid')(inputs[6]))
outputs.append(Conv1D(2, 1, padding='same')(inputs[6]))
outputs.append(Conv1D(3, 4, padding='causal', dilation_rate=2)(inputs[6]))
outputs.append(ZeroPadding1D(2)(inputs[6]))
outputs.append(Cropping1D((2, 3))(inputs[6]))
outputs.append(MaxPooling1D(2)(inputs[6]))
outputs.append(MaxPooling1D(2, strides=2, padding='same')(inputs[6]))
outputs.append(AveragePooling1D(2)(inputs[6]))
outputs.append(AveragePooling1D(2, strides=2, padding='same')(inputs[6]))
outputs.append(GlobalMaxPooling1D()(inputs[6]))
outputs.append(GlobalMaxPooling1D(data_format="channels_first")(inputs[6]))
outputs.append(GlobalAveragePooling1D()(inputs[6]))
outputs.append(
GlobalAveragePooling1D(data_format="channels_first")(inputs[6]))
outputs.append(Conv2D(4, (3, 3))(inputs[4]))
outputs.append(Conv2D(4, (3, 3), use_bias=False)(inputs[4]))
outputs.append(
Conv2D(4, (2, 4), strides=(2, 3), padding='same')(inputs[4]))
outputs.append(
Conv2D(4, (2, 4), padding='same', dilation_rate=(2, 3))(inputs[4]))
outputs.append(SeparableConv2D(3, (3, 3))(inputs[4]))
outputs.append(DepthwiseConv2D((3, 3))(inputs[4]))
outputs.append(DepthwiseConv2D((1, 2))(inputs[4]))
outputs.append(MaxPooling2D((2, 2))(inputs[4]))
# todo: check if TensorFlow 2.1 supports this
#outputs.append(MaxPooling2D((2, 2), data_format="channels_first")(inputs[4])) # Default MaxPoolingOp only supports NHWC on device type CPU
outputs.append(
MaxPooling2D((1, 3), strides=(2, 3), padding='same')(inputs[4]))
outputs.append(AveragePooling2D((2, 2))(inputs[4]))
# todo: check if TensorFlow 2.1 supports this
#outputs.append(AveragePooling2D((2, 2), data_format="channels_first")(inputs[4])) # Default AvgPoolingOp only supports NHWC on device type CPU
outputs.append(
AveragePooling2D((1, 3), strides=(2, 3), padding='same')(inputs[4]))
outputs.append(GlobalAveragePooling2D()(inputs[4]))
outputs.append(
GlobalAveragePooling2D(data_format="channels_first")(inputs[4]))
outputs.append(GlobalMaxPooling2D()(inputs[4]))
outputs.append(GlobalMaxPooling2D(data_format="channels_first")(inputs[4]))
outputs.append(Permute((3, 4, 1, 5, 2))(inputs[0]))
outputs.append(Permute((1, 5, 3, 2, 4))(inputs[0]))
outputs.append(Permute((3, 4, 1, 2))(inputs[2]))
outputs.append(Permute((2, 1, 3))(inputs[4]))
outputs.append(Permute((2, 1))(inputs[6]))
outputs.append(Permute((1, ))(inputs[8]))
outputs.append(Permute((3, 1, 2))(inputs[31]))
outputs.append(Permute((3, 1, 2))(inputs[32]))
outputs.append(BatchNormalization()(Permute((3, 1, 2))(inputs[31])))
outputs.append(BatchNormalization()(Permute((3, 1, 2))(inputs[32])))
outputs.append(BatchNormalization()(inputs[0]))
outputs.append(BatchNormalization(axis=1)(inputs[0]))
outputs.append(BatchNormalization(axis=2)(inputs[0]))
outputs.append(BatchNormalization(axis=3)(inputs[0]))
outputs.append(BatchNormalization(axis=4)(inputs[0]))
outputs.append(BatchNormalization(axis=5)(inputs[0]))
outputs.append(BatchNormalization()(inputs[2]))
outputs.append(BatchNormalization(axis=1)(inputs[2]))
outputs.append(BatchNormalization(axis=2)(inputs[2]))
outputs.append(BatchNormalization(axis=3)(inputs[2]))
outputs.append(BatchNormalization(axis=4)(inputs[2]))
outputs.append(BatchNormalization()(inputs[4]))
# todo: check if TensorFlow 2.1 supports this
#outputs.append(BatchNormalization(axis=1)(inputs[4])) # tensorflow.python.framework.errors_impl.InternalError: The CPU implementation of FusedBatchNorm only supports NHWC tensor format for now.
outputs.append(BatchNormalization(axis=2)(inputs[4]))
outputs.append(BatchNormalization(axis=3)(inputs[4]))
outputs.append(BatchNormalization()(inputs[6]))
outputs.append(BatchNormalization(axis=1)(inputs[6]))
outputs.append(BatchNormalization(axis=2)(inputs[6]))
outputs.append(BatchNormalization()(inputs[8]))
outputs.append(BatchNormalization(axis=1)(inputs[8]))
outputs.append(BatchNormalization()(inputs[27]))
outputs.append(BatchNormalization(axis=1)(inputs[27]))
outputs.append(BatchNormalization()(inputs[14]))
outputs.append(BatchNormalization(axis=1)(inputs[14]))
outputs.append(BatchNormalization(axis=2)(inputs[14]))
outputs.append(BatchNormalization()(inputs[16]))
# todo: check if TensorFlow 2.1 supports this
#outputs.append(BatchNormalization(axis=1)(inputs[16])) # tensorflow.python.framework.errors_impl.InternalError: The CPU implementation of FusedBatchNorm only supports NHWC tensor format for now.
outputs.append(BatchNormalization(axis=2)(inputs[16]))
outputs.append(BatchNormalization(axis=3)(inputs[16]))
outputs.append(BatchNormalization()(inputs[18]))
outputs.append(BatchNormalization(axis=1)(inputs[18]))
outputs.append(BatchNormalization(axis=2)(inputs[18]))
outputs.append(BatchNormalization(axis=3)(inputs[18]))
outputs.append(BatchNormalization(axis=4)(inputs[18]))
outputs.append(BatchNormalization()(inputs[20]))
outputs.append(BatchNormalization(axis=1)(inputs[20]))
outputs.append(BatchNormalization(axis=2)(inputs[20]))
outputs.append(BatchNormalization(axis=3)(inputs[20]))
outputs.append(BatchNormalization(axis=4)(inputs[20]))
outputs.append(BatchNormalization(axis=5)(inputs[20]))
outputs.append(Dropout(0.5)(inputs[4]))
outputs.append(ZeroPadding2D(2)(inputs[4]))
outputs.append(ZeroPadding2D((2, 3))(inputs[4]))
outputs.append(ZeroPadding2D(((1, 2), (3, 4)))(inputs[4]))
outputs.append(Cropping2D(2)(inputs[4]))
outputs.append(Cropping2D((2, 3))(inputs[4]))
outputs.append(Cropping2D(((1, 2), (3, 4)))(inputs[4]))
outputs.append(Dense(3, use_bias=True)(inputs[13]))
outputs.append(Dense(3, use_bias=True)(inputs[14]))
outputs.append(Dense(4, use_bias=False)(inputs[16]))
outputs.append(Dense(4, use_bias=False, activation='tanh')(inputs[18]))
outputs.append(Dense(4, use_bias=False)(inputs[20]))
outputs.append(Reshape(((2 * 3 * 4 * 5 * 6), ))(inputs[0]))
outputs.append(Reshape((2, 3 * 4 * 5 * 6))(inputs[0]))
outputs.append(Reshape((2, 3, 4 * 5 * 6))(inputs[0]))
outputs.append(Reshape((2, 3, 4, 5 * 6))(inputs[0]))
outputs.append(Reshape((2, 3, 4, 5, 6))(inputs[0]))
outputs.append(Reshape((16, ))(inputs[8]))
outputs.append(Reshape((2, 8))(inputs[8]))
outputs.append(Reshape((2, 2, 4))(inputs[8]))
outputs.append(Reshape((2, 2, 2, 2))(inputs[8]))
outputs.append(Reshape((2, 2, 1, 2, 2))(inputs[8]))
outputs.append(
UpSampling2D(size=(1, 2), interpolation='nearest')(inputs[4]))
outputs.append(
UpSampling2D(size=(5, 3), interpolation='nearest')(inputs[4]))
outputs.append(
UpSampling2D(size=(1, 2), interpolation='bilinear')(inputs[4]))
outputs.append(
UpSampling2D(size=(5, 3), interpolation='bilinear')(inputs[4]))
for axis in [-5, -4, -3, -2, -1, 1, 2, 3, 4, 5]:
outputs.append(Concatenate(axis=axis)([inputs[0], inputs[1]]))
for axis in [-4, -3, -2, -1, 1, 2, 3, 4]:
outputs.append(Concatenate(axis=axis)([inputs[2], inputs[3]]))
for axis in [-3, -2, -1, 1, 2, 3]:
outputs.append(Concatenate(axis=axis)([inputs[4], inputs[5]]))
for axis in [-2, -1, 1, 2]:
outputs.append(Concatenate(axis=axis)([inputs[6], inputs[7]]))
for axis in [-1, 1]:
outputs.append(Concatenate(axis=axis)([inputs[8], inputs[9]]))
for axis in [-1, 2]:
outputs.append(Concatenate(axis=axis)([inputs[14], inputs[15]]))
for axis in [-1, 3]:
outputs.append(Concatenate(axis=axis)([inputs[16], inputs[17]]))
for axis in [-1, 4]:
outputs.append(Concatenate(axis=axis)([inputs[18], inputs[19]]))
for axis in [-1, 5]:
outputs.append(Concatenate(axis=axis)([inputs[20], inputs[21]]))
outputs.append(UpSampling1D(size=2)(inputs[6]))
# outputs.append(UpSampling1D(size=2)(inputs[8])) # ValueError: Input 0 of layer up_sampling1d_1 is incompatible with the layer: expected ndim=3, found ndim=2. Full shape received: [None, 16]
outputs.append(Multiply()([inputs[10], inputs[11]]))
outputs.append(Multiply()([inputs[11], inputs[10]]))
outputs.append(Multiply()([inputs[11], inputs[13]]))
outputs.append(Multiply()([inputs[10], inputs[11], inputs[12]]))
outputs.append(Multiply()([inputs[11], inputs[12], inputs[13]]))
shared_conv = Conv2D(1, (1, 1),
padding='valid',
name='shared_conv',
activation='relu')
up_scale_2 = UpSampling2D((2, 2))
x1 = shared_conv(up_scale_2(inputs[23])) # (1, 8, 8)
x2 = shared_conv(up_scale_2(inputs[24])) # (1, 8, 8)
x3 = Conv2D(1, (1, 1),
padding='valid')(up_scale_2(inputs[24])) # (1, 8, 8)
x = Concatenate()([x1, x2, x3]) # (3, 8, 8)
outputs.append(x)
x = Conv2D(3, (1, 1), padding='same', use_bias=False)(x) # (3, 8, 8)
outputs.append(x)
x = Dropout(0.5)(x)
outputs.append(x)
x = Concatenate()([MaxPooling2D((2, 2))(x),
AveragePooling2D((2, 2))(x)]) # (6, 4, 4)
outputs.append(x)
x = Flatten()(x) # (1, 1, 96)
x = Dense(4, use_bias=False)(x)
outputs.append(x)
x = Dense(3)(x) # (1, 1, 3)
outputs.append(x)
outputs.append(Add()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Subtract()([inputs[26], inputs[30]]))
outputs.append(Multiply()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Average()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Maximum()([inputs[26], inputs[30], inputs[30]]))
outputs.append(Concatenate()([inputs[26], inputs[30], inputs[30]]))
intermediate_input_shape = (3, )
intermediate_in = Input(intermediate_input_shape)
intermediate_x = intermediate_in
intermediate_x = Dense(8)(intermediate_x)
intermediate_x = Dense(5)(intermediate_x)
intermediate_model = Model(inputs=[intermediate_in],
outputs=[intermediate_x],
name='intermediate_model')
intermediate_model.compile(loss='mse', optimizer='nadam')
x = intermediate_model(x) # (1, 1, 5)
intermediate_model_2 = Sequential()
intermediate_model_2.add(Dense(7, input_shape=(5, )))
intermediate_model_2.add(Dense(5))
intermediate_model_2.compile(optimizer='rmsprop',
loss='categorical_crossentropy')
x = intermediate_model_2(x) # (1, 1, 5)
x = Dense(3)(x) # (1, 1, 3)
shared_activation = Activation('tanh')
outputs = outputs + [
Activation('tanh')(inputs[25]),
Activation('hard_sigmoid')(inputs[25]),
Activation('selu')(inputs[25]),
Activation('sigmoid')(inputs[25]),
Activation('softplus')(inputs[25]),
Activation('softmax')(inputs[25]),
Activation('softmax')(inputs[25]),
Activation('relu')(inputs[25]),
LeakyReLU()(inputs[25]),
ELU()(inputs[25]),
PReLU()(inputs[24]),
PReLU()(inputs[25]),
PReLU()(inputs[26]),
shared_activation(inputs[25]),
Activation('linear')(inputs[26]),
Activation('linear')(inputs[23]),
x,
shared_activation(x),
]
model = Model(inputs=inputs, outputs=outputs, name='test_model_exhaustive')
model.compile(loss='mse', optimizer='nadam')
# fit to dummy data
training_data_size = 1
data_in = generate_input_data(training_data_size, input_shapes)
initial_data_out = model.predict(data_in)
data_out = generate_output_data(training_data_size, initial_data_out)
model.fit(data_in, data_out, epochs=10)
return model
tokenizer.fit_on_texts(sentences)
word_index = tokenizer.word_index
vocab_size = len(word_index)
print(vocab_size)
sequences = tokenizer.texts_to_sequences(sentences)
padded = pad_sequences(sequences, maxlen=max_len, padding=padding_type, truncating=trunc_type)
weights = get_embedding_weights(vocab_size, word_index)
model = Sequential([
Embedding(vocab_size + 1, embedding_dims, input_length=max_len, weights=[weights], trainable=False),
Dropout(.2),
Conv1D(64, 5, activation='relu'),
MaxPooling1D(pool_size=4),
LSTM(64),
Dense(1, activation='sigmoid')
])
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
split = int(test_portion * len(padded))
labels = np.expand_dims(labels, axis=-1)
print(padded.shape, labels.shape)
testing_sentences = padded[:split]
testing_labels = labels[:split]
training_sentences = padded[split:]
training_labels = labels[split:]
loss_function = 'mse'
n_epochs = 20
optimizer = Adam(lr=0.001)
validation_split = 0.3
train_time_x = time.clock()
#train flat x
inputs = Input(shape=(13,1)) #13 in x dimension + 2 angles
angles = Input(shape=(2,))
x = Conv1D(64, kernel_size=3, padding="same")(inputs)
x = Activation("relu")(x)
x = Conv1D(64, kernel_size=3, padding="same")(x)
x = Activation("relu")(x)
x = BatchNormalization(axis=-1)(x)
x = MaxPooling1D(pool_size=2,padding='same')(x)
x = Dropout(0.25)(x)
x = Conv1D(64, kernel_size=3, padding="same")(x)
x = Activation("relu")(x)
x = Conv1D(64, kernel_size=3, padding="same")(x)
x = Activation("relu")(x)
x = BatchNormalization(axis=-1)(x)
x = MaxPooling1D(pool_size=2,padding='same')(x)
x = Dropout(0.25)(x)
x_cnn = Flatten()(x)
concat_inputs = concatenate([x_cnn,angles])
x = Dense(64)(concat_inputs)
x = Activation("relu")(x)
x = BatchNormalization()(x)
x = Dropout(0.25)(x)
from tensorflow.keras.utils import to_categorical
y_train = to_categorical(y_train)
y_valid = to_categorical(y_valid)
X_train = np.array(X_train).reshape(891, 192, 1)
X_valid = np.array(X_valid).reshape(99, 192, 1)
#Modelin Oluşturulması
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Activation, Conv1D, Dropout, Flatten, MaxPooling1D
model = Sequential()
model.add(Conv1D(512, 1, input_shape=(nb_features, 1)))
model.add(Activation("relu"))
model.add(MaxPooling1D(2))
model.add(Conv1D(256, 1))
model.add(Activation("relu"))
model.add(MaxPooling1D(2))
#verilerlin %25 atıyoruz
model.add(Dropout(0.25))
#verileri düzleştirme
model.add(Flatten())
#Yapay Sinir Ağı
model.add(Dense(2048, activation="relu"))
model.add(Dense(1024, activation="relu"))
#En son sonoflandırma yapalım. Sınıflandırma için softmax kullanılır
model.add(Dense(nb_classes, activation="softmax"))
model.summary()
#Modelin Derlenmesi
x_train=x_train.reshape(x_train.shape[0], 5, 6) x_test=x_test.reshape(x_test.shape[0], 5, 6) x_val=x_val.reshape(x_val.shape[0], 5, 6) x_pred=x_pred.reshape(x_pred.shape[0], 5, 6) x_1_train=x_1_train.reshape(x_1_train.shape[0], 5, 6) x_1_test=x_1_test.reshape(x_1_test.shape[0], 5, 6) x_1_val=x_1_val.reshape(x_1_val.shape[0], 5, 6) x_1_pred=x_1_pred.reshape(x_1_pred.shape[0], 5, 6) ## 모델링 input1=Input(shape=(x_train.shape[1], x_train.shape[2])) # lstm1=LSTM(32, activation='relu')(input1) # dense1=Dense(64, activation='relu')(lstm1) cnn1=Conv1D(256, 2, padding='same', activation='relu')(input1) max1=MaxPooling1D(2)(cnn1) drop1=Dropout(0.2)(max1) cnn1=Conv1D(256, 2, padding='same', activation='relu')(drop1) max1=MaxPooling1D(2)(cnn1) drop1=Dropout(0.2)(max1) flat1=Flatten()(drop1) dense1=Dense(128, activation='relu')(flat1) dense1=Dense(256, activation='relu')(dense1) dense1=Dense(128, activation='relu')(dense1) dense1=Dense(64, activation='relu')(dense1) dense1=Dense(32, activation='relu')(dense1) input2=Input(shape=(x_1_train.shape[1], x_1_train.shape[2])) lstm2=LSTM(256, activation='relu')(input2) drop2=Dropout(0.2)(lstm2) # lstm2=Dense(64, activation='relu')(input2)
def embedding(data):
"""
likely improvements
- downsampling rate (or better yet, use something like SMOTE)
- tokenization number of words (max_words)
- model architecture
- GloVe package (100d or 300d) or better embedding (source code)
"""
############################################################################
# some global text pre-processing
############################################################################
print("\n\npreprocessing\n\n")
# drop nans
print("---------------- starting to remove nans (↓ before)")
print(data["label"].value_counts())
is_nan_idx = data[pd.isnull(data["program"])].index.tolist()
data.drop(data.index[is_nan_idx], inplace=True)
print(data["label"].value_counts())
print("---------------- done nans (↑ after)\n")
# remove duplicates
print("---------------- starting to remove duplicates (↓ before)")
print(data["label"].value_counts())
data.drop_duplicates(inplace=True)
print(data["label"].value_counts())
print("---------------- done duplicates (↑ after)\n")
# print(data.describe())
# remove really small programs
print("---------------- starting to remove small programs (↓ before)")
print(data["label"].value_counts())
to_drop = []
for index, row in data.iterrows():
if len(row["program"]) <= 100:
to_drop.append(data.index.get_loc(index))
data.drop(data.index[to_drop], inplace=True)
print(data["label"].value_counts())
print("---------------- done small programs (↑ after)")
############################################################################
# training
############################################################################
print("\n\ntraining\n\n")
overall_f1 = []
overall_accuracy = []
overall_precision = []
overall_recall = []
overall_kappa = []
# overall_auc = []
overall_matrix = []
overall_test_suite_results = []
for i in range(1):
# quick shuffle
data = shuffle(data)
## downsample uneven classes
class_0, class_1 = data.label.value_counts()
df_majority = data[data.label == 0]
df_minority = data[data.label == 1]
df_majority_downsampled = resample(
df_majority, replace=False, n_samples=int(class_1 * 1), random_state=42
)
df_downsampled = pd.concat([df_majority_downsampled, df_minority])
df = df_downsampled
## make unicode
print("---------------- starting to make unicode")
x = df.program.values.astype("U")
y = np.array(df.label)
print("X", len(x), "Y", len(y))
print("---------------- done unicode")
number_of_splits = 10
skfold = StratifiedKFold(n_splits=number_of_splits, shuffle=True)
# kfold = KFold(n_splits=number_of_splits, shuffle=True)
fold_f1 = []
fold_accuracy = []
fold_precision = []
fold_recall = []
fold_kappa = []
fold_matrix = []
fold_test_suite_results = []
for train_ix, test_ix in skfold.split(x, y):
####################################################################
# folds
####################################################################
print("\n\nfold", number(), "\n\n")
print("---------------- starting initialization per-fold examples")
train_program, test_program = x[train_ix], x[test_ix]
train_class, test_class = y[train_ix], y[test_ix]
print("---------------- done initializing")
## tokenize
max_words = 100000
# max_words = 100
print("---------------- starting tokenization")
tokenize = text.Tokenizer(num_words=max_words, char_level=False)
tokenize.fit_on_texts(train_program) # only fit on train
vocab_size = len(tokenize.word_index) + 1
print("---------------- done tokenization")
print("---------------- starting encoding")
encoded_docs = tokenize.texts_to_sequences(train_program)
encoded_docs_test = tokenize.texts_to_sequences(test_program)
print("---------------- done encoding")
## GloVe embedding
print("---------------- starting GloVe")
embeddings_index = {}
f = open(r"data/TEXT/glove.42B.300d.txt", encoding="utf8")
for line in f:
values = line.split()
word = "".join(values[:-300])
coefs = np.asarray(values[-300:], dtype="float32")
embeddings_index[word] = coefs
f.close()
embedding_matrix = np.zeros((vocab_size, 300))
for word, index in tokenize.word_index.items():
if index > vocab_size - 1:
break
else:
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
embedding_matrix[index] = embedding_vector
# embeddings_index = None
print("---------------- done GloVe")
## add padding
# max_length = int(vocab_size * .001) # testing
max_length = 1000 # performance on adversarial perspective
# max_length = 100 # performance on test suite
print("---------------- starting padding")
padded_docs = pad_sequences(encoded_docs, maxlen=max_length, padding="post")
padded_docs_test = pad_sequences(
encoded_docs_test, maxlen=max_length, padding="post"
)
print("---------------- done padding")
## encode
print("---------------- starting to encode")
encoder = LabelEncoder()
encoder.fit(train_class)
# print("done fit to train class")
y_train = encoder.transform(train_class)
y_test = encoder.transform(test_class)
print("---------------- done encoding\n")
y_test_backend = y_test.copy()
# print("done encoder.transform")
num_classes = np.max(y_train) + 1
y_train = utils.to_categorical(y_train, num_classes)
y_test = utils.to_categorical(y_test, num_classes)
print("y train", len(y_train), "y test", len(y_test))
epochs = 10
## model2 architecture
model = Sequential()
model.add(
Embedding(
vocab_size,
300,
input_length=max_length,
weights=[embedding_matrix],
trainable=True,
)
)
## option-1 for paper
model.add(Conv1D(64, 5, activation="relu"))
model.add(MaxPooling1D(pool_size=4))
model.add(LSTM(300, activation="sigmoid"))
model.add(Dense(2))
model.add(Activation("softmax"))
# model.add(Dense(1, activation='sigmoid'))
# ## option-2 possible better adversarial training (also change downsampling to 1.5)
# model.add(SpatialDropout1D(0.2))
# model.add(Bidirectional(LSTM(128, dropout=0.25, recurrent_dropout=0.25, return_sequences=True)))
# model.add(Bidirectional(LSTM(64, dropout=0.25, recurrent_dropout=0.25, return_sequences=True)))
# model.add(Conv1D(64, 4))
# model.add(GlobalMaxPool1D())
# model.add(Dense(2, activation='relu'))
# model.add(Activation('softmax'))
model.compile(
optimizer="adam",
loss="categorical_crossentropy",
metrics=[categorical_accuracy],
)
# fit the model
history = model.fit(
padded_docs,
y_train,
validation_split=0.3,
epochs=epochs,
verbose=0,
callbacks=[TqdmCallback(verbose=1)],
)
y_softmax = model.predict(padded_docs_test)
y_pred_1d = []
y_pred_percentages = []
for i in range(0, len(y_softmax)):
probs = y_softmax[i]
y_pred_percentages.append(
np.amax(probs)
) # Returns the indices of the maximum values along an axis
predicted_index = np.argmax(probs)
y_pred_1d.append(predicted_index)
print("test length", len(padded_docs_test))
y_pred_1d = np.array(y_pred_1d)
# accuracy: (tp + tn) / (p + n)
accuracy = accuracy_score(y_test_backend, y_pred_1d)
print("Accuracy: %f" % accuracy)
# precision tp / (tp + fp)
precision = precision_score(y_test_backend, y_pred_1d)
print("Precision: %f" % precision)
# recall: tp / (tp + fn)
recall = recall_score(y_test_backend, y_pred_1d)
print("Recall: %f" % recall)
# f1: 2 tp / (2 tp + fp + fn)
f1_output = f1_score_sklearn(y_test_backend, y_pred_1d)
print("F1 score: %f" % f1_output)
# kappa
kappa = cohen_kappa_score(y_test_backend, y_pred_1d)
print("Cohens kappa: %f" % kappa)
# confusion matrix
matrix = confusion_matrix(y_test_backend, y_pred_1d)
print(matrix)
fold_f1.append(f1_output)
fold_accuracy.append(accuracy)
fold_precision.append(precision)
fold_recall.append(recall)
fold_kappa.append(kappa)
fold_matrix.append(matrix)
####################################################################
# testing on test suite
####################################################################
print("\n\ntest suite\n\n")
chunksize = 500000000000000
test_suite = "data/TEXT/test_suite.csv"
for chunk in pd.read_csv(test_suite, chunksize=chunksize):
count = 0
result = chunk
options = [
"program_plain",
"program_jsnice_v2",
"program_plain_obfuscated",
"program_plain_obfuscated_jsnice_v2",
]
results = {}
print(result.label.value_counts())
for option in options:
false_negative = 0
false_positive = 0
true_negative = 0
true_positive = 0
for index, row in result.iterrows():
tester = row[option]
x_predict_array = [tester]
try:
x_predict = tokenize.texts_to_sequences(x_predict_array)
x_predict = pad_sequences(
x_predict, maxlen=max_length, padding="post"
)
prediction = model.predict(x_predict, verbose=0)
predicted_index = np.argmax(prediction)
if predicted_index != row["label"]:
count += 1
if predicted_index == 1 and row["label"] == 0:
false_positive += 1
if predicted_index == 0 and row["label"] == 1:
false_negative += 1
if predicted_index == row["label"]:
if predicted_index == 1 and row["label"] == 1:
true_positive += 1
if predicted_index == 0 and row["label"] == 0:
true_negative += 1
except Exception as e:
print("pass here")
print(e)
# sys.exit()
results[option] = {
"false_negative": false_negative,
"false_positive": false_positive,
"true_positive": true_positive,
"true_negative": true_negative,
}
print(results)
fold_test_suite_results.append(results)
gc.collect()
print(fold_f1)
overall_f1.append(fold_f1)
overall_accuracy.append(fold_accuracy)
overall_precision.append(fold_precision)
overall_recall.append(fold_recall)
overall_kappa.append(fold_kappa)
overall_matrix.append(fold_matrix)
overall_test_suite_results.append(fold_test_suite_results)
print(
overall_f1,
overall_accuracy,
overall_precision,
overall_recall,
overall_kappa,
overall_matrix,
overall_test_suite_results,
)
return (
overall_f1,
overall_accuracy,
overall_precision,
overall_recall,
overall_kappa,
overall_matrix,
overall_test_suite_results,
)
confusion = metrics.confusion_matrix(y_test, ypredout)
print("accuracy:%.2f%%" % (testscores * 100))
print(metrics.classification_report(y_test, ypredout, digits=2))
print(confusion)
# CNN + LSTM
model = Sequential()
model.add(embedding_layer)
model.add(Dropout(0.5))
model.add(
Convolution1D(nb_filter,
filter_length,
padding='valid',
activation=cnn_activation))
model.add(MaxPooling1D(pool_length))
model.add(
LSTM(output_size,
activation=rnn_activation,
recurrent_activation=recurrent_activation))
model.add(Dropout(0.25))
model.add(Dense(2))
model.add(Activation('sigmoid'))
model.compile(loss=loss, optimizer=optimizer, metrics=['accuracy'])
model.summary()
print('CNN + LSTM')
model.fit(x_seq_pad,
y,
batch_size=batch_size,
epochs=no_of_epochs,
model = Sequential()
# model.add(SimpleRNN(16, input_shape=(x_train.shape[1],1), activation='relu', return_sequences=False))
# model.add(GRU(16, input_shape=(x_train.shape[1],1), activation='relu', return_sequences=False))
# model.add(LSTM(16, input_shape=(x_train.shape[1],1), activation='relu', return_sequences=False))
model = Sequential()
model.add(
Conv1D(filters=40,
kernel_size=2,
strides=1,
padding='same',
input_shape=(x_train.shape[1], 1),
activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(Conv1D(16, 2, padding='same'))
model.add(Conv1D(16, 2, padding='same'))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
model.add(Dense(8))
model.add(Dense(8))
model.add(Dense(2))
#3. 컴파일, 핏
model.compile(loss='mse', optimizer='adam', metrics=['mae'])
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint
stop = EarlyStopping(monitor='val_loss', patience=16, mode='min')
def train_model(data, model_path):
x = data['x']
y = data['y']
scaler = MinMaxScaler(feature_range=(-1, 1))
print ("Scaling data ...")
for val in range(x.shape[0]):
x[val] = scaler.fit_transform(x[val])
if val%500 == 0 :
print(val)
(x_train, x_val, y_train, y_val) = train_test_split(x, y, test_size=0.2,
random_state=SEED)
print('Building model...')
n_features = x_train.shape[2]
input_shape = (None, n_features)
model_input = Input(input_shape, name='input')
layer = model_input
for i in range(N_LAYERS):
# second convolutional layer names are used by extract_filters.py
layer = Convolution1D(
filters=CONV_FILTER_COUNT,
kernel_size=FILTER_LENGTH,
name='convolution_' + str(i + 1)
)(layer)
layer = BatchNormalization(momentum=0.9)(layer)
layer = Activation('relu')(layer)
layer = MaxPooling1D(2)(layer)
layer = Dropout(0.5)(layer)
layer = TimeDistributed(Dense(len(GENRES)))(layer)
time_distributed_merge_layer = Lambda(
function=lambda x: K.mean(x, axis=1),
output_shape=lambda shape: (shape[0],) + shape[2:],
name='output_merged'
)
layer = time_distributed_merge_layer(layer)
layer = Activation('softmax', name='output_realtime')(layer)
model_output = layer
model = Model(model_input, model_output)
opt = Adam(lr=0.001)
model.compile(
loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy']
)
model.summary()
print('Training...')
earlyStop = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=2)
model.fit(
x_train, y_train, batch_size=BATCH_SIZE, epochs=EPOCH_COUNT,
validation_data=(x_val, y_val), verbose=1, callbacks=[
ModelCheckpoint(
model_path, save_best_only=True, monitor='val_acc', verbose=1
),
ReduceLROnPlateau(
monitor='val_acc', factor=0.5, patience=10, min_delta=0.01,
verbose=1
),
earlyStop
]
)
return model
for i in range(0,test_labels.shape[0]):
test_labels1[i,int(test_labels[i])]=1
train_labels=train_labels1
test_labels=test_labels1
'''
#Building and compiling the model.
print("Building Model.")
tf.keras.backend.clear_session()
model = Sequential()
#conv1d layer 1
model.add(Conv1D(filters=64,kernel_size=1,activation='relu',input_shape=(num_features-1,1)))
#Maxpool1d layer 1
model.add(MaxPooling1D(pool_size=2,strides=None))
#dropout layer to fight with overfitting.
model.add(Dropout(0.25))
#conv1d layer 2
model.add(Conv1D(filters=64,kernel_size=1,activation='relu'))
#Maxpool1d layer 2
model.add(MaxPooling1D(pool_size=1, strides=None))
#Flatten layer to make data's dimension easily fit to dense layer.
model.add(tf.keras.layers.Flatten())
#first dense layer
model.add(Dense(128,activation='relu'))
#secon dense layer
model.add(Dense(9,activation='softmax'))
model.compile(loss='sparse_categorical_crossentropy', optimizer='rmsprop' ,metrics=['accuracy'])
#fitting dataset on the model.
def evaluate_model(x_train,
y_train,
x_test,
y_test,
nnType,
batch_size=1,
epochs=60):
backend.clear_session()
n_timesteps, n_features, n_outputs = x_train.shape[0], x_train.shape[
1], y_train.shape[1]
nb_neurons_lstm = 100
if args.type == "cnn+lstm":
# reshape data into time steps of sub-sequences
#[batch, timesteps, feature].
epochs = 25
n_steps, n_length = 4, 32
# define model
model = Sequential()
model.add(
TimeDistributed(Conv1D(filters=64,
kernel_size=3,
activation='relu'),
input_shape=(None, n_length, n_features)))
model.add(
TimeDistributed(
Conv1D(filters=64, kernel_size=3, activation='relu')))
model.add(TimeDistributed(Dropout(0.5)))
model.add(TimeDistributed(MaxPooling1D(pool_size=2)))
model.add(TimeDistributed(Flatten()))
model.add(LSTM(units=nb_neurons_lstm))
model.add(Dropout(0.5))
model.add(Dense(100, activation='relu'))
model.add(Dense(n_outputs, activation='softmax'))
#model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
x_train_reshape = x_train.reshape(
(x_train.shape[0], n_steps, n_length, n_features))
x_test_reshape = x_train.reshape(
(x_test.shape[0], n_steps, n_length, n_features))
elif args.type == "lstm":
'''64 windows of data will be exposed to the model before the weights of the model are updated.'''
nb_classes = y_train.shape[1]
print('nb_classes', nb_classes)
model = Sequential()
model.add(
LSTM(units=nb_neurons_lstm,
return_sequences=True,
input_shape=(1, n_features)))
model.add(LSTM(units=nb_neurons_lstm, return_sequences=True))
model.add(LSTM(units=nb_neurons_lstm))
#model.add(LSTM(units=nb_neurons_lstm))
'''This is followed by a dropout layer intended to reduce overfitting of the model to the training data.'''
model.add(Dropout(0.5))
'''Activation function is softmax for multi-class classification.'''
#model.add(Dense(100, activation='relu'))
model.add(Dense(units=n_outputs, activation='softmax'))
model.summary()
'''Because it is a multi-class classification problem, categorical_crossentropy is used as the loss function. '''
# reshape pour avoir un shape: (sample, timestamps, features)
print('x_train-------------', x_train.shape[0])
x_train_reshape = x_train.reshape(x_train.shape[0], 1,
x_train.shape[1]) #60 features
x_test_reshape = x_test.reshape(x_test.shape[0], 1,
x_test.shape[1]) #60 features
elif args.type == "convlstm":
n_steps, n_length = 4, 32
# define model
model = Sequential()
model.add(
ConvLSTM2D(filters=64,
kernel_size=(1, 3),
activation='relu',
input_shape=(n_steps, 1, n_length, n_features)))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(100, activation='relu'))
model.add(Dense(n_outputs, activation='softmax'))
model.summary()
# reshape into subsequences (samples, time steps, rows, cols, channels)
x_train_reshape = x_train.reshape(
(x_train.shape[0], n_steps, 1, n_length, n_features))
x_test_reshape = x_test.reshape(
(x_test.shape[0], n_steps, 1, n_length, n_features))
#y_train=labels[:split]
#y_test=labels[split:]
## Transformer les classes en représentation binaire (one-hot encoding)
#y_train,y_test = transform_labels(y_train,y_test)
#print('train',x_train.shape[0])
#print('train2',x_train.shape[2])
#print('x_train.shape[2]',x_train.shape[2])
#print('x_train.shape[2]',x_train.shape[2])
#input_shape = (x_train.shape[1], x_train.shape[2])
#sequence_input =keras.layers.Input(shape=input_shape)
#nb_classes = y_train.shape[1]
#init = initializers.Orthogonal(gain=0.001)
#mask = keras.layers.Masking(mask_value=0.)(sequence_input)
#X = Noise(0.075)(mask)
#print('num_class',nb_classes)
#input_layer = keras.layers.Input(batch_shape=[batch_size,x_train.shape[1],x_train.shape[2]])
#input_layer = keras.layers.Input(x_train.shape[1:])
#
# X = keras.layers.LSTM(units=nb_neurons,recurrent_activation='sigmoid',return_sequences=True,implementation=2)(input_layer)
# X = keras.layers.Dropout(0.5)(X)
# X = keras.layers.LSTM(units=nb_neurons,recurrent_activation='sigmoid',return_sequences=True,implementation=2)(X)
# X = keras.layers.Dropout(0.5)(X)
# X = keras.layers.LSTM(units=nb_neurons,recurrent_activation='sigmoid',return_sequences=True,implementation=2)(X)
# X = keras.layers.Dropout(0.5)(X)
# X = keras.layers.TimeDistributed(keras.layers.Dense(num_class))(X)
# #X = MeanOverTime()(X)
# X = keras.layers.Activation('softmax')(X)
#model=keras.models.Model(inputs=input_layer,outputs=X)
filepath = 'modelcheckpoint_' + str(args.db) + '.hdf5'
saveto = 'csvLogger_' + str(args.db) + '.csv'
#optimizer = Adam(lr=lr, clipnorm=args.clip)
#pred_dir = os.path.join(rootdir, str(case) + '_pred.txt')
"""spécifier d'utiliser une partie du train pour la validation des hyper-paramèteres"""
percentage_of_train_as_validation = 0.3
if args.train:
early_stop = EarlyStopping(monitor='val_accuracy',
patience=15,
mode='auto',
min_delta=0.0001)
reduce_lr = ReduceLROnPlateau(monitor='val_accuracy',
factor=0.1,
patience=5,
mode='auto',
cooldown=3.,
verbose=1)
checkpoint = ModelCheckpoint(filepath,
monitor='val_accuracy',
verbose=1,
save_best_only=True,
mode='auto')
csv_logger = CSVLogger(saveto)
callbacks_list = [csv_logger, checkpoint, early_stop, reduce_lr]
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
'''Here we do shuffle the windows of input data during training (the default). In this problem, we are interested in harnessing the LSTMs ability to learn and extract features across the time steps in a window, not across windows.'''
#checkpoint = keras.callbacks.ModelCheckpoint('best-model.h5', monitor='val_loss', save_best_only=True)
#model.fit(x_train_reshape, y_train, epochs=epochs, batch_size=batch_size, verbose=True)
#callbacks_list = [csv_logger, checkpoint, early_stop, reduce_lr]
history = model.fit(x_train_reshape,
y_train,
epochs=20,
verbose=True,
batch_size=10,
validation_split=percentage_of_train_as_validation,
callbacks=callbacks_list)
#model.fit(train_x, train_y, validation_data=[valid_x, valid_y], epochs=args.epochs,batch_size=args.batch_size, callbacks=callbacks_list, verbose=2)
epoch_loss_acc('accuracy', history) #loss or accuracy
epoch_loss_acc('loss', history) #loss or accuracy
loss, accuracy = model.evaluate(x_train_reshape,
y_train,
verbose=False)
print("L'accuracy sur l'ensemble du train est:", accuracy)
print("Le loss sur l'ensemble du train est:", loss)
elif args.test:
model.load_weights(filepath)
#model = keras.models.load_model('best-model.h5')
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# evaluate model
loss, accuracy = model.evaluate(x_test_reshape,
y_test,
batch_size=batch_size,
verbose=False)
print("L'accuracy sur l'ensemble du test est:", accuracy)
print("Le loss sur l'ensemble du train est:", loss)
#scores = get_activation(model, test_x, test_y, pred_dir, VA=10, par=9)
#results.append(round(scores, 2))
return accuracy
def CNNResBlockModel(config):
def regularization(lamda):
if config.regularization_method == 'L2':
return keras.regularizers.l2(lamda)
elif config.regularization_method == 'L1':
return keras.regularizers.l1(lamda)
else:
raise Exception('Use Only L2 / L1 regularization')
def activation(activation_name, x):
if activation_name == 'leaky_relu':
return LeakyReLU(alpha=config.alpha)(x)
else:
return Activation(activation_name)(x)
def highway_layer(value, gate_bias=-3):
# https://towardsdatascience.com/review-highway-networks-gating-function-to-highway-image-classification-5a33833797b5
nonlocal i_hidden # to keep i_hidden "global" to all functions under CNNResBlockModel()
dim = K.int_shape(value)[-1]
# gate_bias_initializer = tensorflow.keras.initializers.Constant(gate_bias)
# gate = Dense(units=dim, bias_initializer=gate_bias_initializer)(value)
# gate = Activation("sigmoid")(gate)
# TODO (just for yellow color...) NOTE: to keep dimensions matched, convolution gate instead of regular sigmoid
# gate (T in paper)
gate = Conv2D(size_list[i_hidden + config.CNN_ResBlock_conv_per_block -
1],
kernel_size=filt_list[-1],
padding='same',
activation='sigmoid',
bias_initializer=tensorflow.keras.initializers.Constant(
gate_bias))(value)
# negated (C in paper)
negated_gate = Lambda(lambda x: 1.0 - x,
output_shape=(size_list[-1], ))(gate)
# use ResBlock as the Transformation
transformed = ResBlock(x=value)
transformed_gated = Multiply()([gate, transformed])
# UpSample value if needed
if value.shape.as_list()[-1] != negated_gate.shape.as_list()[-1]:
r = negated_gate.shape.as_list()[-1] / value.shape.as_list()[-1]
assert not (bool(r % 1))
value = tf.keras.layers.UpSampling3D(size=(1, 1, int(r)))(value)
identity_gated = Multiply()([negated_gate, value])
value = Add()([transformed_gated, identity_gated])
return value
def skip_connection_layer(value):
nonlocal i_hidden
# use ResBlock as the Transformation
transformed = ResBlock(x=value)
if value.shape.as_list()[-1] != transformed.shape.as_list()[-1]:
r = transformed.shape.as_list()[-1] / value.shape.as_list()[-1]
assert not (bool(r % 1))
# apply convolution as transformation
value = Conv2D(size_list[i_hidden - 1],
kernel_size=filt_list[i_hidden - 1],
padding='same')(value)
value = Add()([value, transformed])
return value
def ResBlock(x):
for i in range(config.CNN_ResBlock_conv_per_block):
nonlocal i_hidden # to keep i_hidden "global" to all functions under CNNResBlockModel()
lamda_cnn = 0.0 if config.use_l2_in_cnn is False else lamda
x = Conv2D(size_list[i_hidden],
kernel_size=filt_list[i_hidden],
padding='same',
bias_regularizer=regularization(lamda_cnn),
kernel_regularizer=regularization(lamda_cnn),
kernel_initializer=kernel_initalizer)(x)
x = activation(activation_name, x)
if config.use_batch_norm is True:
x = BatchNormalization()(x)
i_hidden = i_hidden + 1
return x
def ResBlockLane(x):
nonlocal i_hidden
# ResBlocks
for i in range(len(config.CNN_ResBlock_highway)):
if config.CNN_ResBlock_highway[i] == "Highway":
x = highway_layer(value=x)
elif config.CNN_ResBlock_highway[i] == "Skip":
x = skip_connection_layer(value=x)
elif config.CNN_ResBlock_highway[i] == "None":
x = ResBlock(x=x)
else:
raise Exception('only Highway/Skip/None is allowed !')
# MaxPool and Dropout
if config.CNN_ResBlock_dropout[i] != 0:
x = Dropout(rate=config.CNN_ResBlock_dropout[i])(x)
x = MaxPooling2D(pool_size=pool_list[i])(x)
return x
global background_implicit_inference
# parameters
kernel_initalizer = config.kernel_initalizer # default is 'glorot_uniform'
lamda = config.Regularization_term
p_dropout = config.dropout
activation_name = config.activation
filt_dim2_list = config.Filter_shape_dim1 if config.Filter_shape_symmetric else config.Filter_shape_dim2
filt_list = [(x, y)
for x, y in zip(config.Filter_shape_dim1, filt_dim2_list)]
pool_list = [
(x, y) for x, y in zip(config.Pool_shape_dim1, config.Pool_shape_dim2)
]
size_list = config.hidden_size
dense_list = config.Dense_size
input_shape = config.model_input_dim
p_dropout_conv1d = config.CNN_ResBlock_dropout_conv1d
p_dropout_after_all_conv2d = config.dropout_after_all_conv2d
p_dropout_dense = config.Dense_dropout
i_hidden = 0
# Input Layer
input_layer = Input(shape=input_shape)
assert len(size_list) == len(filt_list)
assert len(pool_list) == len(config.CNN_ResBlock_highway) == len(
config.CNN_ResBlock_dropout)
assert config.CNN_ResBlock_conv_per_block * len(
config.CNN_ResBlock_highway) == len(size_list)
assert len(config.Conv1D_size) == len(config.Conv1D_kernel)
if config.ResBlockDouble:
x1 = input_layer
x1 = ResBlockLane(x1)
i_hidden = 0 # zero the hidden sizes counter
x2 = input_layer
x2 = ResBlockLane(x2)
x = Add()([x1, x2])
else:
x = input_layer
# ResBlocks
x = ResBlockLane(x)
# Flatten
x = Flatten()(x)
if p_dropout_after_all_conv2d != 0:
x = Dropout(rate=p_dropout_after_all_conv2d)(x)
# Conv1D
if len(config.Conv1D_size) != 0:
x = tf.expand_dims(x, axis=-1)
for i in range(len(config.Conv1D_size)):
x = Conv1D(filters=config.Conv1D_size[i],
kernel_size=config.Conv1D_kernel[i],
kernel_initializer=kernel_initalizer)(x)
x = activation(activation_name, x)
if config.use_batch_norm is True:
x = BatchNormalization()(x)
if p_dropout_conv1d[i] != 0.0:
x = Dropout(rate=p_dropout_conv1d[1])(x)
# post-Conv1D
if len(config.Conv1D_size) != 0:
x = MaxPooling1D(pool_size=config.Conv1D_pool)(x)
# x = BatchNormalization()(x)
x = Flatten()(x)
# Dense
for i in range(len(dense_list)):
x = Dense(dense_list[i],
kernel_regularizer=regularization(lamda),
kernel_initializer=kernel_initalizer)(x)
x = activation(activation_name, x)
if config.use_batch_norm is True:
x = BatchNormalization()(x)
if p_dropout_dense[i] != 0.0:
x = Dropout(rate=p_dropout_dense[i])(x)
# x = Dropout(rate=p_dropout)(x)
# x = BatchNormalization()(x)
if config.learn_background:
x = Dense(3, activation='softmax')(x)
else:
x = Dense(1, activation='sigmoid')(x)
output_layer = x
model = Model(input_layer, output_layer)
if config.learn_background:
if config.background_implicit_inference:
background_implicit_inference = True
model = BlockBackgroundModel(input_layer, output_layer)
# else:
# model = Model(input_layer, output_layer)
# model.summary()
return model
print(x_data.shape)
print(y_data.shape)
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(x_data,
y_data,
test_size=0.2)
# 모델링
from tensorflow.keras.models import Model, load_model
from tensorflow.keras.layers import Dense, Conv1D, MaxPooling1D, Flatten, Dropout, Reshape, Input, Concatenate
input1 = Input(shape=(x_row, x_col))
layer1 = Conv1D(32, 3, activation='relu', padding='same', strides=1)(input1)
layer1 = Conv1D(32, 3, activation='relu', padding='same', strides=1)(layer1)
layer1 = MaxPooling1D(pool_size=2)(layer1)
layer1 = Dropout(0.2)(layer1)
layer1 = Conv1D(32, 3, activation='relu', padding='same', strides=1)(input1)
layer1 = Conv1D(32, 3, activation='relu', padding='same', strides=1)(layer1)
layer1 = MaxPooling1D(pool_size=2)(layer1)
layer1 = Dropout(0.2)(layer1)
layer1 = Flatten()(layer1)
layer1 = Dense(64, activation='relu')(layer1)
layer1 = Dense(64, activation='relu')(layer1)
output1 = Dense(96)(layer1)
model = Model(inputs=input1, outputs=output1)
model.summary()
model.compile(loss='mse', optimizer='adam')
def create_keras_model(window_size, loss, hparams_config=None):
"""Creates the Keras model.
Arguments
window_size: Number of samples per row. Must match window_size
of the datasets that are used to fit/predict.
loss: Loss function of the model.
hparams_config: A dictionary of hyperparameters that can be used
to test multiple configurations (hpo). Default is the
'hparams' dictionary that is defined at the beginning of this
function. This dictionary is used for standard non-hpo jobs
and for any parameter that is not defined in a hpo job.
"""
# Default configuration
hparams = {
cst.CONV_FILTERS: 32,
cst.CONV_KERNEL: 9,
cst.CONV_ACTIVATION: 'relu',
cst.LSTM_NUM_UNITS: 128,
cst.LSTM_ACTIVATION: 'tanh',
cst.DENSE_NUM_UNITS: 32,
cst.DENSE_ACTIVATION: 'relu',
cst.OUTPUT_ACTIVATION: 'relu',
cst.LEARNING_RATE: 0.001,
cst.DROPOUT_RATE_CNN: 0.3,
cst.DROPOUT_RATE_LSTM: 0.3,
cst.CONV_STRIDE: 3
}
# update hyperparameters with arguments from task_hyperparameter.py
if hparams_config:
hparams.update(hparams_config)
# define Inputs
qdlin_in = Input(shape=(window_size, cst.STEPS, cst.INPUT_DIM),
name=cst.QDLIN_NAME) # Shape: 20x1000x1
tdlin_in = Input(shape=(window_size, cst.STEPS, cst.INPUT_DIM),
name=cst.TDLIN_NAME) # Shape: 20x1000x1
ir_in = Input(shape=(window_size, cst.INPUT_DIM),
name=cst.INTERNAL_RESISTANCE_NAME) # Shape: 20x1
dt_in = Input(shape=(window_size, cst.INPUT_DIM),
name=cst.DISCHARGE_TIME_NAME) # Shape: 20x1
qd_in = Input(shape=(window_size, cst.INPUT_DIM),
name=cst.QD_NAME) # Shape: 20x1
# combine all data from detail level
detail_concat = concatenate([qdlin_in, tdlin_in],
axis=3,
name='detail_concat') # Shape: 20x1000x2
# define CNN
cnn_out = TimeDistributed(Conv1D(filters=hparams[cst.CONV_FILTERS],
kernel_size=hparams[cst.CONV_KERNEL],
strides=hparams[cst.CONV_STRIDE],
activation=hparams[cst.CONV_ACTIVATION],
padding='same'),
name='convolution')(detail_concat)
# Add some maxpools to reduce output size
cnn_maxpool = TimeDistributed(MaxPooling1D(), name='conv_pool')(cnn_out)
cnn_out2 = TimeDistributed(Conv1D(filters=hparams[cst.CONV_FILTERS] * 2,
kernel_size=hparams[cst.CONV_KERNEL],
strides=hparams[cst.CONV_STRIDE],
activation=hparams[cst.CONV_ACTIVATION],
padding='same'),
name='conv2')(cnn_maxpool)
cnn_maxpool2 = TimeDistributed(MaxPooling1D(), name='pool2')(cnn_out2)
cnn_out3 = TimeDistributed(Conv1D(filters=hparams[cst.CONV_FILTERS] * 4,
kernel_size=hparams[cst.CONV_KERNEL],
strides=hparams[cst.CONV_STRIDE],
activation=hparams[cst.CONV_ACTIVATION],
padding='same'),
name='conv3')(cnn_maxpool2)
cnn_maxpool3 = TimeDistributed(MaxPooling1D(), name='pool3')(cnn_out3)
cnn_flat = TimeDistributed(Flatten(),
name='convolution_flat')(cnn_maxpool3)
drop_out = TimeDistributed(Dropout(rate=hparams[cst.DROPOUT_RATE_CNN]),
name='dropout_cnn')(cnn_flat)
# combine CNN output with all data from summary level
all_concat = concatenate([drop_out, ir_in, dt_in, qd_in],
axis=2,
name='all_concat') #Shape: 20x4
# define LSTM
lstm_out = LSTM(hparams[cst.LSTM_NUM_UNITS],
activation=hparams[cst.LSTM_ACTIVATION],
name='recurrent')(all_concat)
drop_out_2 = Dropout(rate=hparams[cst.DROPOUT_RATE_LSTM],
name='dropout_lstm')(lstm_out)
# hidden dense layer
hidden_dense = Dense(hparams[cst.DENSE_NUM_UNITS],
name='hidden',
activation=hparams[cst.DENSE_ACTIVATION])(drop_out_2)
# update keras context with custom activation object
get_custom_objects().update({'clippy': Clippy(clipped_relu)})
# use (adapted) Relu activation on the last layer for striclty positive outputs
main_output = Dense(2, name='output', activation='clippy')(hidden_dense)
model = Model(inputs=[qdlin_in, tdlin_in, ir_in, dt_in, qd_in],
outputs=[main_output])
metrics_list = [mae_current_cycle, mae_remaining_cycles, 'accuracy']
model.compile(loss=loss,
optimizer=Adam(lr=hparams[cst.LEARNING_RATE], clipnorm=1.),
metrics=metrics_list)
return model
print ('# of features (X train) :', len(X_train[0]), len(X_train[1]))
# 타겟 클래스 확인
print ('y test:', set(y_test))
print ('y test len:', len(set(y_test)))
# 피쳐 데이터 변형 - 단어 중에서 가장 많이 쓰이는 단어는 남기고 나머지 단어는 0 으로 저장
X_train = sequence.pad_sequences(X_train, maxlen=100)
X_valid = sequence.pad_sequences(X_valid, maxlen=100)
X_test = sequence.pad_sequences(X_te0, maxlen=100)
# LSTM 모형 생성; Embedding - Conv1D, MaxPooling1D, LSTM, Dropout, Dense
model = Sequential()
model.add(Embedding(input_dim=10000, output_dim=100))
model.add(Conv1D(50, kernel_size=3, padding='valid', activation='relu'))
model.add(MaxPooling1D(pool_size=3))
model.add(LSTM(100, activation='tanh'))
model.add(Dropout(0.25))
model.add(Dense(1, activation='sigmoid'))
model.summary()
# 모형 컴파일과 학습
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
epochs = 10
hist = model.fit(X_train, y_train, batch_size=100, epochs=epochs, validation_data=(X_valid, y_valid))
# Model training Evaluate
print ('\nModel training evaluate:')
print(model.evaluate(X_train, y_train)[1])
print(model.evaluate(X_valid, y_valid)[1])
def build_model_with_L2(self,input_shape,nb_classes):
OUTPUT_CLASS = nb_classes # output classes
input1 = Input(shape=input_shape, name='input_ecg')
k = 1 # increment every 4th residual block
p = True # pool toggle every other residual block (end with 2^8)
convfilt = 64
encoder_confilt = 64 # encoder filters' num
convstr = 1
ksize = 16
poolsize = 2
poolstr = 2
drop = 0.5
# First convolutional block (conv,BN, relu)
lcount = 0
x = Conv1D(filters=convfilt,
kernel_size=ksize,
padding='same',
strides=convstr, kernel_regularizer=keras.regularizers.l2(0.001),
kernel_initializer='he_normal', name='layer' + str(lcount))(input1)
lcount += 1
x = BatchNormalization(name='layer' + str(lcount))(x)
lcount += 1
x = Activation('relu')(x)
## Second convolutional block (conv, BN, relu, dropout, conv) with residual net
# Left branch (convolutions)
x1 = Conv1D(filters=convfilt,
kernel_size=ksize,
padding='same',
strides=convstr, kernel_regularizer=keras.regularizers.l2(0.001),
kernel_initializer='he_normal', name='layer' + str(lcount))(x)
lcount += 1
x1 = BatchNormalization(name='layer' + str(lcount))(x1)
lcount += 1
x1 = Activation('relu')(x1)
x1 = Dropout(drop)(x1)
x1 = Conv1D(filters=convfilt,
kernel_size=ksize,
padding='same',
strides=convstr, kernel_regularizer=keras.regularizers.l2(0.001),
kernel_initializer='he_normal', name='layer' + str(lcount))(x1)
lcount += 1
x1 = MaxPooling1D(pool_size=poolsize,
strides=poolstr, padding='same')(x1)
# Right branch, shortcut branch pooling
x2 = MaxPooling1D(pool_size=poolsize,
strides=poolstr, padding='same')(x)
# Merge both branches
x = keras.layers.add([x1, x2])
del x1, x2
# fms = []
## Main loop
p = not p
for l in range(15):
if (l % 4 == 0) and (l > 0): # increment k on every fourth residual block
k += 1
# increase depth by 1x1 Convolution case dimension shall change
xshort = Conv1D(filters=convfilt * k, kernel_size=1, name='layer' + str(lcount))(x)
lcount += 1
else:
xshort = x
# Left branch (convolutions)
# notice the ordering of the operations has changed
x1 = BatchNormalization(name='layer' + str(lcount))(x)
lcount += 1
x1 = Activation('relu')(x1)
x1 = Dropout(drop)(x1)
x1 = Conv1D(filters=convfilt * k,
kernel_size=ksize,
padding='same',
strides=convstr, kernel_regularizer=keras.regularizers.l2(0.001),
kernel_initializer='he_normal', name='layer' + str(lcount))(x1)
lcount += 1
x1 = BatchNormalization(name='layer' + str(lcount))(x1)
lcount += 1
x1 = Activation('relu')(x1)
x1 = Dropout(drop)(x1)
x1 = Conv1D(filters=convfilt * k,
kernel_size=ksize,
padding='same',
strides=convstr, kernel_regularizer=keras.regularizers.l2(0.001),
kernel_initializer='he_normal', name='layer' + str(lcount))(x1)
lcount += 1
if p:
x1 = MaxPooling1D(pool_size=poolsize, strides=poolstr, padding='same')(x1)
# Right branch: shortcut connection
if p:
x2 = MaxPooling1D(pool_size=poolsize, strides=poolstr, padding='same')(xshort)
else:
x2 = xshort # pool or identity
# Merging branches
x = keras.layers.add([x1, x2])
# change parameters
p = not p # toggle pooling
# if l == 5:
# fms.append(x)
# if l == 6:
# fms.append(x)
# fms.append(x)
# fms.append(x)
# x = Conv1D(filters=convfilt * k, kernel_size=ksize, padding='same', strides=convstr, kernel_initializer='he_normal')(x)
# x_reg = Conv1D(filters=convfilt * k, kernel_size=1, padding='same', strides=convstr, kernel_initializer='he_normal')(x)
# Final bit
x = BatchNormalization(name='layer' + str(lcount))(x)
lcount += 1
x = Activation('relu')(x)
x_ecg = Flatten()(x)
# bbox_num = 1
#
# x2od2 = Conv1D(filters=bbox_num * 2, kernel_size=1, padding='same', strides=convstr,
# kernel_initializer='he_normal')(
# fms[0])
# out2 = Reshape((1136, bbox_num, 2), name='aux_output1')(x2od2)
#
# x2od3 = Conv1D(filters=bbox_num * 2, kernel_size=1, padding='same', strides=convstr,
# kernel_initializer='he_normal')(
# fms[1])
# out3 = Reshape((1136, bbox_num, 2), name='aux_output2')(x2od3)
#
# x2od4 = Conv1D(filters=bbox_num * 2, kernel_size=1, padding='same', strides=convstr,
# kernel_initializer='he_normal')(
# fms[2])
# out4 = Reshape((1136, bbox_num, 2), name='aux_output3')(x2od4)
#
# x2od5 = Conv1D(filters=bbox_num * 2, kernel_size=1, padding='same', strides=convstr,
# kernel_initializer='he_normal')(
# fms[3])
# out5 = Reshape((1136, bbox_num, 2), name='aux_output4')(x2od5)
out1 = Dense(OUTPUT_CLASS, activation='softmax', name='main_output',
kernel_regularizer=keras.regularizers.l2(0.001), )(x_ecg)
model = Model(inputs=input1, outputs=out1)
model.summary()
return model
#for i in range(len(X_train)):
# X_train[i,:186]= add_gaussian_noise(X_train[i,:186])
X_train = X_train.reshape(len(X_train), X_train.shape[1],1)
X_test = X_test.reshape(len(X_test), X_test.shape[1],1)
# print(X_train.shape)
# print(y_train.shape)
verbose, epochs, batch_size = 0, 10, 32
im_shape=(X_train.shape[1],1)
print("lassan------------")
im_shape = (X_train.shape[1], 1)
inputs_cnn = Input(shape=(im_shape), name='inputs_cnn')
conv1_1 = Conv1D(64, (6), activation='relu', input_shape=im_shape)(inputs_cnn)
conv1_1 = BatchNormalization()(conv1_1)
pool1 = MaxPooling1D(pool_size=(3), strides=(2), padding="same")(conv1_1)
conv2_1 = Conv1D(64, (3), activation='relu', input_shape=im_shape)(pool1)
conv2_1 = BatchNormalization()(conv2_1)
pool2 = MaxPooling1D(pool_size=(2), strides=(2), padding="same")(conv2_1)
conv3_1 = Conv1D(64, (3), activation='relu', input_shape=im_shape)(pool2)
conv3_1 = BatchNormalization()(conv3_1)
pool3 = MaxPooling1D(pool_size=(2), strides=(2), padding="same")(conv3_1)
flatten = Flatten()(pool3)
dense_end1 = Dense(64, activation='relu')(flatten)
dense_end2 = Dense(32, activation='relu')(dense_end1)
main_output = Dense(5, activation='softmax', name='main_output')(dense_end2)
model = Model(inputs=inputs_cnn, outputs=main_output)
print(model.summary())
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
model = model_CNN_full(input_shape=input_shape, rb=3, dil=6, kernel_size=6)
model.compile(loss=tf.keras.losses.MeanSquaredError(),
optimizer=tf.keras.optimizers.Adam(learning_rate=0.05),
metrics=tf.keras.metrics.MeanSquaredError())
print(model.summary())
# Save the weights using the `checkpoint_path` format
model.load_weights('./gdrive/MyDrive/model/model-finetune-onelabel-classparams.h5')
# take output after last addition layer and before Conv1D(1) adding fine-tuning layers
x = Conv1D(32, kernel_size=1, strides=1, padding='same', dilation_rate=1)(model.layers[-4].output)
x = BatchNormalization(axis=-1)(x)
x = MaxPooling1D(pool_size=2, strides=1, padding='same')(x)
x = Conv1D(32, kernel_size=1, strides=1, padding='same', dilation_rate=1)(x)
x = BatchNormalization(axis=-1)(x)
x = Conv1D(1, kernel_size=1, strides=1, padding='same', dilation_rate=1)(x)
x = keras.layers.Flatten()(x)
o = Dense(1, activation='relu')(x)
# constructing a model with the same inputs but a diff output
model2 = Model(inputs=model.input, outputs=[o])
# copy the weights
for l_tg,l_sr in zip(model2.layers[:-4], model.layers[:-4]):
wk0=l_sr.get_weights()
l_tg.set_weights(wk0)
# freeze: making all the parameters in the inner layers non-trainable
def build_cnn(input_shape, drop_rate=0.5, output_size=16):
model = Sequential()
# Layer 1
model.add(
Conv1D(kernel_size=3,
filters=128,
strides=3,
padding='valid',
kernel_initializer='he_uniform',
input_shape=input_shape))
model.add(BatchNormalization())
model.add(Activation('relu'))
# Layer 2
model.add(
Conv1D(kernel_size=3,
filters=128,
padding='same',
kernel_initializer='he_uniform'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling1D(pool_size=3, strides=3))
# Layer 3
model.add(
Conv1D(kernel_size=3,
filters=128,
padding='same',
kernel_initializer='he_uniform'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling1D(pool_size=3, strides=3))
# Layer 4
model.add(
Conv1D(kernel_size=3,
filters=256,
padding='same',
kernel_initializer='he_uniform'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling1D(pool_size=3, strides=3))
# Layer 5
model.add(
Conv1D(kernel_size=3,
filters=256,
padding='same',
kernel_initializer='he_uniform'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling1D(pool_size=3, strides=3))
# Layer 6
model.add(
Conv1D(kernel_size=3,
filters=256,
padding='same',
kernel_initializer='he_uniform'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling1D(pool_size=3, strides=3))
# Layer 7
model.add(
Conv1D(kernel_size=3,
filters=256,
padding='same',
kernel_initializer='he_uniform'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling1D(pool_size=3, strides=3))
# Layer 8
model.add(
Conv1D(kernel_size=3,
filters=256,
padding='same',
kernel_initializer='he_uniform'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling1D(pool_size=3, strides=3))
# # Layer 9
# model.add(Conv1D (kernel_size=3, filters=256, padding='same', kernel_initializer='he_uniform'))
# model.add(BatchNormalization())
# model.add(Activation('relu'))
# model.add(MaxPooling1D(pool_size=3, strides=3))
# # Layer 10
# model.add(Conv1D (kernel_size=3, filters=512, padding='same', kernel_initializer='he_uniform'))
# model.add(BatchNormalization())
# model.add(Activation('relu'))
# model.add(MaxPooling1D(pool_size=3, strides=3))
# Layer 11
model.add(Dropout(drop_rate))
model.add(Flatten())
# Layer 12
model.add(Dense(output_size))
model.add(Activation('softmax'))
return model
