def cnn_features_model(n_timesteps, n_features, n_outputs, nb_features): input = Input(shape=(n_timesteps, n_features)) conv_1 = Convolution1D(12, 5, input_shape=(n_timesteps, n_features), padding='same', activation='relu')(input) bat_1 = BatchNormalization()(conv_1) # drop_1 = Dropout(0.3)(bat_1) maxp_1 = MaxPooling1D(pool_size=2, padding='same', strides=2)(bat_1) conv_2 = Convolution1D(12, 5, input_shape=(n_timesteps, n_features), padding='same', activation='relu')(maxp_1) bat_2 = BatchNormalization()(conv_2) # drop_2 = Dropout(0.3)(conv_2) maxp_2 = MaxPooling1D(pool_size=2, padding='same', strides=2)(bat_2) conv_3 = Convolution1D(20, 5, input_shape=(n_timesteps, n_features), padding='same', activation='relu')(maxp_2) bat_3 = BatchNormalization()(conv_3) drop_3 = Dropout(0.3)(bat_3) maxp_3 = MaxPooling1D(pool_size=2, padding='same', strides=2)(drop_3) conv_4 = Convolution1D(16, 5, input_shape=(n_timesteps, n_features), padding='same', activation='relu')(maxp_3) bat_4 = BatchNormalization()(conv_4) # drop_4 = Dropout(0.3)(conv_4) maxp_4 = MaxPooling1D(pool_size=2, padding='same', strides=2)(bat_4) # drop = Dropout(0.3)(maxp_4) seq_features = GlobalAveragePooling1D()(maxp_4) other_features = Input(shape=(nb_features, )) model = Concatenate()([seq_features, other_features]) # model.add(Flatten()) model = Dense(n_outputs, activation='softmax', kernel_regularizer=regularizers.l2(0.2))(model) model = Model([input, other_features], model) model.compile( optimizer=Adam(lr=0.0003), # loss='categorical_crossentropy', # loss=focal_loss(gamma=2,alpha=1), loss=ghm.ghm_class_loss, metrics=['categorical_accuracy']) return model
def build_cos_model(input_size, cos_dist_lvl, n_neurons, n_layers, batch_norm=True, loss=MeanSquaredError(), optimizer=SGD(learning_rate=0.05, momentum=0.025)): in_1 = Input(shape=(input_size, ), name="input_1") in_2 = Input(shape=(input_size, ), name="input_2") if cos_dist_lvl == 0: model = Concatenate(name="concatenate")([in_1, in_2]) else: model = Multiply(name="pointwise_multiply")([in_1, in_2]) if cos_dist_lvl >= 2: norm_1 = Lambda( lambda tensor: tf.norm(tensor, axis=1, keepdims=True), name="norm_input_1")(in_1) norm_2 = Lambda( lambda tensor: tf.norm(tensor, axis=1, keepdims=True), name="norm_input_2")(in_2) norm_mul = Multiply(name="multiply_norms")([norm_1, norm_2]) model = Lambda(lambda tensors: tf.divide(tensors[0], tensors[1]), name="divide")([model, norm_mul]) if cos_dist_lvl >= 3: model = Lambda( lambda tensor: tf.reduce_sum(tensor, axis=1, keepdims=True), name="sum")(model) if cos_dist_lvl >= 4: model = ValueMinusInput(1, name="one_minus_input")(model) if batch_norm: model = BatchNormalization(name="input_normalization")(model) for i in range(n_layers): model = Dense(n_neurons, activation='sigmoid', name="dense_{}".format(i))(model) model_out = Dense(1, activation='sigmoid', name="classify")(model) model = Model([in_1, in_2], model_out) model.compile(loss=loss, optimizer=optimizer, metrics=[ BinaryAccuracy(), Precision(), Recall(), TrueNegatives(), FalsePositives(), FalseNegatives(), TruePositives() ]) return model
def build_eucl_model(input_size, eucl_dist_lvl, n_neurons, n_layers, batch_norm=True, loss=MeanSquaredError(), optimizer=SGD(learning_rate=0.05, momentum=0.025)): in_1 = Input(shape=(input_size, ), name="input_1") in_2 = Input(shape=(input_size, ), name="input_2") if eucl_dist_lvl == 0: model = Concatenate(name="concatenate")([in_1, in_2]) else: model = Subtract(name="subtract")([in_1, in_2]) if eucl_dist_lvl >= 2: model = Lambda(lambda tensor: tf.square(tensor), name="square")(model) if eucl_dist_lvl >= 3: model = Lambda( lambda tensor: tf.reduce_sum(tensor, axis=1, keepdims=True), name="sum")(model) if eucl_dist_lvl >= 4: model = Lambda(lambda tensor: tf.sqrt(tensor), name="root")(model) if batch_norm: model = BatchNormalization(name="input_normalization")(model) for i in range(n_layers): model = Dense(n_neurons, activation='sigmoid', name="dense_{}".format(i))(model) model_out = Dense(1, activation='sigmoid', name="classify")(model) model = Model([in_1, in_2], model_out) model.compile(loss=loss, optimizer=optimizer, metrics=[ BinaryAccuracy(), Precision(), Recall(), TrueNegatives(), FalsePositives(), FalseNegatives(), TruePositives() ]) return model
def discriminator(height,width,channels,label_dim_1,label_dim_2,learning_rate): input_image = Input((height,width,channels)) input_label = Input((label_dim_1,)) input_gen = Input((label_dim_2,)) label = Concatenate()([input_label, input_gen]) label = Dense(HEIGHT * WIDTH)(label) label = LeakyReLU()(label) label = Reshape((HEIGHT, WIDTH, 1))(label) disc = Concatenate()([input_image,label]) disc = Conv2D(128, 4,strides=2,padding="same")(disc) disc = LeakyReLU()(disc) disc = Conv2D(256, 4,strides=2,padding="same")(disc) disc = LeakyReLU()(disc) disc = Conv2D(512, 4,strides=2,padding="same")(disc) #disc = BatchNormalization()(disc) disc = LeakyReLU()(disc) disc = Conv2D(1024, 4,strides=2,padding="same")(disc) disc = LeakyReLU()(disc) disc = Flatten()(disc) disc = Dropout(0.4)(disc) disc = Dense(1, activation="sigmoid")(disc) disc = Model([input_image,input_label,input_gen],disc) discriminator_optimizer = Adam(lr=LEARNING_RATE,beta_1=0.5) disc.compile(discriminator_optimizer, loss="binary_crossentropy",metrics=['binary_accuracy']) return disc
(train_images_mnist.shape[0], 28, 28, 1)) test_images_mnist = test_images_mnist.reshape( (test_images_mnist.shape[0], 28, 28, 1)) # print(train_images_mnist.shape) # print(test_images_mnist) print('test_images_mnist:', test_images_mnist.shape) print('test_labels_mnist:', test_labels_mnist.shape) sgd = optimizers.SGD(learning_rate=0.01, decay=1e-6, momentum=0.9, nesterov=True) model.compile( optimizer=sgd, loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy']) history = model.fit(train_images_mnist, train_labels_mnist, batch_size=256, epochs=200, validation_data=(test_images_mnist, test_labels_mnist)) plt.plot(history.history["accuracy"]) plt.plot(history.history['val_accuracy']) plt.plot(history.history['loss']) plt.plot(history.history['val_loss']) plt.title("model accuracy") plt.ylabel("Accuracy") plt.xlabel("Epoch")
def LSTM_model_veracity(x_train_embeddings, x_train_metafeatures, y_train, x_test_embeddings, x_test_metafeatures, params, eval=False, use_embeddings=True, use_metafeatures=True, Early_Stopping=True, log_path=""): # Parameter search log_dir = log_path + datetime.datetime.now().strftime("%d%m%Y-%H%M%S") num_lstm_units = int(params['num_lstm_units']) num_lstm_layers = int(params['num_lstm_layers']) num_dense_layers = int(params['num_dense_layers']) num_dense_units = int(params['num_dense_units']) num_epochs = params['num_epochs'] learn_rate = params['learn_rate'] mb_size = params['mb_size'] l2reg = params['l2reg'] dropout = params['dropout'] attention = params['attention'] # Defining input shapes if use_embeddings: emb_shape = x_train_embeddings[0].shape if use_metafeatures: metafeatures_shape = x_train_metafeatures[0].shape # Creating the two inputs if use_embeddings: emb_input = Input(shape=emb_shape, name='Embeddings') if use_metafeatures: metafeatures_input = Input(shape=metafeatures_shape, name='Metafeatures') # Adding masks to account for zero paddings if use_embeddings: emb_mask = Masking(mask_value=0, input_shape=(None, emb_shape))(emb_input) if use_metafeatures: metafeatures_mask = (Masking( mask_value=0, input_shape=(None, metafeatures_shape)))(metafeatures_input) # Adding attention and LSTM layers with varying layers and units using parameter search if attention == 1: for nl in range(num_lstm_layers): if use_embeddings: emb_LSTM_query = Bidirectional( LSTM(num_lstm_units, dropout=dropout, recurrent_dropout=0.2, return_sequences=True))(emb_mask) emb_LSTM_value = Bidirectional( LSTM(num_lstm_units, dropout=dropout, recurrent_dropout=0.2, return_sequences=True))(emb_mask) if use_metafeatures: metafeatures_LSTM_query = Bidirectional( LSTM(num_lstm_units, dropout=dropout, recurrent_dropout=0.2, return_sequences=True))(metafeatures_mask) metafeatures_LSTM_value = Bidirectional( LSTM(num_lstm_units, dropout=dropout, recurrent_dropout=0.2, return_sequences=True))(metafeatures_mask) if use_embeddings: emb_LSTM = AdditiveAttention(name='Attention_Embeddings')( [emb_LSTM_query, emb_LSTM_value]) if use_metafeatures: metafeatures_LSTM = AdditiveAttention( name='Attention_Metafeatures')( [metafeatures_LSTM_query, metafeatures_LSTM_value]) else: if use_embeddings: emb_LSTM = Bidirectional( LSTM(num_lstm_units, dropout=dropout, recurrent_dropout=dropout, return_sequences=True))(emb_mask) if use_metafeatures: metafeatures_LSTM = Bidirectional( LSTM(num_lstm_units, dropout=dropout, recurrent_dropout=dropout, return_sequences=True))(metafeatures_mask) if use_embeddings and use_metafeatures: # Concatenating the two inputs model = Concatenate()([emb_LSTM, metafeatures_LSTM]) elif use_metafeatures: model = metafeatures_LSTM # Adding attention and another LSTM to the concatenated layers if attention == 1: model_query = Bidirectional( LSTM(num_lstm_units, dropout=dropout, recurrent_dropout=0.2, return_sequences=False))(model) model_value = Bidirectional( LSTM(num_lstm_units, dropout=dropout, recurrent_dropout=0.2, return_sequences=False))(model) model = AdditiveAttention(name='Attention_Model')( [model_query, model_value]) else: model = Bidirectional( LSTM(num_lstm_units, dropout=dropout, recurrent_dropout=dropout, return_sequences=False))(model) # Adding dense layer with varying layers and units using parameter search for nl in range(num_dense_layers): model = Dense(num_dense_units)(model) model = LeakyReLU()(model) # Adding dropout to the model model = Dropout(dropout)(model) # Adding softmax dense layer with varying l2 regularizers using parameter search output = Dense(3, activation='softmax', activity_regularizer=regularizers.l2(l2reg), name='labels')(model) # Model output if use_embeddings and use_metafeatures: model = Model(inputs=[emb_input, metafeatures_input], outputs=output) elif use_metafeatures: model = Model(inputs=metafeatures_input, outputs=output) #model = Model(inputs=emb_input, outputs=output) # Plotting the model #plot_model(model, to_file='model_plot.png', show_shapes=True, show_layer_names=True) # Adding Adam optimizer with varying learning rate using parameter search adam = optimizers.Adam(lr=learn_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0) # Compiling model model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['accuracy']) callback_list = [] #TensorBoard tensorboard_callback = TensorBoard(log_dir=log_dir, histogram_freq=1) callback_list.append(tensorboard_callback) #Early_Stopping if Early_Stopping: earlystop_callback = EarlyStopping(monitor='val_accuracy', min_delta=0.0001, patience=5) callback_list.append(earlystop_callback) #plot_model(model, "model.png") if Early_Stopping: # Fitting the model with varying batch sizes and epochs using parameter search if use_embeddings and use_metafeatures: model.fit( { 'Embeddings': x_train_embeddings, 'Metafeatures': x_train_metafeatures }, y_train, batch_size=mb_size, epochs=num_epochs, shuffle=True, class_weight=None, verbose=1, callbacks=callback_list, validation_split=.1) elif use_metafeatures: model.fit(x_train_metafeatures, y_train, batch_size=mb_size, epochs=num_epochs, shuffle=True, class_weight=None, verbose=1, callbacks=callback_list, validation_split=.1) else: # Fitting the model with varying batch sizes and epochs using parameter search if use_embeddings and use_metafeatures: model.fit( { 'Embeddings': x_train_embeddings, 'Metafeatures': x_train_metafeatures }, y_train, batch_size=mb_size, epochs=num_epochs, shuffle=True, class_weight=None, verbose=1, callbacks=callback_list) elif use_metafeatures: model.fit(x_train_metafeatures, y_train, batch_size=mb_size, epochs=num_epochs, shuffle=True, class_weight=None, verbose=1, callbacks=callback_list) # Evaluation time if eval == True: model.save('output\\model_veracity.h5') json_string = model.to_json() with open('output\\model_architecture_veracity.json', 'w') as fout: json.dump(json_string, fout) model.save_weights('output\\model_veracity_weights.h5') # Getting confidence of the model if use_embeddings and use_metafeatures: pred_probabilities = model.predict( [x_test_embeddings, x_test_metafeatures], batch_size=mb_size, verbose=0) confidence = np.max(pred_probabilities, axis=1) # Getting predictions of the model y_prob = model.predict([x_test_embeddings, x_test_metafeatures], batch_size=mb_size) Y_pred = y_prob.argmax(axis=-1) elif use_metafeatures: pred_probabilities = model.predict(x_test_metafeatures, batch_size=mb_size, verbose=0) confidence = np.max(pred_probabilities, axis=1) # Getting predictions of the model y_prob = model.predict(x_test_metafeatures, batch_size=mb_size) Y_pred = y_prob.argmax(axis=-1) return Y_pred, confidence
def compileComponent(self,verbose=False,component_type=None): if(component_type=='discriminator'): stream = open(self.layout_dir+"/discriminator_layout.yaml","r+") elif(component_type=='generator'): stream = open(self.layout_dir+"/generator_layout.yaml","r+") data = yaml.load(stream, Loader=yaml.FullLoader) g1_instructions= data['join'][0] g2_instructions= data['join'][1] gc_instructions = data['layers'] if(component_type=='discriminator'): g_in_1 = Input((1,)) elif(component_type=='generator'): g_in_1 = Input((self.noise,)) g_in_2=[] for i in range(self.dimensionality): g_in_2.append(Input((1,))) g1 = (g_in_1) for layer_info in g1_instructions['layers']: if(self.overrides.get('activation')): activation = self.overrides.get('activation') else: activation = layer_info.get('activation') if(component_type=='discriminator' and self.overrides.get('d_nodes')): units = self.overrides.get('d_nodes') elif(component_type=='generator' and self.overrides.get('g_nodes')): units = self.overrides.get('g_nodes') else: units = layer_info.get('nodes') if(self.overrides.get('dropout_amount')): rate = self.overrides.get('dropout_amount') else: rate = layer_info.get('dropout_amount') if(self.overrides.get('leaky_amount')): alpha = self.overrides.get('leaky_amount') else: alpha = layer_info.get('leaky_amount') if(layer_info['layer_type']=='dense'): g1=Dense(units=units,activation=activation)(g1) elif(layer_info['layer_type']=='dropout'): g1=Dropout(rate=rate)(g1) elif(layer_info['layer_type']=='selu'): g1=LeakyReLU(alpha=alpha)(g1) elif(layer_info['layer_type']=='batchnorm'): g1=BatchNormalization()(g1) g2=[] for i in range(self.dimensionality): g2_current = (g_in_2[i]) for layer_info in g2_instructions['layers']: if(self.overrides.get('activation')): activation = self.overrides.get('activation') else: activation = layer_info.get('activation') if(component_type=='discriminator' and self.overrides.get('d_nodes')): units = self.overrides.get('d_nodes') elif(component_type=='generator' and self.overrides.get('g_nodes')): units = self.overrides.get('g_nodes') else: units = layer_info.get('nodes') if(self.overrides.get('dropout_amount')): rate = self.overrides.get('dropout_amount') else: rate = layer_info.get('dropout_amount') if(self.overrides.get('leaky_amount')): alpha = self.overrides.get('leaky_amount') else: alpha = layer_info.get('leaky_amount') if(layer_info['layer_type']=='dense'): g2_current=Dense(units=units,activation=activation)(g2_current) elif(layer_info['layer_type']=='dropout'): g2_current=Dropout(rate=rate)(g2_current) elif(layer_info['layer_type']=='selu'): g2_current=LeakyReLU(alpha=alpha)(g2_current) elif(layer_info['layer_type']=='batchnorm'): g2_current=BatchNormalization()(g2_current) g2.append(g2_current) gc = Concatenate()([g1]+g2) for layer_info in gc_instructions: if(self.overrides.get('activation')): activation = self.overrides.get('activation') else: activation = layer_info.get('activation') if(component_type=='discriminator' and self.overrides.get('d_nodes')): units = self.overrides.get('d_nodes') elif(component_type=='generator' and self.overrides.get('g_nodes')): units = self.overrides.get('g_nodes') else: units = layer_info.get('nodes') if(self.overrides.get('dropout_amount')): rate = self.overrides.get('dropout_amount') else: rate = layer_info.get('dropout_amount') if(self.overrides.get('leaky_amount')): alpha = self.overrides.get('leaky_amount') else: alpha = layer_info.get('leaky_amount') if(layer_info['layer_type']=='dense'): gc=Dense(units=units,activation=activation)(gc) elif(layer_info['layer_type']=='dropout'): gc=Dropout(rate=rate)(gc) elif(layer_info['layer_type']=='selu'): gc=LeakyReLU(alpha=alpha)(gc) elif(layer_info['layer_type']=='batchnorm'): gc=BatchNormalization()(gc) if(component_type=='generator'): gc = Dense(1, activation="sigmoid")(gc) gc = Model(name="Generator", inputs=[g_in_1]+g_in_2, outputs=[gc]) elif(component_type=='discriminator'): gc = Dense(2, activation="softmax")(gc) gc = Model(name="Discriminator", inputs=[g_in_1]+g_in_2, outputs=[gc]) gc.compile(loss="categorical_crossentropy", optimizer=Adam(self.d_training_rate, beta_1=self.d_beta1), metrics=["accuracy"]) if(verbose): gc.summary() return gc