sentimeme.py

# -*- coding: utf-8 -*-
"""SentiMeme.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1FjwMqCWVw-ButYzxP2N7VU528PPDaGsE
"""

import re
import string
import numpy as np
import pandas as pd
from tqdm import tqdm
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score

import tensorflow as tf
from tensorflow.keras import Sequential, Model
from tensorflow.keras.layers import Conv2D, MaxPool2D, GlobalAveragePooling2D
from tensorflow.keras.layers import Dense, Flatten, BatchNormalization, Activation, Dropout
from tensorflow.keras.layers import Conv1D, Embedding, GlobalAveragePooling1D 
from tensorflow.keras.optimizers import Adam, RMSprop
from tensorflow.keras.preprocessing import image

from PIL import ImageFile
ImageFile.LOAD_TRUNCATED_IMAGES = True

pip install keras-self-attention

from google.colab import drive
drive.mount('/content/drive')

ls

df = pd.read_csv('/content/drive/MyDrive/Deep Learning/Deep Learning - Final Project/Image/memotion_dataset_7k/labels.csv')
#df = pd.read_csv('/content/drive/MyDrive/Deep Learning - Final Project/Image/memotion_dataset_7k/labels.csv')

df.drop(df.columns[df.columns.str.contains('unnamed',case = False)],axis = 1, inplace = True)
df.head()

cleaned = df.copy()
cleaned.dropna(inplace=True)
cleaned.isnull().any()

width = 100
height = 100
X = []
for i in tqdm(range(cleaned.shape[0])):
    if i in [119, 4799, 6781, 6784, 6786]:
        pass
    else:
        path = '/content/drive/MyDrive/Deep Learning - Final Project/Image/memotion_dataset_7k/images/'+cleaned['image_name'][i]
        img = image.load_img(path,target_size=(width,height,3))
        img = image.img_to_array(img)
        img = img/255.0
        X.append(img)
        
X = np.array(X)

np.save('/content/drive/MyDrive/Deep Learning - Final Project/Image/memotion_dataset_7k/images_to_array.npy', X)

X.shape

X = np.load('/content/drive/MyDrive/Deep Learning/Deep Learning - Final Project/Image/memotion_dataset_7k/images_to_array.npy')
#X = np.load('/content/drive/MyDrive/Deep Learning - Final Project/Image/memotion_dataset_7k/images_to_array.npy')

rows_to_drop = ['image_120.jpg',
              'image_4800.jpg',
              'image_6782.jpg',
              'image_6785.jpg',
              'image_6787.jpg',
              'image_6988.jpg',
              'image_6989.jpg',
              'image_6990.png',
              'image_6991.jpg',
              'image_6992.jpg']

for images in rows_to_drop:
    cleaned.drop(cleaned[cleaned['image_name'] == images].index, inplace=True)

cleaned['overall_sentiment'].value_counts()

target = cleaned['overall_sentiment']
target = pd.get_dummies(target)

target

cleaned

"""
A Keras version of `botnet`.
Original TensorFlow version: https://gist.github.com/aravindsrinivas/56359b79f0ce4449bcb04ab4b56a57a2
"""

import math
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.python.keras import backend
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import special_math_ops
from tensorflow.python.util.tf_export import keras_export

BATCH_NORM_DECAY = 0.9
BATCH_NORM_EPSILON = 1e-5


@keras_export("keras.layers.MHSAWithRelativePosition")
class MHSAWithRelativePosition(keras.layers.MultiHeadAttention):
    def __init__(self, num_heads=4, bottleneck_dimension=512, relative=True, **kwargs):
        self.key_dim = bottleneck_dimension // num_heads
        super(MHSAWithRelativePosition, self).__init__(num_heads=num_heads, key_dim=self.key_dim, **kwargs)
        self.num_heads, self.bottleneck_dimension, self.relative = num_heads, bottleneck_dimension, relative

    def _build_from_signature(self, featuremap):
        super(MHSAWithRelativePosition, self)._build_from_signature(query=featuremap, value=featuremap)
        _, hh, ww, _ = featuremap.shape
        stddev = self.key_dim ** -0.5
        self.rel_emb_w = self.add_weight(
            name="r_width",
            shape=(self.key_dim, 2 * ww - 1),
            initializer=tf.random_normal_initializer(stddev=stddev),
            trainable=True,
            dtype=featuremap.dtype,
        )
        self.rel_emb_h = self.add_weight(
            name="r_height",
            shape=(self.key_dim, 2 * hh - 1),
            initializer=tf.random_normal_initializer(stddev=stddev),
            trainable=True,
            dtype=featuremap.dtype,
        )

    def get_config(self):
        base_config = super(MHSAWithRelativePosition, self).get_config()
        base_config.pop("key_dim", None)
        base_config.update(
            {"num_heads": self.num_heads, "bottleneck_dimension": self.bottleneck_dimension, "relative": self.relative}
        )
        return base_config

    def rel_to_abs(self, rel_pos):
        """
        Converts relative indexing to absolute.
        Input: [bs, heads, height, width, 2*width - 1]
        Output: [bs, heads, height, width, width]
        """
        _, heads, hh, ww, dim = rel_pos.shape
        col_pad = tf.zeros_like(rel_pos[:, :, :, :, :1], dtype=rel_pos.dtype)
        rel_pos = tf.concat([rel_pos, col_pad], axis=-1)
        flat_x = tf.reshape(rel_pos, [-1, heads, hh, ww * 2 * ww])
        flat_pad = tf.zeros_like(flat_x[:, :, :, : ww - 1], dtype=rel_pos.dtype)
        flat_x_padded = tf.concat([flat_x, flat_pad], axis=-1)
        final_x = tf.reshape(flat_x_padded, [-1, heads, hh, ww + 1, 2 * ww - 1])
        final_x = final_x[:, :, :, :ww, ww - 1 :]
        return final_x

    def relative_logits_1d(self, query, rel_k, transpose_mask):
        """
        Compute relative logits along one dimenion.
        `q`: [bs, heads, height, width, dim]
        `rel_k`: [dim, 2*width - 1]
        """
        _, _, hh, _, _ = query.shape
        rel_logits = tf.matmul(query, rel_k)
        rel_logits = self.rel_to_abs(rel_logits)
        rel_logits = tf.expand_dims(rel_logits, axis=3)
        rel_logits = tf.tile(rel_logits, [1, 1, 1, hh, 1, 1])
        rel_logits = tf.transpose(rel_logits, transpose_mask)
        return rel_logits

    def relative_logits(self, query):
        query = tf.transpose(query, [0, 3, 1, 2, 4])
        rel_logits_w = self.relative_logits_1d(query=query, rel_k=self.rel_emb_w, transpose_mask=[0, 1, 2, 4, 3, 5])
        query = tf.transpose(query, [0, 1, 3, 2, 4])
        rel_logits_h = self.relative_logits_1d(query=query, rel_k=self.rel_emb_h, transpose_mask=[0, 1, 4, 2, 5, 3])
        return rel_logits_h + rel_logits_w

    def call(self, inputs, attention_mask=None, return_attention_scores=False, training=None):
        if not self._built_from_signature:
            self._build_from_signature(featuremap=inputs)
        #   N = `num_attention_heads`
        #   H = `size_per_head`
        # `query` = [B, T, N ,H]
        query = self._query_dense(inputs)

        # `key` = [B, S, N, H]
        key = self._key_dense(inputs)

        # `value` = [B, S, N, H]
        value = self._value_dense(inputs)

        query = math_ops.multiply(query, 1.0 / math.sqrt(float(self._key_dim)))
        attention_scores = special_math_ops.einsum(self._dot_product_equation, key, query)
        if self.relative:
            attention_scores += self.relative_logits(query)
        attention_scores = self._masked_softmax(attention_scores, attention_mask)
        attention_scores_dropout = self._dropout_layer(attention_scores, training=training)
        attention_output = special_math_ops.einsum(self._combine_equation, attention_scores_dropout, value)

        # attention_output = self._output_dense(attention_output)
        hh, ww = inputs.shape[1], inputs.shape[2]
        attention_output = tf.reshape(attention_output, [-1, hh, ww, self.num_heads * self.key_dim])

        if return_attention_scores:
            return attention_output, attention_scores
        return attention_output


def batchnorm_with_activation(inputs, activation="relu", zero_gamma=False, name=""):
    """Performs a batch normalization followed by an activation. """
    bn_axis = 3 if backend.image_data_format() == "channels_last" else 1
    gamma_initializer = tf.zeros_initializer() if zero_gamma else tf.ones_initializer()
    nn = layers.BatchNormalization(
        axis=bn_axis,
        momentum=BATCH_NORM_DECAY,
        epsilon=BATCH_NORM_EPSILON,
        gamma_initializer=gamma_initializer,
        name=name + "bn",
    )(inputs)
    if activation:
        nn = layers.Activation(activation=activation, name=name + activation)(nn)
    return nn


def conv2d_no_bias(inputs, filters, kernel_size, strides=1, padding="VALID", name=""):
    return layers.Conv2D(filters, kernel_size, strides=strides, padding=padding, use_bias=False, name=name + "conv")(inputs)


def bot_block(
    featuremap,
    heads=4,
    proj_factor=4,
    activation="relu",
    pos_enc_type="relative",
    strides=1,
    target_dimension=2048,
    name="all2all",
    use_MHSA=True,
):
    if strides != 1 or featuremap.shape[-1] != target_dimension:
        padding = "SAME" if strides == 1 else "VALID"
        shortcut = conv2d_no_bias(featuremap, target_dimension, 1, strides=strides, padding=padding, name=name + "_0_")
        shortcut = batchnorm_with_activation(shortcut, activation=activation, zero_gamma=False, name=name + "_0_")
    else:
        shortcut = featuremap

    bottleneck_dimension = target_dimension // proj_factor

    if use_MHSA:    # BotNet block
        nn = conv2d_no_bias(featuremap, bottleneck_dimension, 1, strides=1, padding="VALID", name=name + "_1_")
        nn = batchnorm_with_activation(nn, activation=activation, zero_gamma=False, name=name + "_1_")
        nn = MHSAWithRelativePosition(num_heads=heads, bottleneck_dimension=bottleneck_dimension, name=name + "_2_mhsa")(nn)
        if strides != 1:
            nn = layers.AveragePooling2D(pool_size=(2, 2), strides=(2, 2), padding="same")(nn)
    else:   # ResNet block
        nn = conv2d_no_bias(featuremap, bottleneck_dimension, 1, strides=strides, padding="VALID", name=name + "_1_")
        nn = batchnorm_with_activation(nn, activation=activation, zero_gamma=False, name=name + "_1_")
        nn = conv2d_no_bias(nn, bottleneck_dimension, 3, strides=1, padding="SAME", name=name + "_2_")
    nn = batchnorm_with_activation(nn, activation=activation, zero_gamma=False, name=name + "_2_")

    nn = conv2d_no_bias(nn, target_dimension, 1, strides=1, padding="VALID", name=name + "_3_")
    nn = batchnorm_with_activation(nn, activation=None, zero_gamma=True, name=name + "_3_")

    nn = layers.Add(name=name + "_add")([shortcut, nn])
    return layers.Activation(activation, name=name + "_out")(nn)


def bot_stack(
    featuremap,
    target_dimension=2048,
    num_layers=3,
    strides=2,
    activation="relu",
    heads=4,
    proj_factor=4,
    pos_enc_type="relative",
    name="all2all_stack",
    use_MHSA=True,
):
    """ c5 Blockgroup of BoT Blocks. Use `activation=swish` for `silu` """
    for i in range(num_layers):
        featuremap = bot_block(
            featuremap,
            heads=heads,
            proj_factor=proj_factor,
            activation=activation,
            pos_enc_type=pos_enc_type,
            strides=strides if i == 0 else 1,
            target_dimension=target_dimension,
            name=name + "_block{}".format(i+1),
            use_MHSA=use_MHSA,
        )
    return featuremap


def BotNet(
    stack_fn,
    preact,
    use_bias,
    model_name="botnet",
    activation="relu",
    include_top=True,
    weights=None,
    input_shape=None,
    classes=1000,
    classifier_activation="softmax",
    **kwargs
):
    img_input = layers.Input(shape=input_shape)

    nn = layers.ZeroPadding2D(padding=((3, 3), (3, 3)), name="conv1_pad")(img_input)
    nn = layers.Conv2D(64, 7, strides=2, use_bias=use_bias, name="conv1_conv")(nn)

    if not preact:
        nn = batchnorm_with_activation(nn, activation=activation, zero_gamma=False, name="conv1_")
    nn = layers.ZeroPadding2D(padding=((1, 1), (1, 1)), name="pool1_pad")(nn)
    nn = layers.MaxPooling2D(3, strides=2, name="pool1_pool")(nn)

    nn = stack_fn(nn)
    if preact:
        nn = batchnorm_with_activation(nn, activation=activation, zero_gamma=False, name="post_")
    if include_top:
        nn = layers.GlobalAveragePooling2D(name="avg_pool")(nn)
        nn = layers.Dense(classes, activation=classifier_activation, name="predictions")(nn)
    return keras.models.Model(img_input, nn, name=model_name)


def BotNet50(
    strides=2, activation="relu", include_top=True, weights=None, input_tensor=None, input_shape=None, classes=1000, **kwargs
):
    def stack_fn(nn):
        nn = bot_stack(nn, 64 * 4, 3, strides=1, activation=activation, name="conv2", use_MHSA=False)
        nn = bot_stack(nn, 128 * 4, 4, strides=2, activation=activation, name="conv3", use_MHSA=False)
        nn = bot_stack(nn, 256 * 4, 6, strides=2, activation=activation, name="conv4", use_MHSA=False)
        nn = bot_stack(nn, 512 * 4, 3, strides=strides, activation=activation, use_MHSA=True)
        return nn

    return BotNet(stack_fn, False, True, "botnet50", activation, include_top, weights, input_shape, classes, **kwargs)

data_augmentation = tf.keras.Sequential([
  tf.keras.layers.experimental.preprocessing.RandomFlip('horizontal'),
  tf.keras.layers.experimental.preprocessing.RandomContrast([.5,2]),
  tf.keras.layers.experimental.preprocessing.RandomRotation(0.2),
  tf.keras.layers.experimental.preprocessing.RandomZoom(0.1)
])

preprocess_input = tf.keras.applications.resnet_v2.preprocess_input

rescale = tf.keras.layers.experimental.preprocessing.Rescaling(1./127.5, offset= -1)

X_train, X_test, y_train, y_test = train_test_split(X, target, test_size = 0.2, stratify=target)

from sklearn.utils import class_weight
#y_ints = [y_train.values.argmax() for y in y_train]
class_w=class_weight.compute_class_weight('balanced',np.unique(y_train),y_train)
print(class_w)

def image_model():
    image_input = tf.keras.Input(shape=(100, 100, 3), name = 'image_input')
    image_layers = data_augmentation(image_input)
    image_layers = preprocess_input(image_layers)
    layer_bm_1 = BotNet50(input_shape=(100, 100, 3) ,include_top=False)(image_layers)
    layer_bm_1 = Conv2D(2048, kernel_size=2,padding='valid')(layer_bm_1)
    image_layers = GlobalAveragePooling2D()(layer_bm_1)
    image_layers = Dropout(0.2, name = 'dropout_layer')(image_layers)
    return image_input, image_layers

image_input, image_layers = image_model()

from tensorflow.keras.layers.experimental.preprocessing import TextVectorization
vocab_size = 10000
sequence_length = 50

vectorize_layer = TextVectorization(
    max_tokens=vocab_size,
    output_mode='int',
    output_sequence_length=sequence_length)

text_ds = np.asarray(cleaned['text_corrected'])
vectorize_layer.adapt(tf.convert_to_tensor(text_ds))

path_to_glove_file = "/content/drive/MyDrive/Deep Learning/Deep Learning - Final Project/Image/glove.6B/glove.6B.200d.txt"
#path_to_glove_file = "/content/drive/MyDrive/Deep Learning - Final Project/Image/glove.6B/glove.6B.200d.txt"

embeddings_index = {}
with open(path_to_glove_file) as f:
    for line in f:
        word, coefs = line.split(maxsplit=1)
        coefs = np.fromstring(coefs, "f", sep=" ")
        embeddings_index[word] = coefs

print("Found %s word vectors." % len(embeddings_index))

voc = vectorize_layer.get_vocabulary()
word_index = dict(zip(voc, range(len(voc))))
num_tokens = len(voc) + 2
embedding_dim = 200
hits = 0
misses = 0

# Prepare embedding matrix
embedding_matrix = np.zeros((num_tokens, embedding_dim))
for word, i in word_index.items():
    embedding_vector = embeddings_index.get(word)
    if embedding_vector is not None:
        # Words not found in embedding index will be all-zeros.
        # This includes the representation for "padding" and "OOV"
        embedding_matrix[i] = embedding_vector
        hits += 1
    else:
        misses += 1
print("Converted %d words (%d misses)" % (hits, misses))

X_text_train, X_text_test, y_text_train, y_text_test = train_test_split(cleaned['text_corrected'], target, test_size = 0.2, stratify=target)

from tensorflow.keras.layers import Embedding
# from keras_self_attention import SeqSelfAttention


def text_model():
    text_input = tf.keras.Input(shape=(None,), dtype=tf.string, name='text_input')
    text_layers = vectorize_layer(text_input)
    embedding = Embedding(num_tokens,embedding_dim, embeddings_initializer=keras.initializers.Constant(embedding_matrix),trainable=False)(text_layers)
    #query = Dense()
    #embedding = tf.keras.layers.Embedding(vocab_size, 16, name="embedding")(text_layers)
    text_layers = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(512, activation='relu', return_sequences=True))(embedding)
    text_layers = tf.keras.layers.BatchNormalization()(text_layers)
    #text_layers = tf.keras.layers.Bidirectional(tf.keras.layers.GRU(512, activation='relu', return_sequences=True))(embedding)
    #text_layers = tf.keras.layers.BatchNormalization()(text_layers)
    text_layers = tf.keras.layers.Dropout(0.3)(text_layers)
    text_layers = tf.keras.layers.Conv1D(128, 7, padding="valid", activation="relu", strides=3)(text_layers)
    
    text_layers = tf.keras.layers.Conv1D(128, 7, padding="valid", activation="relu", strides=3)(text_layers)
    
    text_layers = tf.keras.layers.GlobalMaxPooling1D()(text_layers)
    
    
    text_layers = tf.keras.layers.Dense(2048, activation="relu")(text_layers)
    text_layers = tf.keras.layers.Dropout(0.5)(text_layers)
    return text_input, text_layers
    
text_input, text_layers = text_model()

def model(layer_1, layer_2, image_input, text_input):
    concatenate = tf.keras.layers.concatenate([layer_1, layer_2], axis=1)
    semi_final_layer = tf.keras.layers.Dense(2048, activation='relu')(concatenate)
    
    prediction_layer = tf.keras.layers.Dense(3, activation='softmax', name = 'task_a')

    output = prediction_layer(semi_final_layer)

    model = tf.keras.Model(inputs = [image_input, text_input] , 
                           outputs = output)
    return model

model = model(image_layers, text_layers, image_input, text_input)

import os
# Define the checkpoint directory to store the checkpoints
checkpoint_dir = '/content/drive/MyDrive/Deep Learning/Deep Learning - Final Project/Model/Test'

# Name of the checkpoint files
checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt_{epoch}")

lr_schedule = keras.optimizers.schedules.ExponentialDecay(
    initial_learning_rate=1e-1,
    decay_steps=400,
    decay_rate=0.9)
#optimizer = keras.optimizers.SGD(learning_rate=lr_schedule)

class PrintLR(tf.keras.callbacks.Callback):
  def on_epoch_end(self, epoch, logs=None):
    print('\nLearning rate for epoch {} is {}'.format(epoch + 1,
                                                      model.optimizer.lr.numpy()))

callbacks = [
    tf.keras.callbacks.TensorBoard(log_dir='./logs'),
    tf.keras.callbacks.LearningRateScheduler(lr_schedule),
    PrintLR()
]

class_weights = {0: 3.68646865 ,
                1:1.05896852,
                2: 0.56040538}
model.compile(optimizer=tf.keras.optimizers.Adam(),
              loss = tf.keras.losses.CategoricalCrossentropy(from_logits=True),
              metrics=['accuracy'])
history = model.fit(x = {"image_input": X_train, "text_input": X_text_train},
                    y = y_train,
                    validation_split=0.15,
                    batch_size=256,
                    epochs=25,
                    class_weight=class_weights
                   )

df_history = pd.DataFrame(history.history)
df_history

model.save_weights('/content/drive/MyDrive/Deep Learning - Final Project/Model/Text/Combined_weights_Auto')

#Combined_weights_Best all negative wrong
model.save_weights('/content/drive/MyDrive/Deep Learning/Deep Learning - Final Project/Model/Combined_weights_Match_Array')
df_history.to_csv('/content/drive/MyDrive/Deep Learning/Deep Learning - Final Project/Model/Combined_weights_Match_Array.csv',index=False)

model.load_weights('/content/drive/MyDrive/Deep Learning/Deep Learning - Final Project/Model/Combined')

#architecture
# Botnet - Only image
# Weight Decay
#Micro F1 score for Task A is  0.49749463135289906
#Macro F1 score for Task A is  0.48311618493453923

# Bi-LSTM and Bi-GRU Keras Embeddings - Only text
# Training Accuracy : 31
# Testing Accuracy : 30
# Weight Decay
#Micro F1 score for Task A is  0.6850393700787402
#Macro F1 score for Task A is  0.40654205607476634

# Bi-LSTM and Bi-GRU Glove Embeddings - Text with glove embeddings
# Training Accuracy : 31
# Testing Accuracy : 30
# Weight Decay
#Micro F1 score for Task A is  0.6850393700787402
#Macro F1 score for Task A is  0.40654205607476634


# BOTNET and Bi-LSTM/GRU Glove Embeddings

# Weight Decay
# Micro F1 score for Task A is  0.5368647100930566
# Macro F1 score for Task A is  0.4783176582967069


# BOTNET and Bi-LSTM/GRU Glove Embeddings and Attention

# Weight Decay
# Micro F1 score for Task A is  0.31496062992125984
# Macro F1 score for Task A is  0.23952095808383236

eval_ = model.evaluate(x = {"image_input": X_test, "text_input": X_text_test},
                    y = y_test,
                    batch_size=32,
                    verbose=1
                   )

prediction = model.predict(x = {"image_input": X_test, "text_input": X_text_test})
pred = np.zeros_like(prediction)
pred[np.arange(len(prediction)), prediction.argmax(1)] = 1
y_true = y_test.values
micro_f1_score = f1_score(y_true[:,1], pred[:,1], average='micro')
macro_f1_score = f1_score(y_true[:,1], pred[:,1], average='macro')

print("Micro F1 score for Task A is ", micro_f1_score)
print("Macro F1 score for Task A is ", macro_f1_score)

print(y_true[:,0].sum())
print(y_true[:,1].sum())
print(y_true[:,2].sum())

print(pred[:,0].sum()) #13
print(pred[:,1].sum()) #128
print(pred[:,2].sum()) #536

"""FOR IMAGE:"""

#For image
model = BotNet50(input_shape=(100, 100, 3) ,classes=3)
class_weights = {0: 7.,
                1:2.,
                2: 1.}
model.compile(optimizer=tf.keras.optimizers.Adam(),
              loss = tf.keras.losses.CategoricalCrossentropy(from_logits=False),
              metrics=['accuracy'])

history = model.fit(x = X_train,
                    y = y_train,
                    validation_split=0.15,
                    batch_size=256,
                    epochs=25,
                    callbacks=callbacks,class_weight =class_weights
                   )

df_history = pd.DataFrame(history.history)
model.save_weights('/content/drive/MyDrive/Deep Learning/Deep Learning - Final Project/Model/Botnet')
df_history.to_csv('/content/drive/MyDrive/Deep Learning/Deep Learning - Final Project/Model/Botnet.csv',index=False)

prediction = model.predict(x=X_test)#(x = {"image_input": X_test, "text_input": X_text_test})
pred = np.zeros_like(prediction)
pred[np.arange(len(prediction)), prediction.argmax(1)] = 1
micro_f1_score = f1_score(y_true[:,1], pred[:,1], average='micro')
macro_f1_score = f1_score(y_true[:,1], pred[:,1], average='macro')

print("Micro F1 score for Task A is ", micro_f1_score)
print("Macro F1 score for Task A is ", macro_f1_score)

"""For Text"""

X_text_train, X_text_test, y_text_train, y_text_test = train_test_split(cleaned['text_corrected'], target, test_size = 0.2, stratify=target)

path_to_glove_file = "/content/drive/MyDrive/Deep Learning/Deep Learning - Final Project/Image/glove.6B/glove.6B.200d.txt"

embeddings_index = {}
with open(path_to_glove_file) as f:
    for line in f:
        word, coefs = line.split(maxsplit=1)
        coefs = np.fromstring(coefs, "f", sep=" ")
        embeddings_index[word] = coefs

print("Found %s word vectors." % len(embeddings_index))

from tensorflow.keras.layers.experimental.preprocessing import TextVectorization
vocab_size = 10000
sequence_length = 50

vectorize_layer = TextVectorization(
    max_tokens=vocab_size,
    output_mode='int',
    output_sequence_length=sequence_length)

text_ds = np.asarray(cleaned['text_corrected'])
vectorize_layer.adapt(tf.convert_to_tensor(text_ds))

model = model(text_layers, text_input)
class_weights = {0: 7.,
                1:2.,
                2: 1.}
model.compile(optimizer=tf.keras.optimizers.Adam(),
              loss = tf.keras.losses.CategoricalCrossentropy(from_logits=False),
              metrics=['accuracy'])

history = model.fit(x = X_text_train,
                    y = y_text_train,
                    validation_split=0.15,
                    batch_size=256,
                    epochs=25,
                    callbacks=callbacks,class_weight =class_weights
                   )

df_history = pd.DataFrame(history.history)
model.save_weights('/content/drive/MyDrive/Deep Learning/Deep Learning - Final Project/Model/LSTM_Glove')
df_history.to_csv('/content/drive/MyDrive/Deep Learning/Deep Learning - Final Project/Model/LSTM_Glove.csv',index=False)

model = model(text_layers, text_input)
model.load_weights('/content/drive/MyDrive/Deep Learning/Deep Learning - Final Project/Model/LSTM')

prediction = model.predict(x=X_text_test)#(x = {"image_input": X_test, "text_input": X_text_test})
pred = np.zeros_like(prediction)
pred[np.arange(len(prediction)), prediction.argmax(1)] = 1
y_true = y_test.values
micro_f1_score = f1_score(y_true[:,1], pred[:,1], average='micro')
macro_f1_score = f1_score(y_true[:,1], pred[:,1], average='macro')

print("Micro F1 score for Task A is ", micro_f1_score)
print("Macro F1 score for Task A is ", macro_f1_score)

# For graphs

import pandas as pd
df_history = pd.read_csv('/content/drive/MyDrive/Deep Learning - Final Project/Model/Combined.csv')
# df_history = pd.read_csv('/content/drive/MyDrive/Deep Learning - Final Project/Model/Botnet.csv')
# df_history = pd.read_csv('/content/drive/MyDrive/Deep Learning - Final Project/Model/LSTM_Glove.csv')

import matplotlib.pyplot as plt

fig, axes = plt.subplots(1,2, figsize=(15, 5))
fig.tight_layout(pad=5.0)

axes[0].plot(df_history.loss, label = 'Training Loss')
axes[0].plot(df_history.val_loss, label = 'Training Loss')
axes[0].set_xlabel('No of Epochs')
axes[0].set_ylabel('Train Loss')
axes[0].set_title('Train Loss vs Epochs')
axes[0].legend()

# axes[1].plot(df_history.val_loss, label = 'Validation Loss', color="red", linestyle='dashed')
# axes[1].set_xlabel('No of Epochs')
# axes[1].set_ylabel('Validation Loss')
# axes[1].set_title('Validation Loss vs Epochs')
# axes[1].legend()

plt.show()

# For presentation

model.load_weights('/content/drive/MyDrive/Deep Learning - Final Project/Model/Combined')

import random, matplotlib
matplotlib.rcParams.update({'font.size': 13})
fig, axes = plt.subplots(figsize=(5, 4))
fig.tight_layout(pad=5.0)

x = list(y_test.columns)

l = 1328#1396#1249#random.randint(0,X_test.shape[0])
print(l)
#axes[0].imshow(X[l,:,:,:])

axes.bar(x, model.predict(x = {"image_input": X_test, "text_input": X_text_test})[l,:])
axes.set_xlabel('')
axes.set_ylabel('Probability')
# axes[1].set_title('Humour Prob.')
axes.set_xticks(x)
axes.set_ylim(0,1)
plt.show()

y_test.iloc[1328,:]

import random, matplotlib
matplotlib.rcParams.update({'font.size': 13})
fig, axes = plt.subplots(figsize=(5, 4))
fig.tight_layout(pad=5.0)

x = list(y_test.columns)

l = 1328#1396#1249#random.randint(0,X_test.shape[0])
print(l)
axes.imshow(X[l,:,:,:])

X_train.shape

a= np.where(y_true[:,0]==1)
b =np.where(pred[:,0]==1)

count=0
for i in range(0,len(a[0])):
  for j in range(0,len(b[0])):
    if(a[0][i]!=b[0][j]):
      count+=1
      print(a[0][i])