from config import Config

from pprint import pprint, pformat

from logger import model_logger

log = model_logger.getLogger('main')

import logging

log.setLevel(logging.INFO)

import pickle

import random

from nltk.tokenize import WordPunctTokenizer

word_punct_tokenizer = WordPunctTokenizer()

word_tokenize = word_punct_tokenizer.tokenize

from torch import nn, optim

from torch.nn import functional as F

from torch.autograd import Variable

import torch

from trainer import Trainer, Feeder, Predictor

from datafeed import DataFeed

from utilz import tqdm, ListTable

from collections import namedtuple, defaultdict

Sample = namedtuple('Sample', ['id','comment_text',

'toxic','severe_toxic','obscene',

'threat','insult','identity_hate'])

if Config().flush:

import csv

train_dataset = csv.reader(open('dataset/train.csv'))

test_dataset = csv.reader(open('dataset/test.csv'))

# ### Unicode to ascii text

import unicodedata

train_datapoints = []

for i in list(train_dataset)[1:]:

_id, c, t, st, o, t, ins, ih = i

t, st, o, t, ins, ih = (int(_) for _ in [t, st, o, t, ins, ih])

c = unicodedata.normalize('NFD', c).encode('ascii','ignore').decode()

train_datapoints.append(Sample(_id, c.lower(), t, st, o, t, ins, ih))

test_datapoints = []

for i in list(test_dataset)[1:]:

_id, c = i

c = unicodedata.normalize('NFD', c).encode('ascii','ignore').decode()

test_datapoints.append(Sample(_id, c.lower(), 0, 0, 0, 0, 0, 0))

print('train: {}, test: {}'.format(len(train_datapoints), len(test_datapoints)))

if Config().find_lengths:

lengths = [len(word_tokenize(p.comment_text)) for p in tqdm(train_datapoints)]

print('min/max lengths train_datapoints: {}/{}'.format(min(lengths), max(lengths)))

lengths = [len(word_tokenize(p.comment_text)) for p in tqdm(test_datapoints)]

print('min/max lengths test_datapoints: {}/{}'.format(min(lengths), max(lengths)))

config = Config()

seq_len_criteria = lambda p: len(word_tokenize(p.comment_text)) < config.seq_len_limit and len(word_tokenize(p.comment_text)) > 0

train_datapoints = [p for p in tqdm(train_datapoints) if seq_len_criteria(p)]

#test_datapoints = [p for p in tqdm(test_datapoints) if seq_len_criteria(p)]

print('train: {}, test: {}'.format(len(train_datapoints), len(test_datapoints)))

cond = lambda x: sum(x[2:]) >= 1

classified_train_datapoints = [p for p in train_datapoints if cond(p)]

sort_key = lambda p: len(word_tokenize(p.comment_text))

split_index = int( len(classified_train_datapoints) * Config().split_ratio )

classified_datapoints = sorted(classified_train_datapoints[:split_index], key=sort_key), sorted(classified_train_datapoints[split_index:], key=sort_key)

split_index = int( len(train_datapoints) * Config().split_ratio )

non_classified_datapoints = sorted(train_datapoints[:split_index], key=sort_key), sorted(train_datapoints[split_index:], key=sort_key)

test_datapoints = sorted(test_datapoints, key=lambda p: -len(word_tokenize(p.comment_text)))

# ## Build vocabulary

# #### buils INPUT_VOCAB

datapoints = train_datapoints + test_datapoints

WORD_FREQ = defaultdict(int)

CHAR_FREQ = defaultdict(int)

CHAR_VOCAB = [' '] + list(set([c for dp in tqdm(datapoints) for c in dp.comment_text]))

CHAR_INDEX = {c: i for i, w in enumerate(CHAR_VOCAB)}

OUTPUT_VOCAB = ['toxic','severe_toxic','obscene', 'threat','insult','identity_hate']

INPUT_VOCAB = [word for dp in tqdm(datapoints) for word in word_tokenize(dp.comment_text)]

INPUT_VOCAB = INPUT_VOCAB + OUTPUT_VOCAB

for word in INPUT_VOCAB:

WORD_FREQ[word] += 1

WORD_FREQ_PAIRS = sorted(WORD_FREQ.items(), key=lambda x: -x[1])

INPUT_VOCAB = [ x[0] for x in WORD_FREQ_PAIRS ]

print(WORD_FREQ_PAIRS[:100], WORD_FREQ_PAIRS[-100:])

print('Vocab size: {}'.format(len(INPUT_VOCAB)))

INPUT_VOCAB = ['<<PAD>>', '<<UNK>>'] + INPUT_VOCAB + OUTPUT_VOCAB

WORD_INDEX = defaultdict(lambda : INPUT_VOCAB.index('<<UNK>>'))

INPUT_VOCAB = INPUT_VOCAB[ :Config().input_vocab_size ]

WORD_INDEX.update( {w: i for i, w in enumerate(INPUT_VOCAB)} )

OUTPUT_WORD_INDEX = {w: i for i, w in enumerate(OUTPUT_VOCAB)}

OUTPUT_IDS = [OUTPUT_WORD_INDEX[i] for i in OUTPUT_VOCAB]

PAD = WORD_INDEX[INPUT_VOCAB[0]]

print('selvakumar is so stupid that he has no sense of purpose', WORD_INDEX['selvakumar is so stupid that he has no sense of purpose'])

# caching

pickle.dump([CHAR_VOCAB, CHAR_INDEX,

INPUT_VOCAB, OUTPUT_VOCAB,

OUTPUT_IDS, PAD,

dict(WORD_INDEX), WORD_FREQ_PAIRS,

test_datapoints, train_datapoints, classified_datapoints, non_classified_datapoints], open('cache.pkl', 'wb'))

else:

[CHAR_VOCAB, CHAR_INDEX,

INPUT_VOCAB, OUTPUT_VOCAB,

OUTPUT_IDS, PAD,

WORD_INDEX_DICT, WORD_FREQ_PAIRS,

test_datapoints, train_datapoints, classified_datapoints, non_classified_datapoints] = pickle.load(open('cache.pkl', 'rb'))

WORD_INDEX = defaultdict(lambda : INPUT_VOCAB.index('<<UNK>>'))

WORD_INDEX.update(WORD_INDEX_DICT)

#train_datapoints, test_datapoints = train_datapoints[:2000], test_datapoints[:2000]

print(sorted(list(WORD_INDEX.items()), key=lambda x: x[1])[:10], WORD_INDEX['<<PAD>>'], INPUT_VOCAB[0], INPUT_VOCAB[ WORD_INDEX['<<PAD>>'] ])

# ## tests INPUTVOCAB and WORD_INDEX mapping

_i = train_datapoints[random.choice(range(len(train_datapoints)))]

print(_i.comment_text)

print("======")

print(' '.join( [INPUT_VOCAB[i] for i in

[WORD_INDEX[j] for j in word_tokenize(_i.comment_text)]]) )

# ## Baseline model

class Model(nn.Module):

return ret

# ### Batching utils

import numpy as np

def seq_maxlen(seqs):

return max([len(seq) for seq in seqs])

print(PAD)

def pad_seq(seqs, maxlen=0, PAD=PAD):

return seqs

def batchop(datapoints, *args, **kwargs):

return indices, (seq, ), (target,)

def char_emb_batchop(datapoints, *args, **kwargs):

return indices, (seq, ), (target,)

def test_batchop(datapoints, *args, **kwargs):

return indices, (seq, ), ()

# ## Attention based model

import math

class AttModel(nn.Module):

return ret

class DecoderModel(nn.Module):

return ret

class BiLSTMDecoderModel(nn.Module):

return ret

# ## Loss and accuracy function

def loss(output, target, loss_function=nn.NLLLoss(), *args, **kwargs):

return (loss/batch_size)

def accuracy(output, target, *args, **kwargs):

return (f1/batch_size)

# ### repr_function to build human readable output from model

from IPython.display import HTML

from IPython.display import display

def repr_function(output, feed, batch_index):

return results

def test_repr_function(output, feed, batch_index):

return results

# ## Experiment on model 1

import random

def experiment(epochs=10, checkpoint=1, train_datapoints=train_datapoints):

trainer.train()

# ## Experiment on model using attention

import random

import gc

def experiment(eons=1000, epochs=1, checkpoint=1):

dump.close()

# ## Experiment on model using attention

import random

import gc

def experiment(eons=1000, epochs=1, checkpoint=1):

return locals()

exp_image = experiment()