u":‑\)":"Happy face or smiley",

u":\)":"Happy face or smiley",

u":-\]":"Happy face or smiley",

u":\]":"Happy face or smiley",

u":-3":"Happy face smiley",

u":3":"Happy face smiley",

u":->":"Happy face smiley",

u":>":"Happy face smiley",

u"8-\)":"Happy face smiley",

u":o\)":"Happy face smiley",

u":-\}":"Happy face smiley",

u":\}":"Happy face smiley",

u":-\)":"Happy face smiley",

u":c\)":"Happy face smiley",

u":\^\)":"Happy face smiley",

u"=\]":"Happy face smiley",

u"=\)":"Happy face smiley",

u":‑D":"Laughing, big grin or laugh with glasses",

u":D":"Laughing, big grin or laugh with glasses",

u"8‑D":"Laughing, big grin or laugh with glasses",

u"8D":"Laughing, big grin or laugh with glasses",

u"X‑D":"Laughing, big grin or laugh with glasses",

u"XD":"Laughing, big grin or laugh with glasses",

u"=D":"Laughing, big grin or laugh with glasses",

u"=3":"Laughing, big grin or laugh with glasses",

u"B\^D":"Laughing, big grin or laugh with glasses",

u":-\)\)":"Very happy",

u":‑\(":"Frown, sad, andry or pouting",

u":-\(":"Frown, sad, andry or pouting",

u":\(":"Frown, sad, andry or pouting",

u":‑c":"Frown, sad, andry or pouting",

u":c":"Frown, sad, andry or pouting",

u":‑<":"Frown, sad, andry or pouting",

u":<":"Frown, sad, andry or pouting",

u":‑\[":"Frown, sad, andry or pouting",

u":\[":"Frown, sad, andry or pouting",

u":-\|\|":"Frown, sad, andry or pouting",

u">:\[":"Frown, sad, andry or pouting",

u":\{":"Frown, sad, andry or pouting",

u":@":"Frown, sad, andry or pouting",

u">:\(":"Frown, sad, andry or pouting",

u":'‑\(":"Crying",

u":'\(":"Crying",

u":'‑\)":"Tears of happiness",

u":'\)":"Tears of happiness",

u"D‑':":"Horror",

u"D:<":"Disgust",

u"D:":"Sadness",

u"D8":"Great dismay",

u"D;":"Great dismay",

u"D=":"Great dismay",

u"DX":"Great dismay",

u":‑O":"Surprise",

u":O":"Surprise",

u":‑o":"Surprise",

u":o":"Surprise",

u":-0":"Shock",

u"8‑0":"Yawn",

u">:O":"Yawn",

u":-\*":"Kiss",

u":\*":"Kiss",

u":X":"Kiss",

u";‑\)":"Wink or smirk",

u";\)":"Wink or smirk",

u"\*-\)":"Wink or smirk",

u"\*\)":"Wink or smirk",

u";‑\]":"Wink or smirk",

u";\]":"Wink or smirk",

u";\^\)":"Wink or smirk",

u":‑,":"Wink or smirk",

u";D":"Wink or smirk",

u":‑P":"Tongue sticking out, cheeky, playful or blowing a raspberry",

u":P":"Tongue sticking out, cheeky, playful or blowing a raspberry",

u"X‑P":"Tongue sticking out, cheeky, playful or blowing a raspberry",

u"XP":"Tongue sticking out, cheeky, playful or blowing a raspberry",

u":‑Þ":"Tongue sticking out, cheeky, playful or blowing a raspberry",

u":Þ":"Tongue sticking out, cheeky, playful or blowing a raspberry",

u":b":"Tongue sticking out, cheeky, playful or blowing a raspberry",

u"d:":"Tongue sticking out, cheeky, playful or blowing a raspberry",

u"=p":"Tongue sticking out, cheeky, playful or blowing a raspberry",

u">:P":"Tongue sticking out, cheeky, playful or blowing a raspberry",

u":‑/":"Skeptical, annoyed, undecided, uneasy or hesitant",

u":/":"Skeptical, annoyed, undecided, uneasy or hesitant",

u":-[.]":"Skeptical, annoyed, undecided, uneasy or hesitant",

u">:[(\\\)]":"Skeptical, annoyed, undecided, uneasy or hesitant",

u">:/":"Skeptical, annoyed, undecided, uneasy or hesitant",

u":[(\\\)]":"Skeptical, annoyed, undecided, uneasy or hesitant",

u"=/":"Skeptical, annoyed, undecided, uneasy or hesitant",

u"=[(\\\)]":"Skeptical, annoyed, undecided, uneasy or hesitant",

u":L":"Skeptical, annoyed, undecided, uneasy or hesitant",

u"=L":"Skeptical, annoyed, undecided, uneasy or hesitant",

u":S":"Skeptical, annoyed, undecided, uneasy or hesitant",

u":‑\|":"Straight face",

u":\|":"Straight face",

u":$":"Embarrassed or blushing",

u":‑x":"Sealed lips or wearing braces or tongue-tied",

u":x":"Sealed lips or wearing braces or tongue-tied",

u":‑#":"Sealed lips or wearing braces or tongue-tied",

u":#":"Sealed lips or wearing braces or tongue-tied",

u":‑&":"Sealed lips or wearing braces or tongue-tied",

u":&":"Sealed lips or wearing braces or tongue-tied",

u"O:‑\)":"Angel, saint or innocent",

u"O:\)":"Angel, saint or innocent",

u"0:‑3":"Angel, saint or innocent",

u"0:3":"Angel, saint or innocent",

u"0:‑\)":"Angel, saint or innocent",

u"0:\)":"Angel, saint or innocent",

u":‑b":"Tongue sticking out, cheeky, playful or blowing a raspberry",

u"0;\^\)":"Angel, saint or innocent",

u">:‑\)":"Evil or devilish",

u">:\)":"Evil or devilish",

u"\}:‑\)":"Evil or devilish",

u"\}:\)":"Evil or devilish",

u"3:‑\)":"Evil or devilish",

u"3:\)":"Evil or devilish",

u">;\)":"Evil or devilish",

u"\|;‑\)":"Cool",

u"\|‑O":"Bored",

u":‑J":"Tongue-in-cheek",

u"#‑\)":"Party all night",

u"%‑\)":"Drunk or confused",

u"%\)":"Drunk or confused",

u":-###..":"Being sick",

u":###..":"Being sick",

u"<:‑\|":"Dump",

u"\(>_<\)":"Troubled",

u"\(>_<\)>":"Troubled",

u"\(';'\)":"Baby",

u"\(\^\^>``":"Nervous or Embarrassed or Troubled or Shy or Sweat drop",

u"\(\^_\^;\)":"Nervous or Embarrassed or Troubled or Shy or Sweat drop",

u"\(-_-;\)":"Nervous or Embarrassed or Troubled or Shy or Sweat drop",

u"\(~_~;\) \(・\.・;\)":"Nervous or Embarrassed or Troubled or Shy or Sweat drop",

u"\(-_-\)zzz":"Sleeping",

u"\(\^_-\)":"Wink",

u"\(\(\+_\+\)\)":"Confused",

u"\(\+o\+\)":"Confused",

u"\(o\|o\)":"Ultraman",

u"\^_\^":"Joyful",

u"\(\^_\^\)/":"Joyful",

u"\(\^O\^\)／":"Joyful",

u"\(\^o\^\)／":"Joyful",

u"\(__\)":"Kowtow as a sign of respect, or dogeza for apology",

u"_\(\._\.\)_":"Kowtow as a sign of respect, or dogeza for apology",

u"<\(_ _\)>":"Kowtow as a sign of respect, or dogeza for apology",

u"<m\(__\)m>":"Kowtow as a sign of respect, or dogeza for apology",

u"m\(__\)m":"Kowtow as a sign of respect, or dogeza for apology",

u"m\(_ _\)m":"Kowtow as a sign of respect, or dogeza for apology",

u"\('_'\)":"Sad or Crying",

u"\(/_;\)":"Sad or Crying",

u"\(T_T\) \(;_;\)":"Sad or Crying",

u"\(;_;":"Sad of Crying",

u"\(;_:\)":"Sad or Crying",

u"\(;O;\)":"Sad or Crying",

u"\(:_;\)":"Sad or Crying",

u"\(ToT\)":"Sad or Crying",

u";_;":"Sad or Crying",

u";-;":"Sad or Crying",

u";n;":"Sad or Crying",

u";;":"Sad or Crying",

u"Q\.Q":"Sad or Crying",

u"T\.T":"Sad or Crying",

u"QQ":"Sad or Crying",

u"Q_Q":"Sad or Crying",

u"\(-\.-\)":"Shame",

u"\(-_-\)":"Shame",

u"\(一一\)":"Shame",

u"\(；一_一\)":"Shame",

u"\(=_=\)":"Tired",

u"\(=\^\·\^=\)":"cat",

u"\(=\^\·\·\^=\)":"cat",

u"=_\^= ":"cat",

u"\(\.\.\)":"Looking down",

u"\(\._\.\)":"Looking down",

u"\^m\^":"Giggling with hand covering mouth",

u"\(\・\・?":"Confusion",

u"\(?_?\)":"Confusion",

u">\^_\^<":"Normal Laugh",

u"<\^!\^>":"Normal Laugh",

u"\^/\^":"Normal Laugh",

u"\（\*\^_\^\*）" :"Normal Laugh",

u"\(\^<\^\) \(\^\.\^\)":"Normal Laugh",

u"\(^\^\)":"Normal Laugh",

u"\(\^\.\^\)":"Normal Laugh",

u"\(\^_\^\.\)":"Normal Laugh",

u"\(\^_\^\)":"Normal Laugh",

u"\(\^\^\)":"Normal Laugh",

u"\(\^J\^\)":"Normal Laugh",

u"\(\*\^\.\^\*\)":"Normal Laugh",

u"\(\^—\^\）":"Normal Laugh",

u"\(#\^\.\^#\)":"Normal Laugh",

u"\（\^—\^\）":"Waving",

u"\(;_;\)/~~~":"Waving",

u"\(\^\.\^\)/~~~":"Waving",

u"\(-_-\)/~~~ \($\·\·\)/~~~":"Waving",

u"\(T_T\)/~~~":"Waving",

u"\(ToT\)/~~~":"Waving",

u"\(\*\^0\^\*\)":"Excited",

u"\(\*_\*\)":"Amazed",

u"\(\*_\*;":"Amazed",

u"\(\+_\+\) \(@_@\)":"Amazed",

u"\(\*\^\^\)v":"Laughing,Cheerful",

u"\(\^_\^\)v":"Laughing,Cheerful",

u"\(\(d[-_-]b\)\)":"Headphones,Listening to music",

u'\(-"-\)':"Worried",

u"\(ーー;\)":"Worried",

u"\(\^0_0\^\)":"Eyeglasses",

u"\(\＾ｖ\＾\)":"Happy",

u"\(\＾ｕ\＾\)":"Happy",

u"\(\^\)o\(\^\)":"Happy",

u"\(\^O\^\)":"Happy",

u"\(\^o\^\)":"Happy",

u"\)\^o\^\(":"Happy",

u":O o_O":"Surprised",

u"o_0":"Surprised",

u"o\.O":"Surpised",

u"\(o\.o\)":"Surprised",

u"oO":"Surprised",

u"\(\*￣m￣\)":"Dissatisfied",

u"\(‘A`\)":"Snubbed or Deflated"

AFAIK=As Far As I Know

AFK=Away From Keyboard

ASAP=As Soon As Possible

ATK=At The Keyboard

ATM=At The Moment

A3=Anytime, Anywhere, Anyplace

BAK=Back At Keyboard

BBL=Be Back Later

BBS=Be Back Soon

BFN=Bye For Now

B4N=Bye For Now

BRB=Be Right Back

BRT=Be Right There

BTW=By The Way

B4=Before

B4N=Bye For Now

CU=See You

CUL8R=See You Later

CYA=See You

FAQ=Frequently Asked Questions

FC=Fingers Crossed

FWIW=For What It's Worth

FYI=For Your Information

GAL=Get A Life

GG=Good Game

GN=Good Night

GMTA=Great Minds Think Alike

GR8=Great!

G9=Genius

IC=I See

ICQ=I Seek you (also a chat program)

ILU=ILU: I Love You

IMHO=In My Honest/Humble Opinion

IMO=In My Opinion

IOW=In Other Words

IRL=In Real Life

KISS=Keep It Simple, Stupid

LDR=Long Distance Relationship

LMAO=Laugh My A.. Off

LOL=Laughing Out Loud

LTNS=Long Time No See

L8R=Later

MTE=My Thoughts Exactly

M8=Mate

NRN=No Reply Necessary

OIC=Oh I See

PITA=Pain In The A..

PRT=Party

PRW=Parents Are Watching

QPSA? Que Pasa?

ROFL=Rolling On The Floor Laughing

ROFLOL=Rolling On The Floor Laughing Out Loud

ROTFLMAO=Rolling On The Floor Laughing My A.. Off

SK8=Skate

STATS=Your sex and age

ASL=Age, Sex, Location

THX=Thank You

TTFN=Ta-Ta For Now!

TTYL=Talk To You Later

U=You

SMH = Shake My Head

U2=You Too

U4E=Yours For Ever

WB=Welcome Back

WTF=What The F...

WTG=Way To Go!

WUF=Where Are You From?

W8=Wait...

7K=Sick:-D Laugher

'''

lowercase: takes in a string and lowercases it

Parameters

----------

doc: Python string

Returns

-------

doc: Python strin converted to lowercase

'''

doc = doc.lower()

'''

remove_punctuation: takes in a string and

revoes the punctuation via RegEx

Parameters

----------

doc: Python string

Returns

-------

doc: Python string with punctuation removed

'''

'''

remove_url: takes in a string and removes

html values '<.X?>' using RegEx

Parameters

----------

doc: Python string

Returns

-------

doc: Python string with removed html text

'''

html_pattern = re.compile('<.*?>')

'''

remove_url: takes in a string and removes

url values 'https://, etc.' using RegEx

Parameters

----------

doc: Python string

Returns

-------

doc: Python string with removed url text

'''

url = re.compile(r"https?://\S+|www\.\S+")

'''

convert_emoticons: takes in a string and converts

emoticons ':), :(, :-), etc.' with word values.

Parameters

----------

doc: Python string

Returns

-------

doc: Python string with converted emoticons

'''

for i in EMOTICONS:

doc = re.sub(u'('+i+')', "_".join(EMOTICONS[i].replace(",","").split()), doc)

'''

convert_emojis: takes in a string and converts

emojis '😀, 😂, etc.' with word values.

Parameters

----------

doc: Python string

Returns

-------

doc: Python string with converted emojis

'''

doc = emoji.demojize(doc)

doc = re.sub('[:]', '', doc)

'''

convert_text_speech: takes in a string and converts any

abbreviated text speech, 'ttyl, imo, wtf, etc.' and converts

it into its component words

Parameters

----------

doc: Python string

Returns

-------

doc: Python string with converted text speech

'''

text = []

#removes punctuation because that throws off the conversion

doc = doc.translate(str.maketrans('', '', string.punctuation))

for i in doc.split():

if i.upper() in chat_list:

text.append(chat_dict[i.upper()])

else:

text.append(i)

'''

remove_stopwords: takes in a list of strings removes

those strings that contain stopwords imported from the

nlkt.corpus library

Parameters

----------

doc: Python string

Returns

-------

doc: Python list with stop words removed

'''

doc = [word for word in doc if word not in sw]

'''

set_english: takes in a string of a text column

outputs the predicted languge of that string to a list

which is used to mask a pandas DataFrame.

Parameters

----------

doc: Python string

Returns

-------

doc: Python list with predicted language

'''

identifier = LanguageIdentifier.from_modelstring(model, norm_probs=True)

lst =[]

for i in df[text_column].values:

lang, score = identifier.classify(i)

lst.append(lang)

'''

lemmatize_words: takes in a list of strings and

outputs the lemmatized version of each word,

lemmatizes based on part of speech.

Parameters

----------

doc: Python string

Returns

-------

doc: Python list with lemmatized words

'''

pos_tagged_text = nltk.pos_tag(doc)

doc = [lemmatizer.lemmatize(word, wordnet_map.get(pos[0], wordnet.NOUN)) for word, pos in pos_tagged_text]

'''

remove_numbers: removes all numbers from a string

Parameters

----------

doc: Python string

Returns

-------

doc: Python list with numbers removed

'''

doc = re.sub(r'\w*\d\w*', '', doc).strip()

'''

remove_non_english_characters: takes in a list of strings and

removes all non-english characters, including ASCII.

Parameters

----------

doc: Python string

Returns

-------

doc: Python list with non-English characters removed

'''

doc = unicodedata.normalize('NFKD', doc).encode('ASCII', 'ignore').decode('utf8')

'''

preprocessing: takes in a string and applies

text preprocessing to that string.

(For each function definition, refer to above.)

Parameters

----------

doc: Python string

Returns

-------

doc: Python list with preprocessed text

'''

doc = lowercase(doc)

doc = remove_punctuation(doc)

doc = convert_emoticons(doc)

doc = convert_emojis(doc)

doc = remove_url(doc)

doc = remove_html(doc)

doc = convert_text_speech(doc)

doc = remove_numbers(doc)

doc = remove_non_english_characters(doc)

doc = word_tokenize(doc)

doc = remove_stopwords(doc)

doc = lemmatize_words(doc)

return doc