def process_email(email_contents: str) -> List[int]:
"""Pre-process the body of an email and return a list of indices of the
words contained in the email.
:param email_contents: the body of an email
:return: a list of indices of the words contained in the email
"""
# Load the vocabulary.
vocabulary_dict = get_vocabulary_dict()
# Initialize the return value.
word_indices = []
# ========================== Preprocess Email ===========================
# Find the Headers ( \n\n and remove )
# Uncomment the following lines if you are working with raw emails with the
# full headers
# header_token = '\n\n'
# header_start = email_contents.find(header_token)
# email_contents = email_contents[header_start+len(header_token):]
# Convert email content to lower case.
email_contents = email_contents.lower()
# Strip all HTML
# Looks for any expression that starts with < and ends with > and replace
# and does not have any < or > in the tag it with a space
email_contents = re.sub('<[^<>]+>', ' ', email_contents)
# Handle numbers.
# Convert all sequences of digits (0-9) to a 'number' token.
email_contents = re.sub('[0-9]+', 'number', email_contents)
# Handle URLs.
# Convert all strings starting with http:// or https:// to a 'httpaddr' token.
email_contents = re.sub('(http://|https://)+\S*', 'httpaddr',
email_contents)
# Handle email addresses.
# Convert all strings with @ in the middle to a 'emailaddr' token.
email_contents = re.sub('[\S*]+(@)+\S*', 'emailaddr', email_contents)
# Handle $ sign
# Convert all sequences of $ signs to a 'dollar' token.
email_contents = re.sub('[$]', 'dollar', email_contents)
# ========================== Tokenize Email ===========================
# Output the email to screen as well
print('\n==== Processed Email ====\n\n')
# Process file
col = 0
# Tokenize and also get rid of any punctuation
tokens = re.split('[ @$/#.-:&*\+=\[\]?!\(\)\{\},'
'">_<;#\n\r]', email_contents)
for token in tokens:
# Remove any non alphanumeric characters
token = re.sub('[^a-zA-Z0-9]', '', token)
# Stem the word
token = PorterStemmer().stem(token.strip())
# Skip the word if it is too short
if len(token) < 1:
continue
# Look up the word in the dictionary and add to word_indices if
# found
for i, word in vocabulary_dict.items():
if token == word:
word_indices.append(i)
# Print to screen, ensuring that the output lines are not too long
if (col + len(token) + 1) > 78:
print('')
col = 0
print('{} '.format(token), end='', flush=True)
col = col + len(token) + 1
# Print footer
print('\n\n=========================\n')
return word_indices
print('\nEvaluating the trained Linear SVM on a test set ...\n')
# Predict the labelling.
y_pred = clf.predict(X_test)
# Compute the training accuracy.
acc_test = np.mean(y_pred == y_test)
print('Test Accuracy: {:.2f}%\n'.format(acc_test * 100))
# input('Program paused. Press enter to continue.\n')
weights = clf.coef_.reshape(-1)
idx = np.argsort(-weights)
vocabulary_dict = get_vocabulary_dict()
print('\nTop predictors of spam: \n')
for i in range(15):
print(' {word:<20}: {weight:10.6f}'.format(word=vocabulary_dict[idx[i]],
weight=weights[idx[i]]))
print('\n\n')
# input('\nProgram paused. Press enter to continue.\n')
filename = 'data/spamSample1.txt'
# Read and predict
file_contents = read_file(filename)
word_indices = process_email(file_contents)
x = email_features(word_indices)
def process_email(email_contents: str) -> List[int]:
"""Pre-process the body of an email and return a list of indices of the
words contained in the email.
:param email_contents: the body of an email
:return: a list of indices of the words contained in the email
"""
vocabulary_dict = get_vocabulary_dict()
# FIXME: Initialize the return value.
word_indices = list()
# ========================== Preprocess Email ===========================
# Find the Headers ( \n\n and remove )
# Uncomment the following lines if you are working with raw emails with the
# full headers
# header_token = '\n\n'
# header_start = email_contents.find(header_token)
# email_contents = email_contents[header_start+len(header_token):]
# FIXME: Convert email content to lower case.
email_contents = email_contents.lower()
# Strip all HTML
# Looks for any expression that starts with < and ends with > and replace
# and does not have any < or > in the tag it with a space
email_contents = re.sub('<[^<>]+>', ' ', email_contents)
# FIXME: Handle numbers.
# Convert all sequences of digits (0-9) to a 'number' token.
email_contents = re.sub('\d+', 'number', email_contents)
# FIXME: Handle URLs.
# Convert all strings starting with http:// or https:// to a 'httpaddr' token.
email_contents = re.sub('(http://|https://)[\w\.-]+', 'httpaddr', email_contents)
# FIXME: Handle email addresses.
# Convert all strings with @ in the middle to a 'emailaddr' token.
email_contents = re.sub('[\w\.-]+@[\w\.-]+', 'emailaddr', email_contents)
# FIXME: Handle $ sign
# Convert all sequences of $ signs to a 'dollar' token.
email_contents = re.sub('\$+', 'dollar', email_contents)
# ========================== Tokenize Email ===========================
# Output the email to screen as well
print('\n==== Processed Email ====\n\n')
# Process file
col = 0
# Tokenize and also get rid of any punctuation
tokens = re.split('[ @$/#.-:&*\+=\[\]?!\(\)\{\},''">_<;#\n\r]', email_contents)
for token in tokens:
# Remove any non alphanumeric characters
token = re.sub('[^a-zA-Z0-9]', '', token)
# Stem the word
token = PorterStemmer().stem(token.strip())
# Skip the word if it is too short
if len(token) < 1:
continue
# Look up the word in the dictionary and add to word_indices if
# found
# FIXME: ======================= YOUR CODE HERE ======================
# Instructions: Fill in this function to add the index of str to
# word_indices if it is in the vocabulary. At this point
# of the code, you have a stemmed word from the email in
# the variable str. You should look up str in the
# vocabulary list (vocabulary_dict). If a match exists, you
# should add the index of the word to the word_indices
# vector. Concretely, if str = 'action', then you should
# look up the vocabulary list to find where in vocabulary_dict
# 'action' appears. For example, if vocabulary_dict{18} =
# 'action', then, you should add 18 to the word_indices
# vector (e.g., word_indices = [word_indices ; 18]; ).
#
# Note: vocabulary_dict{idx} returns a the word with index idx in the
# vocabulary list.
#
# Note: You can use strcmp(str1, str2) to compare two strings (str1 and
# str2). It will return 1 only if the two strings are equivalent.
#
if token in vocabulary_dict.values():
index = get_key(token, vocabulary_dict)
if index == 0:
raise ValueError
else:
word_indices.append(index)
# ========================= END OF YOUR CODE ==========================
# Print to screen, ensuring that the output lines are not too long
if (col + len(token) + 1) > 78:
print('')
col = 0
print('{} '.format(token), end='', flush=True)
col = col + len(tokens) + 1
# Print footer
print('\n\n=========================\n')
return word_indices