def count_offensive_words(docs, pickle_path_pattern=None):
"""Count the number of offensive words in the documents.
This function counts the number of occurrences of all the expressions inside the Flame Dictionary.
If the pickled results of the function (corresponding to the same input) already exists, the function is
bypassed. If not, after the function is done, the results are stored as pickles.
Args:
- docs: (list: strings) List of documents. Each row represents an author and contains one string.
- pickle_path_pattern: (string) The path pattern for the pickle. This needs to include “<HASH>”, which will be
replaced with the hash of the input of the function. Refer to the docstring of the *generate_pickle_path* function.
Returns:
- counts_of_expressions_dict: A Python dictionary
• Keys: (int) Flame level
• Values: (NumPy array) Counts of occurrences of expressions in that Flame level. Each row
represents an author, and each column represents an expression in the Flame level of the key.
Note: List of expressions can be accessed by calling *ProcessDataFiles.load_flame_dictionary*.
"""
pickle_path = generate_pickle_path(docs, pickle_path_pattern)
# Bypass: If the pickled results already exist, load (unpickle) and return them and skip the rest of the function
if (pickle_path is not None) and os.path.isfile(pickle_path):
with open(pickle_path, 'rb') as pickle_input_file:
unpickled_object = pickle.load(pickle_input_file)
print(
'Function bypassed: The counts of offensive words was loaded from pickle "%s" instead.',
pickle_path)
return unpickled_object
# Load the Flame Dictionary
# %% TODO: Prevent loading the dictionary every time...
flame_dictionary, flame_expressions_dict = process_data_files.load_flame_dictionary(
)
'''
↳
*flame_dictionary*
Keys: (string) Expression
Values: (int) Flame level
*flame_expressions_dict*
Keys: (int) Flame level
Values: (list: strings) Expressions
'''
# keys_dict_view = flame_dictionary.keys()
# expressions = list(keys_dict_view)
# Preprocess the merged tweets of authors
preprocessed_docs = [] # Create an empty list
for author_index, doc in enumerate(docs):
preprocessed_docs.append(preprocess_tweet(doc))
# Log after preprocessing the merged tweets of every 200 authors
if author_index % 200 == 0:
logger.debug(
"@ %.2f seconds, progress: Preprocessed the tweets of author_index = %d",
time.process_time(), author_index)
print(
"@ %.2f seconds: Finished preprocessing the tweets in *count_offensive_words()*",
time.process_time())
# Create a dictionary of five NumPy arrays full of zeros
counts_of_expressions_dict = {} # Create an empty dictionary
for flame_index in range(1, 6):
counts_of_expressions_dict[flame_index] = np.zeros(
(len(preprocessed_docs), len(flame_expressions_dict[flame_index])),
dtype=int)
# Compile regex patterns into regex objects for all expressions, and store them in five separate lists, based on
# Flame level (similar to *flame_expressions_dict*).
'''
- Most regex operations are available as module-level functions as well as methods on compiled
regular expressions. The functions are shortcuts that don’t require you to compile a regex object first,
but miss some fine-tuning parameters.
- Compiling a regex pattern and storing the resulting regex object for reuse is more efficient when the
expression will be used several times in a single program. Even though the most recent patterns passed to
re.compile() and the module-level matching functions are cached, the size of this cache is limited.
More info: https://docs.python.org/3/library/re.html#re.compile
Here, we are dealing with 2,600+ expressions, so the built-in cache cannot help. Storing the regex objects,
decreased the processing time of each Author from 1.6 seconds to 0.7 seconds (on my machine).
'''
'''
- In Python code, Regular Expressions will often be written using the raw string notation (r"text").
Without it, every backslash in a regular expression would have to be prefixed with another one to escape it.
- The shorthand \b matches a word boundary, without consuming any characters. Word boundary characters
include space, . ! " ' - * and much more.
- Some examples of matches of the /\bWORD\b/ pattern: WORD's, prefix-WORD, WORD-suffix, "WORD".
%% TODO: To increase the performance of regex:
1. You can combine the patterns using | for all expressions of the same level of Flame.
https://stackoverflow.com/questions/1782586/speed-of-many-regular-expressions-in-python#comment1669596_1782712
2. You can first use str.find to find potential matches, and then check those matches with regex.
'''
regex_objects_dict = {
1: [],
2: [],
3: [],
4: [],
5: []
} # Create a dictionary of 5 empty lists
for flame_index in range(1, 6):
for expression in flame_expressions_dict[flame_index]:
regex_pattern = r'\b' + expression + r'\b'
regex_object = re.compile(regex_pattern, re.IGNORECASE)
regex_objects_dict[flame_index].append(regex_object)
print(
"@ %.2f seconds: Finished compiling the regex patterns into regex objects.",
time.process_time())
# Count the matches of each expression for each author
for author_index, merged_tweets_of_author in enumerate(preprocessed_docs):
for flame_index in range(1, 6):
for expression_index in range(
len(flame_expressions_dict[flame_index])):
# ↳ Note: We are assuming that the lists inside *flame_expressions_dict* have not been manipulated since
# the lists inside *regex_objects_dict* were created.
list_of_matches = regex_objects_dict[flame_index][
expression_index].findall(merged_tweets_of_author)
count = len(list_of_matches)
# count = merged_tweets_of_author.count(expression)
counts_of_expressions_dict[flame_index][
author_index, expression_index] = count
# Log after counting the offensive words for every 100 authors
if author_index % 100 == 0:
logger.debug(
"@ %.2f seconds, progress: Counted (regex) the offensive words for author_index = %d",
time.process_time(), author_index)
print(
"@ %.2f seconds: Finished counting the occurrences of offensive words",
time.process_time())
# Pickle the output variable
if pickle_path is not None:
# Create the directory if it does not exist.
os.makedirs(os.path.dirname(pickle_path), exist_ok=True)
# Pickle
with open(pickle_path, 'wb') as pickle_output_file:
pickle.dump(counts_of_expressions_dict, pickle_output_file)
print('The counts of offensive words was pickled to: "%s"',
pickle_path)
return counts_of_expressions_dict
def extract_features_offensive_words(docs_train, docs_test):
"""Extract offensive words features
This function performs the following tasks for the training and test datasets:
1. Gets the counts of offensive words from the *count_offensive_words()* function.
2. Concatenates the count arrays for the desired Flame levels into X_train and X_test.
3. Transforms the count matrix (NumPy array) to a normalized TF or TF-IDF representation.
4. Performs dimensionality reduction on the normalized matrix using truncated SVD (aka LSA).
Moreover, the function collects and returns the feature names (offensive expressions) for the desired Flame
levels in the following format: “expression (flame level)”
Important constants:
DESIRED_FLAME_LEVELS: (Tuple: Ints) Desired flame levels. You can select any of the levels: 1, 2, 3, 4, and 5.
"""
# Count the number of occurrences of all the offensive expressions in the training set and test set
counts_of_offensive_words_dict_train = count_offensive_words(
docs_train,
"pickles/counts_of_offensive_words_dict_train, <HASH>.pickle")
counts_of_offensive_words_dict_test = count_offensive_words(
docs_test,
"pickles/counts_of_offensive_words_dict_test, <HASH>.pickle")
# Load the Flame Dictionary (to produce the list of feature names)
flame_dictionary, flame_expressions_dict = process_data_files.load_flame_dictionary(
)
'''
↳
*flame_dictionary*
Keys: (string) Expression
Values: (int) Flame level
*flame_expressions_dict*
Keys: (int) Flame level
Values: (list: strings) Expressions
'''
# Log the min, max, and shape of the offensive words count arrays (just to make sure the pickles were loaded
# correctly.
for flame_index in range(1, 6):
array = counts_of_offensive_words_dict_train[flame_index]
logger.debug("Flame level %d: min = %d | max = %-3d | shape = %s",
flame_index, array.min(), array.max(), array.shape)
for flame_index in range(1, 6):
array = counts_of_offensive_words_dict_test[flame_index]
logger.debug("Flame level %d: min = %d | max = %-3d | shape = %s",
flame_index, array.min(), array.max(), array.shape)
# Create empty lists
arrays_list_train = []
arrays_list_test = []
feature_names_offensive_words = []
# Concatenate the counts NumPy arrays and the feature names for the desired Flame levels
DESIRED_FLAME_LEVELS = (1, 2, 3, 4, 5)
for flame_index in DESIRED_FLAME_LEVELS:
arrays_list_train.append(
counts_of_offensive_words_dict_train[flame_index])
arrays_list_test.append(
counts_of_offensive_words_dict_test[flame_index])
# Add the expressions to the list of feature names in the form: “expression (flame level)”
for expression in flame_expressions_dict[flame_index]:
feature_names_offensive_words.append("{} ({})".format(
expression, flame_index))
X_train_offensive_words_counts = np.concatenate(arrays_list_train, axis=1)
X_test_offensive_words_counts = np.concatenate(arrays_list_test, axis=1)
# • Transform the count matrix (NumPy array) to a normalized TF or TF-IDF representation
# Build a TF-IDF transformer object
tfidf_transformer = TfidfTransformer(norm='l2',
use_idf=False,
sublinear_tf=False)
# ↳ With these parameters, the transformer does not make any changes: norm=None, use_idf=False, sublinear_tf=False
'''
↳ With normalization, each row (= author) is normalized to have a sum of absolute values / squares equal to 1.
L^1-norm: Sum of the absolute value of the numbers (here, TF or TF-IDF of the offensive expressions)
L^2-norm: Sum of the square of the numbers ”...
More info: http://www.chioka.in/differences-between-l1-and-l2-as-loss-function-and-regularization/
'''
# Fit and transform
X_train_offensive_words_tfidf = tfidf_transformer.fit_transform(
X_train_offensive_words_counts)
X_test_offensive_words_tfidf = tfidf_transformer.transform(
X_test_offensive_words_counts)
# • Dimensionality reduction using truncated SVD (aka LSA)
# Build a truncated SVD (LSA) transformer object
svd_offensive_words = TruncatedSVD(n_components=10, random_state=42)
# Fit the LSI model and perform dimensionality reduction
x_train_offensive_words_tfidf_reduced = svd_offensive_words.fit_transform(
X_train_offensive_words_tfidf)
print(
"@ %.2f seconds: Finished dimensionality reduction (LSA) in *extract_features_offensive_words()* on "
"the training dataset", time.process_time())
x_test_offensive_words_tfidf_reduced = svd_offensive_words.transform(
X_test_offensive_words_tfidf)
print(
"@ %.2f seconds: Finished dimensionality reduction (LSA) in *extract_features_offensive_words()* on "
"the test dataset", time.process_time())
return x_train_offensive_words_tfidf_reduced, x_test_offensive_words_tfidf_reduced, feature_names_offensive_words