#A tokenizer that converts the input string to lowercase and then splits it by white spaces.
words = Tokenizer(inputCol="message", outputCol="words").transform(df)
words.show(truncate=False)
# 正規表達法資料清洗及斷詞
# A regex based tokenizer that extracts tokens either by using the provided regex pattern (in Java dialect) to split the text (default) or repeatedly matching the regex (if gaps is false). Optional parameters also allow filtering tokens using a minimal length. It returns an array of strings that can be empty.
words = RegexTokenizer(inputCol="message",
outputCol="words",
pattern="\\W+").transform(df)
words.show(truncate=False)
# StopWordsRemover is feature transformer that filters out stop words from input.
stop_words_removed = StopWordsRemover(
inputCol="words", outputCol="stop_words_removed").transform(words)
stop_words_removed.show(truncate=False)
# 變成n字一組
# NGram is a feature transformer that converts the input array of strings into an array of n-grams. Null values in the input array are ignored. It returns an array of n-grams where each n-gram is represented by a space-separated string of words. When the input is empty, an empty array is returned. When the input array length is less than n (number of elements per n-gram), no n-grams are returned.
ngram_df = NGram(n=2, inputCol="words",
outputCol="ngrams").transform(words)
ngram_df.show(truncate=False)
ngram_df.select("ngrams").show(truncate=False)
# TF-IDF is a numerical statistic that is intended to reflect how important a word is to a document in a collection or corpus.[1] It is often used as a weighting factor in searches of information retrieval, text mining, and user modeling.
df = words.select("words")
df.show(truncate=False)
# Hashing TF is TF with hashing enabled to allow the feature vector to be a set value
df_tf = HashingTF(
