schema = StructType([StructField("asin", StringType(), True), StructField("price", FloatType(), True)])) reviews = reviews.join(prices, ["asin"], how="leftsemi") # # Use nltk.word_tokenizer to tokenize words # @udf(ArrayType(StringType())) # def tokenize(string): # return word_tokenize(string) # reviews = reviews.withColumn("words", tokenize("reviewText")) reviews = RegexTokenizer(inputCol="reviewText", outputCol="words", pattern="\\W").transform(reviews) reviews = reviews.drop("reviewText") reviews = reviews.withColumn("num_words", size("words")) reviews = reviews.drop("words") reviews = reviews.groupBy("asin").agg(avg("num_words").alias("average_review_length")) reviews = reviews.drop("num_words") data = reviews.join(prices, ["asin"]) data = data.drop("asin") data = data.repartition(20) xy = data.rdd.map(lambda row: (row.average_review_length, row.price)) xy = xy.coalesce(8) x = xy.map(lambda v: v[0]) y = xy.map(lambda v: v[1]) n = x.count()
# @udf(ArrayType(StringType())) # def tokenize(string): # return word_tokenize(string) # df = df.withColumn("words", tokenize("reviewText")) df = RegexTokenizer(inputCol="reviewText", outputCol="words", pattern="\\W").transform(df) df = df.drop("reviewText") cv_model = CountVectorizer(inputCol="words", outputCol="tf").fit(df) vocabulary = cv_model.vocabulary df = cv_model.transform(df) df = df.drop("words") df.cache() df = IDF(inputCol="tf", outputCol="tfidf").fit(df).transform(df) df = df.drop("tf") df.unpersist() @udf(MapType(StringType(), FloatType())) def create_map(vector): zipped = zip(vector.indices, vector.values) return dict((vocabulary[int(x)], float(y)) for (x, y) in zipped) results = df.withColumn("tfidf", create_map("tfidf")) results.write.json("hdfs:/output/tfidf", mode="overwrite") spark.stop()