from __future__ import print_function # this line must appear as the first line!

import os

import sys

import json

import random

from pyspark import SparkContext

from pyspark.streaming import StreamingContext

from pyspark.streaming.kafka import KafkaUtils

from mf_based_recommender import MFBasedRecommender

from pyspark.mllib.recommendation import MatrixFactorizationModel

#from org.apache.spark import *

from mf_based_recommendation_trainer import MFBasedRecommendationTrainer

def load_model(spark_context):

return model

def recommend_business_for_user(model, user_id, topk=100):

return model.recommendProducts(user_id, topk)

if __name__ == "__main__":

sc = SparkContext(appName="UserRequestsStreamingHandler")

#sc.setLogLevel("ERROR")

ssc = StreamingContext(sc, 1.5) # 1 second window

# note that kafka zookeeper default port is 2181 not 9092!

skQuorum = "localhost:2181"

topic = ["request"]

kafkaParams = {"metadata.broker.list":"localhost:9092"}

#kafkaStream = KafkaUtils.createDirectStream(ssc, topic, kafkaParams)

kafkaStream = KafkaUtils.createStream(ssc, 'localhost:2181', 'request-group', {'request': 5})

# init recommender

#recommender = MFBasedRecommender()

model = load_model(sc)

# should broadcast the model. Otherwise, this error will be shown:

# "It appears that you are attempting to reference SparkContext from a broadcast "

# Exception: It appears that you are attempting to reference SparkContext from a

# broadcast variable, action, or transformation. SparkContext can only be used on

# the driver, not in code that it run on workers. For more information, see SPARK-5063.

# http://stackoverflow.com/questions/31396323/spark-error-it-appears-that-you-are-attempting-to-reference-sparkcontext-from-a

# handle kafka streams

lines = kafkaStream.map(lambda x: x[1])

def get_user_request(line):

return user_id, city_name, keywords

def simulate_recommendation(request):

return yelper_logo + prefix + msg + suffix

# calculate recommended businesses

recoms = lines.map(get_user_request).map(simulate_recommendation)

recoms.pprint()

ssc.start()

ssc.awaitTermination()