def __init__(self, sc, dataset_path):
"""Init the recommendation engine given a Spark context and a dataset path
"""
logger.info("Starting up the Recommendation Engine: ")
self.sc = sc
# Load ratings data for later use
logger.info("Loading Ratings data...")
ratings_file_path = os.path.join(dataset_path, 'ratings.csv')
ratings_raw_RDD = self.sc.textFile(ratings_file_path)
ratings_raw_data_header = ratings_raw_RDD.take(1)[0]
self.ratings_RDD = ratings_raw_RDD.filter(lambda line: line!=ratings_raw_data_header)\
.map(lambda line: line.split(",")).map(lambda tokens: (int(tokens[0]),int(tokens[1]),float(tokens[2]))).cache()
# Load movies data for later use
logger.info("Loading Movies data...")
movies_file_path = os.path.join(dataset_path, 'movies.csv')
movies_raw_RDD = self.sc.textFile(movies_file_path)
movies_raw_data_header = movies_raw_RDD.take(1)[0]
self.movies_RDD = movies_raw_RDD.filter(lambda line: line!=movies_raw_data_header)\
.map(lambda line: line.split(",")).map(lambda tokens: (int(tokens[0]),tokens[1],tokens[2])).cache()
self.movies_titles_RDD = self.movies_RDD.map(
lambda x: (int(x[0]), x[1])).cache()
# Pre-calculate movies ratings counts
self.__count_and_average_ratings()
self.redis_utils = RedisUtils("localhost", 6379)
# Train the model
self.rank = 8
self.seed = 5
self.iterations = 10
self.regularization_parameter = 0.1
self.__train_model()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Created on 2018/09/20
@filename: main.py
@author: yanghang
Description:
"""
import sys
import os
package_path = os.getcwd()+os.sep+"utils"
sys.path.append(package_path)
from redis_utils import RedisUtils
from config_utils import ConfigUtils
if __name__ == '__main__':
config = ConfigUtils('conf/config.ini')
REDIS_HOST = config.get_value('redis', 'host')
REDIS_PORT = config.get_value('redis', 'port')
MYSQL_HOST = config.get_value('mysql', 'host')
MYSQL_PORT = config.get_value('mysql', 'port')
MYSQL_USERNAME = config.get_value('mysql', 'username')
MYSQL_PASSWORD = config.get_value('mysql', 'password')
MYSQL_DATABASE = config.get_value('mysql', 'database')
redis = RedisUtils(REDIS_HOST,REDIS_PORT)
redis.set_str('hello','11111')
print(redis.get_str('hello'))
class RedisConfigurationReader(ConfigurationReader):
def __init__(self, config_file):
super().__init__(config_file)
def get_redis_settings(self):
try:
return self.configuration["redis_cache_settings"]
except KeyError:
raise Exception(
'The redis configurations are not configured, please configure in yml file'
)
r = RedisUtils()
config = RedisConfigurationReader('./conf/redis_settings.yml')
config.read_config_file()
r.strict_connect(**config.get_redis_settings())
print(r.__dict__)
app = Flask(__name__)
example_blueprint = Blueprint("example_blueprint", __name__)
app.config['SWAGGER'] = {
"swagger_version":
"2.0",
"title":
"Flasgger",
"headers": [
