def sub():
redis = RedisUtils().get_connection()
p = redis.pubsub()
p.subscribe("mychannel1", "mychannel2")
for message in p.listen():
print message
RedisUtils().release_connection(redis)
def sub1(self):
redis = RedisUtils().get_connection()
p = redis.pubsub()
p.subscribe("mychannel1", "mychannel2")
for message in p.listen():
print("Get message %s " % message['data'])
RedisUtils().release_connection(redis)
def test_pipeline():
redis = RedisUtils().get_connection()
pipe = redis.pipeline()
assert pipe.set("mykey1", "myvalue1")
assert pipe.set("mykey2", "myvalue2")
pipe.execute()
assert redis.get("mykey1") == "myvalue1"
assert redis.get("mykey2") == "myvalue2"
RedisUtils().release_connection(redis)
def run(self):
redis = RedisUtils().get_connection()
for i in range(1, 100):
print("publish mychannel1 %s " % "mychannel1_" + str(i))
redis.publish("mychannel1", "mychannel1_" + str(i))
print("publish mychannel2 %s " % "mychannel2_" + str(i))
redis.publish("mychannel2", "mychannel2_" + str(i))
time.sleep(2)
RedisUtils().release_connection(redis)
def __init__(self, sc, dataset_path):
"""Init the recommendation engine given a Spark context and a dataset path
"""
logger.info("Starting up the Recommendation Engine: ")
self.sc = sc
# Load ratings data for later use
logger.info("Loading Ratings data...")
ratings_file_path = os.path.join(dataset_path, 'ratings.csv')
ratings_raw_RDD = self.sc.textFile(ratings_file_path)
ratings_raw_data_header = ratings_raw_RDD.take(1)[0]
self.ratings_RDD = ratings_raw_RDD.filter(lambda line: line!=ratings_raw_data_header)\
.map(lambda line: line.split(",")).map(lambda tokens: (int(tokens[0]),int(tokens[1]),float(tokens[2]))).cache()
# Load movies data for later use
logger.info("Loading Movies data...")
movies_file_path = os.path.join(dataset_path, 'movies.csv')
movies_raw_RDD = self.sc.textFile(movies_file_path)
movies_raw_data_header = movies_raw_RDD.take(1)[0]
self.movies_RDD = movies_raw_RDD.filter(lambda line: line!=movies_raw_data_header)\
.map(lambda line: line.split(",")).map(lambda tokens: (int(tokens[0]),tokens[1],tokens[2])).cache()
self.movies_titles_RDD = self.movies_RDD.map(
lambda x: (int(x[0]), x[1])).cache()
# Pre-calculate movies ratings counts
self.__count_and_average_ratings()
self.redis_utils = RedisUtils("localhost", 6379)
# Train the model
self.rank = 8
self.seed = 5
self.iterations = 10
self.regularization_parameter = 0.1
self.__train_model()
def sub2(self):
redis = RedisUtils().get_connection()
p = redis.pubsub()
def my_handler(message):
print("Get message %s " % message['data'])
p.subscribe(**{'mychannel1': my_handler})
p.subscribe(**{'mychannel2': my_handler})
while True:
message = p.get_message()
if message:
time.sleep(0.001)
RedisUtils().release_connection(redis)
def test_transaction1():
redis = RedisUtils().get_connection()
pipe = redis.pipeline()
try:
pipe.watch("mykey1", "mykey2")
pipe.multi()
assert pipe.set("mykey1", "myvalue1")
assert pipe.set("mykey2", "myvalue2")
time.sleep(10)
pipe.execute()
print redis.get("mykey1")
print redis.get("mykey2")
except WatchError as e:
print e
finally:
pipe.reset()
RedisUtils().release_connection(redis)
def test_getset():
redis = RedisUtils().get_connection()
assert redis.set("mykey", "myvalue")
assert redis.get("mykey") == "myvalue"
assert redis.get("nokey") is None
RedisUtils().release_connection(redis)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Created on 2018/09/20
@filename: main.py
@author: yanghang
Description:
"""
import sys
import os
package_path = os.getcwd()+os.sep+"utils"
sys.path.append(package_path)
from redis_utils import RedisUtils
from config_utils import ConfigUtils
if __name__ == '__main__':
config = ConfigUtils('conf/config.ini')
REDIS_HOST = config.get_value('redis', 'host')
REDIS_PORT = config.get_value('redis', 'port')
MYSQL_HOST = config.get_value('mysql', 'host')
MYSQL_PORT = config.get_value('mysql', 'port')
MYSQL_USERNAME = config.get_value('mysql', 'username')
MYSQL_PASSWORD = config.get_value('mysql', 'password')
MYSQL_DATABASE = config.get_value('mysql', 'database')
redis = RedisUtils(REDIS_HOST,REDIS_PORT)
redis.set_str('hello','11111')
print(redis.get_str('hello'))
class RedisConfigurationReader(ConfigurationReader):
def __init__(self, config_file):
super().__init__(config_file)
def get_redis_settings(self):
try:
return self.configuration["redis_cache_settings"]
except KeyError:
raise Exception(
'The redis configurations are not configured, please configure in yml file'
)
r = RedisUtils()
config = RedisConfigurationReader('./conf/redis_settings.yml')
config.read_config_file()
r.strict_connect(**config.get_redis_settings())
print(r.__dict__)
app = Flask(__name__)
example_blueprint = Blueprint("example_blueprint", __name__)
app.config['SWAGGER'] = {
"swagger_version":
"2.0",
"title":
"Flasgger",
"headers": [
def pub():
redis = RedisUtils().get_connection()
redis.publish("mychannel1", "mychannel1_" + str(datetime.now()))
redis.publish("mychannel2", "mychannel2_" + str(datetime.now()))
RedisUtils().release_connection(redis)