def handler(event, context):
"""
This function puts the user recommendations into Redis
after fetching them from S3
"""
try:
redis_host_endpoint = os.environ["REDIS_HOST_ENDPOINT"]
s3_bucket_name = os.environ["S3_BUCKET"].split(":::")[1]
result = boto3.client("s3").get_object(Bucket=s3_bucket_name,
Key="batchpredictions.json")
user_predictions = json.loads(result["Body"].read().decode("utf-8"))
users = user_predictions["data"]
c = connect(redis_host_endpoint)
r = rediscluster.RedisCluster(connection_pool=c)
for user in users:
dict = {}
movies = user["movieId"]
count = 1
for movie in movies:
dict[count] = movie
count += 1
r.hmset(user["userId"], dict)
return f"Successfully inserted the user recommendations"
except Exception as e:
return f"Failed to insert the user recommendations due to {e}"
def setUp(self):
pymemcache.client.Client(('localhost', 11211)).flush_all()
redis.Redis().flushall()
redis.sentinel.Sentinel([
("localhost", 26379)
]).master_for("localhost-redis-sentinel").flushall()
rediscluster.RedisCluster("localhost", 7000).flushall()
def main():
if not (len(sys.argv) == 3 or len(sys.argv) == 4):
print("Usage: host port password")
exit(1)
host = sys.argv[1]
port = sys.argv[2]
password = None
if len(sys.argv) == 4:
password = sys.argv[3]
# Add more nodes here to the list
startup_nodes = [{"host": host, "port": port}]
rc = rediscluster.RedisCluster(
startup_nodes=startup_nodes,
decode_responses=True,
password=password,
ssl=True,
ssl_keyfile="../../../testscripts/tls/db_key.pem",
ssl_certfile="../../../testscripts/tls/db_cert.pem",
ssl_cert_reqs="required",
ssl_ca_certs="../../../testscripts/tls/ca_cert.pem",
# ssl_check_hostname=False, This option is not supported yet
)
print("Set: {}".format(rc.set("foo", "bar")))
print("Get: {}".format(rc.get("foo")))
def setUp(self):
pymemcache.client.Client(('localhost', 22122)).flush_all()
redis.from_url("redis://*****:*****@localhost:7389").flushall()
redis.sentinel.Sentinel([
("localhost", 26379)
]).master_for("localhost-redis-sentinel").flushall()
rediscluster.RedisCluster("localhost", 7000).flushall()
def _get_test_client(self):
startup_nodes = [{
'host': self.TEST_HOST,
'port': int(port)
} for port in self.TEST_PORTS.split(',')]
if REDISCLUSTER_VERSION >= (2, 0, 0):
return rediscluster.RedisCluster(startup_nodes=startup_nodes)
else:
return rediscluster.StrictRedisCluster(startup_nodes=startup_nodes)
def get_redis_connection(cluster_mode=False):
if cluster_mode:
startup_nodes = [{"host": cnt.REDIS_SERVER, "port": str(cnt.REDIS_PORT)}]
r = rediscluster.RedisCluster(startup_nodes=startup_nodes, decode_responses=True,
skip_full_coverage_check=True)
else:
r = redis.StrictRedis(host=cnt.REDIS_SERVER, port=cnt.REDIS_PORT, db=0, decode_responses=True)
return r
def setUp(self):
pymemcache.client.Client(('localhost', 11211)).flush_all()
redis.Redis().flushall()
redis.sentinel.Sentinel([
("localhost", 26379)
]).master_for("localhost-redis-sentinel").flushall()
rediscluster.RedisCluster("localhost", 7000).flushall()
tb = testbed.Testbed()
tb.activate()
tb.init_memcache_stub()
def getClusterConnection(self):
if self.useTLS:
# workaround for error on
# got an unexpected keyword argument 'ssl'
# we enforce the connection_class instead of setting ssl=True
pool = ClusterConnectionPool(
startup_nodes=self.getMasterNodesList(),
connection_class=SSLClusterConnection,
ssl_cert_reqs=None,
ssl_keyfile=self.shards[0].getTLSKeyFile(),
ssl_certfile=self.shards[0].getTLSCertFile(),
ssl_ca_certs=self.shards[0].getTLSCACertFile(),
)
if pool.connection_kwargs:
pool.connection_kwargs.pop('ssl', None)
return rediscluster.RedisCluster(
startup_nodes=self.getMasterNodesList(), connection_pool=pool)
else:
return rediscluster.RedisCluster(
startup_nodes=self.getMasterNodesList(), decode_responses=True)
def resolve_destination(dststring):
conn = resolve_host(dststring)
if not conn.info('cluster').get('cluster_enabled', None):
return conn
if not rediscluster:
raise RuntimeError(
'cluster destination specified and redis-py-cluster not installed')
host, port = dststring.split(':')
return rediscluster.RedisCluster(
startup_nodes=[{'host': host, 'port': port}], max_connections=1000)
def setUp(self):
pymemcache.client.Client(('localhost', 11211)).flush_all()
redis.from_url('unix:///var/tmp/limits.redis.sock').flushall()
redis.Redis().flushall()
redis.sentinel.Sentinel([
("localhost", 26379)
]).master_for("localhost-redis-sentinel").flushall()
rediscluster.RedisCluster("localhost", 7000).flushall()
if RUN_GAE:
from google.appengine.ext import testbed
tb = testbed.Testbed()
tb.activate()
tb.init_memcache_stub()
def init(config_file_path=None, **kwargs):
"""
Init the storage client.
Basically, we set the Redis client and connects it to the instance/cluster.
"""
if __debug__:
logger.debug(HEADER + "Initializing the storage client.")
global redis_connection
global hosts
# If config_file_path is None we will assume that we only have localhost
# as storage node
if config_file_path is None:
try:
import StringIO as sio
except ImportError:
from io import StringIO as sio
config_file_handler = sio.StringIO('localhost\n')
else:
config_file_handler = open(config_file_path)
# As accorded in the API standar, this file must contain all the hosts
# names with no port, one per line
hosts = [x.strip() for x in config_file_handler.readlines()]
config_file_handler.close()
# If we have more than one host then we will assume that our backend is a
# Redis cluster. If not, we will assume that we are dealing with a Redis
# standalone instance
if len(hosts) > 1:
# Given that cluster clients are capable to perform master
# slave hierarchy discovery, we will simply connect to the first
# node we got
redis_connection = \
rediscluster.RedisCluster(host=hosts[0], port=REDIS_PORT)
else:
# We are in standalone mode
redis_connection = \
redis.StrictRedis(host=hosts[0], port=REDIS_PORT)
# StrictRedis is not capable to know if we had success when connecting by
# simply calling the constructor. We need to perform an actual query to
# the backend
# If we had no success this first line should crash
redis_connection.set('PYCOMPSS_TEST', 'OK')
# Beware py2 vs py3 - b'string' works for both.
assert redis_connection.get('PYCOMPSS_TEST') == b'OK'
redis_connection.delete('PYCOMPSS_TEST')
if __debug__:
logger.debug(HEADER + "Initialization finished successfully.")
def main():
if not (len(sys.argv) == 3 or len(sys.argv) == 4):
print("Usage: host port password")
exit(1)
host = sys.argv[1]
port = sys.argv[2]
password = None
if len(sys.argv) == 4:
password = sys.argv[3]
# Add more nodes here to the list
startup_nodes = [{"host": host, "port": port}]
rc = rediscluster.RedisCluster(startup_nodes=startup_nodes,
decode_responses=True,
password=password)
print("Set: {}".format(rc.set("foo", "bar")))
print("Get: {}".format(rc.get("foo")))
def get_redis():
"""
Obtains a cluster aware redis connection.
:return ClusterConnectionPool: The redis connection pool
"""
global _redis_instance
# Due to a pending release from the `redis-py-cluster` project, pulling in
# connection code here. Once https://github.com/Grokzen/redis-py-cluster/pull/195
# is released, this can be reset to the original line (left for ease of transition).
# return rediscluster.RedisCluster.from_url(redis_config.URL, skip_full_coverage_check=True)
if _redis_instance is None:
connection_pool = ClusterConnectionPool.from_url(redis_config.URL, skip_full_coverage_check=True)
_redis_instance = rediscluster.RedisCluster(connection_pool=connection_pool, skip_full_coverage_check=True)
return _redis_instance
def get_cluster_conn(cluster_nodes):
cluster_list = []
redis_list = cluster_nodes.split(",")
for item in redis_list:
redis_node = {}
rel = item.split(":")
redis_node['host'] = rel[0]
redis_node['port'] = rel[1]
cluster_list.append(redis_node)
print("当前连接测试的Redis集群信息是 : ", cluster_list)
try:
conn = rediscluster.RedisCluster(startup_nodes=cluster_list,
decode_responses=True,
max_connections=300,
socket_connect_timeout=0.4)
conn.ping()
return conn
except Exception as ex:
print(ex.__cause__)
print("Redis集群连接异常")
def handler(event, context):
"""
This function puts gets the user recommendations
from Redis based on the user id.
"""
try:
print(event)
redis_host_endpoint = os.environ["REDIS_HOST_ENDPOINT"]
if 'queryStringParameters' not in event:
raise Exception("userId and rank is not in the input provided")
if "userId" not in event['queryStringParameters']:
raise Exception("UserId is not in the input provided")
if "rank" not in event['queryStringParameters']:
raise Exception("Movie rank is not in the input provided")
print(redis_host_endpoint)
c = connect(redis_host_endpoint)
r_cluter_on = rediscluster.RedisCluster(connection_pool=c)
user_id = event['queryStringParameters']['userId']
rank = event['queryStringParameters']['rank']
result = r_cluter_on.hmget(user_id, rank)
if result[0] == None:
return {
"statusCode":
200,
"body":
f"There is no number {rank} recommended movie for user {user_id} "
}
else:
return {
"statusCode":
200,
"body":
f"The number {rank} recommended movie for user {user_id} is {result[0]}"
}
except Exception as e:
return {"statusCode": 500, "body": f"Found an exception: '{e}'"}
def get_redis_client(redis_type='single',
host='127.0.0.1',
port=6379,
db=0,
pwd=None,
timeout=3):
if redis_type == 'single':
pool = redis.ConnectionPool(host=host,
port=port,
db=db,
password=pwd,
socket_timeout=timeout,
socket_connect_timeout=timeout,
encoding='utf-8',
decode_responses=True)
client = redis.StrictRedis(connection_pool=pool)
else:
nodes = [{"host": host, "port": port}]
client = rediscluster.RedisCluster(startup_nodes=nodes,
decode_responses=True,
socket_timeout=timeout,
socket_connect_timeout=timeout)
return client
def get_RedisConnection(redis_server,
redis_port=6379,
redis_password=None,
db=0):
#redis连接单节点与集群不一样
if isinstance(redis_server, basestring):
return redis.Redis(connection_pool=redis.ConnectionPool(
host=redis_server, port=redis_port, password=redis_password, db=0))
if isinstance(redis_server, list):
if len(redis_server) == 1:
return redis.Redis(
connection_pool=redis.ConnectionPool(host=redis_server[0],
port=redis_port,
password=redis_password,
db=db))
else:
startup_nodes = []
for _server in redis_server:
_host, _port = _server.split(':')
startup_nodes.append({'host': _host, 'port': _port})
return rediscluster.RedisCluster(startup_nodes=startup_nodes,
decode_responses=True,
password=redis_password)
raise Exception('no redis_server')
def setUp(self):
rediscluster.RedisCluster("localhost", 7000).flushall()
#необходимо для работы с переменными окружения
import os
import sys
#необходимо для работы с API openweathermap
import pyowm
from pyowm.utils.config import get_default_config
from pyowm.utils import timestamps
import rediscluster
#задание параметров redis'а
redis_host, redis_port = '194.61.2.84', 6379
rc = rediscluster.RedisCluster(startup_nodes=[{
"host": redis_host,
"port": redis_port
}],
decode_responses=True)
time_storage = 60 * 10 #10 минут - актуальность данных о погоде в городе
config_dict = get_default_config()
config_dict['language'] = 'ru'
app_key = os.environ.get('OWM_APP_KEY')
owm = pyowm.OWM(app_key, config_dict)
my_id = sys.argv[1]
def current_weather(city: str):
#обращение к внешнему сервису
def _get_redis():
"""
Connects to the redis database as specified by the
environmental variables.
The redis API already uses an internal, thread-safe
connection pool. Therefore, no additional connection
pool has to be created.
:return: A Redis connection pointing towards our redis instance.
"""
# get the main config from environment variables or
# use sensible default values
redis_host = os.getenv("REDIS_HOST", "redis")
redis_port = int(os.getenv("REDIS_PORT", 6379))
redis_database = int(os.getenv("REDIS_DATABASE", 0))
use_redis_cluster = os.getenv("USE_REDIS_CLUSTER", "False") == "True"
# load the password from file if set
redis_password_file = os.getenv("REDIS_PASSWORD_FILE", None)
redis_password = None
if redis_password_file and os.path.isfile(redis_password_file):
with open(redis_password_file, "r") as reader:
for line in reader:
line = line.strip()
if len(line) > 0:
redis_password = line
# environment variable overwrites file
redis_env_password = os.getenv("REDIS_PASSWORD", None)
if redis_env_password:
redis_password = redis_env_password
LOGGER.debug("Connecting to Redis at " + redis_host + ":" +
str(redis_port))
# TODO: select redis cluster or the default redis depending on some (new) config variable
startup_nodes = [{"host": redis_host, "port": redis_port}]
if use_redis_cluster:
redis_connection = rediscluster.RedisCluster(
startup_nodes=startup_nodes,
password=redis_password,
retry_on_timeout=False,
socket_keepalive=False,
socket_timeout=3,
socket_connect_timeout=3)
else:
redis_connection = redis.Redis(host=redis_host,
port=redis_port,
db=redis_database,
password=redis_password,
retry_on_timeout=False,
socket_keepalive=False,
socket_timeout=3,
socket_connect_timeout=3)
return redis_connection
def setUp(self):
rediscluster.RedisCluster("localhost", 7000).flushall()
self.storage_url = "redis+cluster://localhost:7000"
self.storage = RedisClusterStorage("redis+cluster://localhost:7000")
def getClusterConnection(self):
return rediscluster.RedisCluster(startup_nodes=[{'host': 'localhost', 'port': self.shards[0].getMasterPort()}],
decode_responses=True)
import rediscluster
host = "47.100.30.48"
conn = rediscluster.RedisCluster(
startup_nodes=[
# {"host": host, "port": "7000"},
# {"host": host, "port": "8001"},
{
"host": host,
"port": "9000"
},
# {"host": host, "port": "9001"},
# {"host": host, "port": "7001"},
# {"host": host, "port": "8000"}
],
decode_responses=True)
conn.set("x1", "hello world", ex=5) # ex代表seconds，px代表ms
val = conn.get("x1")
print(val)
import rediscluster
nodes = [{
"host": "172.16.44.142",
"port": 7000
}, {
"host": "172.16.44.142",
"port": 7001
}, {
"host": "172.16.44.142",
"port": 7002
}, {
"host": "172.16.44.142",
"port": 7003
}, {
"host": "172.16.44.142",
"port": 7004
}, {
"host": "172.16.44.142",
"port": 7005
}]
cluster = rediscluster.RedisCluster(startup_nodes=nodes, decode_responses=True)
# print(cluster.get('name'))
print(cluster.set('name', 'test'))
def get_conn(nodes):
"""
get a connection to the redis cluster
parameter 'nodes' is like [{'host':'xxx','port':yyy},{'host':'xx','port':yy}]
"""
return rediscluster.RedisCluster(startup_nodes=nodes)
def __init__(self, host, port, nodes):
self.client = rediscluster.RedisCluster(host,
port,
startup_nodes=nodes,
decode_responses=True)
self.client.ping()
