def handler(event, context): """ This function puts the user recommendations into Redis after fetching them from S3 """ try: redis_host_endpoint = os.environ["REDIS_HOST_ENDPOINT"] s3_bucket_name = os.environ["S3_BUCKET"].split(":::")[1] result = boto3.client("s3").get_object(Bucket=s3_bucket_name, Key="batchpredictions.json") user_predictions = json.loads(result["Body"].read().decode("utf-8")) users = user_predictions["data"] c = connect(redis_host_endpoint) r = rediscluster.RedisCluster(connection_pool=c) for user in users: dict = {} movies = user["movieId"] count = 1 for movie in movies: dict[count] = movie count += 1 r.hmset(user["userId"], dict) return f"Successfully inserted the user recommendations" except Exception as e: return f"Failed to insert the user recommendations due to {e}"
def setUp(self): pymemcache.client.Client(('localhost', 11211)).flush_all() redis.Redis().flushall() redis.sentinel.Sentinel([ ("localhost", 26379) ]).master_for("localhost-redis-sentinel").flushall() rediscluster.RedisCluster("localhost", 7000).flushall()
def main(): if not (len(sys.argv) == 3 or len(sys.argv) == 4): print("Usage: host port password") exit(1) host = sys.argv[1] port = sys.argv[2] password = None if len(sys.argv) == 4: password = sys.argv[3] # Add more nodes here to the list startup_nodes = [{"host": host, "port": port}] rc = rediscluster.RedisCluster( startup_nodes=startup_nodes, decode_responses=True, password=password, ssl=True, ssl_keyfile="../../../testscripts/tls/db_key.pem", ssl_certfile="../../../testscripts/tls/db_cert.pem", ssl_cert_reqs="required", ssl_ca_certs="../../../testscripts/tls/ca_cert.pem", # ssl_check_hostname=False, This option is not supported yet ) print("Set: {}".format(rc.set("foo", "bar"))) print("Get: {}".format(rc.get("foo")))
def setUp(self): pymemcache.client.Client(('localhost', 22122)).flush_all() redis.from_url("redis://*****:*****@localhost:7389").flushall() redis.sentinel.Sentinel([ ("localhost", 26379) ]).master_for("localhost-redis-sentinel").flushall() rediscluster.RedisCluster("localhost", 7000).flushall()
def _get_test_client(self): startup_nodes = [{ 'host': self.TEST_HOST, 'port': int(port) } for port in self.TEST_PORTS.split(',')] if REDISCLUSTER_VERSION >= (2, 0, 0): return rediscluster.RedisCluster(startup_nodes=startup_nodes) else: return rediscluster.StrictRedisCluster(startup_nodes=startup_nodes)
def get_redis_connection(cluster_mode=False): if cluster_mode: startup_nodes = [{"host": cnt.REDIS_SERVER, "port": str(cnt.REDIS_PORT)}] r = rediscluster.RedisCluster(startup_nodes=startup_nodes, decode_responses=True, skip_full_coverage_check=True) else: r = redis.StrictRedis(host=cnt.REDIS_SERVER, port=cnt.REDIS_PORT, db=0, decode_responses=True) return r
def setUp(self): pymemcache.client.Client(('localhost', 11211)).flush_all() redis.Redis().flushall() redis.sentinel.Sentinel([ ("localhost", 26379) ]).master_for("localhost-redis-sentinel").flushall() rediscluster.RedisCluster("localhost", 7000).flushall() tb = testbed.Testbed() tb.activate() tb.init_memcache_stub()
def getClusterConnection(self): if self.useTLS: # workaround for error on # got an unexpected keyword argument 'ssl' # we enforce the connection_class instead of setting ssl=True pool = ClusterConnectionPool( startup_nodes=self.getMasterNodesList(), connection_class=SSLClusterConnection, ssl_cert_reqs=None, ssl_keyfile=self.shards[0].getTLSKeyFile(), ssl_certfile=self.shards[0].getTLSCertFile(), ssl_ca_certs=self.shards[0].getTLSCACertFile(), ) if pool.connection_kwargs: pool.connection_kwargs.pop('ssl', None) return rediscluster.RedisCluster( startup_nodes=self.getMasterNodesList(), connection_pool=pool) else: return rediscluster.RedisCluster( startup_nodes=self.getMasterNodesList(), decode_responses=True)
def resolve_destination(dststring): conn = resolve_host(dststring) if not conn.info('cluster').get('cluster_enabled', None): return conn if not rediscluster: raise RuntimeError( 'cluster destination specified and redis-py-cluster not installed') host, port = dststring.split(':') return rediscluster.RedisCluster( startup_nodes=[{'host': host, 'port': port}], max_connections=1000)
def setUp(self): pymemcache.client.Client(('localhost', 11211)).flush_all() redis.from_url('unix:///var/tmp/limits.redis.sock').flushall() redis.Redis().flushall() redis.sentinel.Sentinel([ ("localhost", 26379) ]).master_for("localhost-redis-sentinel").flushall() rediscluster.RedisCluster("localhost", 7000).flushall() if RUN_GAE: from google.appengine.ext import testbed tb = testbed.Testbed() tb.activate() tb.init_memcache_stub()
def init(config_file_path=None, **kwargs): """ Init the storage client. Basically, we set the Redis client and connects it to the instance/cluster. """ if __debug__: logger.debug(HEADER + "Initializing the storage client.") global redis_connection global hosts # If config_file_path is None we will assume that we only have localhost # as storage node if config_file_path is None: try: import StringIO as sio except ImportError: from io import StringIO as sio config_file_handler = sio.StringIO('localhost\n') else: config_file_handler = open(config_file_path) # As accorded in the API standar, this file must contain all the hosts # names with no port, one per line hosts = [x.strip() for x in config_file_handler.readlines()] config_file_handler.close() # If we have more than one host then we will assume that our backend is a # Redis cluster. If not, we will assume that we are dealing with a Redis # standalone instance if len(hosts) > 1: # Given that cluster clients are capable to perform master # slave hierarchy discovery, we will simply connect to the first # node we got redis_connection = \ rediscluster.RedisCluster(host=hosts[0], port=REDIS_PORT) else: # We are in standalone mode redis_connection = \ redis.StrictRedis(host=hosts[0], port=REDIS_PORT) # StrictRedis is not capable to know if we had success when connecting by # simply calling the constructor. We need to perform an actual query to # the backend # If we had no success this first line should crash redis_connection.set('PYCOMPSS_TEST', 'OK') # Beware py2 vs py3 - b'string' works for both. assert redis_connection.get('PYCOMPSS_TEST') == b'OK' redis_connection.delete('PYCOMPSS_TEST') if __debug__: logger.debug(HEADER + "Initialization finished successfully.")
def main(): if not (len(sys.argv) == 3 or len(sys.argv) == 4): print("Usage: host port password") exit(1) host = sys.argv[1] port = sys.argv[2] password = None if len(sys.argv) == 4: password = sys.argv[3] # Add more nodes here to the list startup_nodes = [{"host": host, "port": port}] rc = rediscluster.RedisCluster(startup_nodes=startup_nodes, decode_responses=True, password=password) print("Set: {}".format(rc.set("foo", "bar"))) print("Get: {}".format(rc.get("foo")))
def get_redis(): """ Obtains a cluster aware redis connection. :return ClusterConnectionPool: The redis connection pool """ global _redis_instance # Due to a pending release from the `redis-py-cluster` project, pulling in # connection code here. Once https://github.com/Grokzen/redis-py-cluster/pull/195 # is released, this can be reset to the original line (left for ease of transition). # return rediscluster.RedisCluster.from_url(redis_config.URL, skip_full_coverage_check=True) if _redis_instance is None: connection_pool = ClusterConnectionPool.from_url(redis_config.URL, skip_full_coverage_check=True) _redis_instance = rediscluster.RedisCluster(connection_pool=connection_pool, skip_full_coverage_check=True) return _redis_instance
def get_cluster_conn(cluster_nodes): cluster_list = [] redis_list = cluster_nodes.split(",") for item in redis_list: redis_node = {} rel = item.split(":") redis_node['host'] = rel[0] redis_node['port'] = rel[1] cluster_list.append(redis_node) print("当前连接测试的Redis集群信息是 : ", cluster_list) try: conn = rediscluster.RedisCluster(startup_nodes=cluster_list, decode_responses=True, max_connections=300, socket_connect_timeout=0.4) conn.ping() return conn except Exception as ex: print(ex.__cause__) print("Redis集群连接异常")
def handler(event, context): """ This function puts gets the user recommendations from Redis based on the user id. """ try: print(event) redis_host_endpoint = os.environ["REDIS_HOST_ENDPOINT"] if 'queryStringParameters' not in event: raise Exception("userId and rank is not in the input provided") if "userId" not in event['queryStringParameters']: raise Exception("UserId is not in the input provided") if "rank" not in event['queryStringParameters']: raise Exception("Movie rank is not in the input provided") print(redis_host_endpoint) c = connect(redis_host_endpoint) r_cluter_on = rediscluster.RedisCluster(connection_pool=c) user_id = event['queryStringParameters']['userId'] rank = event['queryStringParameters']['rank'] result = r_cluter_on.hmget(user_id, rank) if result[0] == None: return { "statusCode": 200, "body": f"There is no number {rank} recommended movie for user {user_id} " } else: return { "statusCode": 200, "body": f"The number {rank} recommended movie for user {user_id} is {result[0]}" } except Exception as e: return {"statusCode": 500, "body": f"Found an exception: '{e}'"}
def get_redis_client(redis_type='single', host='127.0.0.1', port=6379, db=0, pwd=None, timeout=3): if redis_type == 'single': pool = redis.ConnectionPool(host=host, port=port, db=db, password=pwd, socket_timeout=timeout, socket_connect_timeout=timeout, encoding='utf-8', decode_responses=True) client = redis.StrictRedis(connection_pool=pool) else: nodes = [{"host": host, "port": port}] client = rediscluster.RedisCluster(startup_nodes=nodes, decode_responses=True, socket_timeout=timeout, socket_connect_timeout=timeout) return client
def get_RedisConnection(redis_server, redis_port=6379, redis_password=None, db=0): #redis连接单节点与集群不一样 if isinstance(redis_server, basestring): return redis.Redis(connection_pool=redis.ConnectionPool( host=redis_server, port=redis_port, password=redis_password, db=0)) if isinstance(redis_server, list): if len(redis_server) == 1: return redis.Redis( connection_pool=redis.ConnectionPool(host=redis_server[0], port=redis_port, password=redis_password, db=db)) else: startup_nodes = [] for _server in redis_server: _host, _port = _server.split(':') startup_nodes.append({'host': _host, 'port': _port}) return rediscluster.RedisCluster(startup_nodes=startup_nodes, decode_responses=True, password=redis_password) raise Exception('no redis_server')
def setUp(self): rediscluster.RedisCluster("localhost", 7000).flushall()
#необходимо для работы с переменными окружения import os import sys #необходимо для работы с API openweathermap import pyowm from pyowm.utils.config import get_default_config from pyowm.utils import timestamps import rediscluster #задание параметров redis'а redis_host, redis_port = '194.61.2.84', 6379 rc = rediscluster.RedisCluster(startup_nodes=[{ "host": redis_host, "port": redis_port }], decode_responses=True) time_storage = 60 * 10 #10 минут - актуальность данных о погоде в городе config_dict = get_default_config() config_dict['language'] = 'ru' app_key = os.environ.get('OWM_APP_KEY') owm = pyowm.OWM(app_key, config_dict) my_id = sys.argv[1] def current_weather(city: str): #обращение к внешнему сервису
def _get_redis(): """ Connects to the redis database as specified by the environmental variables. The redis API already uses an internal, thread-safe connection pool. Therefore, no additional connection pool has to be created. :return: A Redis connection pointing towards our redis instance. """ # get the main config from environment variables or # use sensible default values redis_host = os.getenv("REDIS_HOST", "redis") redis_port = int(os.getenv("REDIS_PORT", 6379)) redis_database = int(os.getenv("REDIS_DATABASE", 0)) use_redis_cluster = os.getenv("USE_REDIS_CLUSTER", "False") == "True" # load the password from file if set redis_password_file = os.getenv("REDIS_PASSWORD_FILE", None) redis_password = None if redis_password_file and os.path.isfile(redis_password_file): with open(redis_password_file, "r") as reader: for line in reader: line = line.strip() if len(line) > 0: redis_password = line # environment variable overwrites file redis_env_password = os.getenv("REDIS_PASSWORD", None) if redis_env_password: redis_password = redis_env_password LOGGER.debug("Connecting to Redis at " + redis_host + ":" + str(redis_port)) # TODO: select redis cluster or the default redis depending on some (new) config variable startup_nodes = [{"host": redis_host, "port": redis_port}] if use_redis_cluster: redis_connection = rediscluster.RedisCluster( startup_nodes=startup_nodes, password=redis_password, retry_on_timeout=False, socket_keepalive=False, socket_timeout=3, socket_connect_timeout=3) else: redis_connection = redis.Redis(host=redis_host, port=redis_port, db=redis_database, password=redis_password, retry_on_timeout=False, socket_keepalive=False, socket_timeout=3, socket_connect_timeout=3) return redis_connection
def setUp(self): rediscluster.RedisCluster("localhost", 7000).flushall() self.storage_url = "redis+cluster://localhost:7000" self.storage = RedisClusterStorage("redis+cluster://localhost:7000")
def getClusterConnection(self): return rediscluster.RedisCluster(startup_nodes=[{'host': 'localhost', 'port': self.shards[0].getMasterPort()}], decode_responses=True)
import rediscluster host = "47.100.30.48" conn = rediscluster.RedisCluster( startup_nodes=[ # {"host": host, "port": "7000"}, # {"host": host, "port": "8001"}, { "host": host, "port": "9000" }, # {"host": host, "port": "9001"}, # {"host": host, "port": "7001"}, # {"host": host, "port": "8000"} ], decode_responses=True) conn.set("x1", "hello world", ex=5) # ex代表seconds,px代表ms val = conn.get("x1") print(val)
import rediscluster nodes = [{ "host": "172.16.44.142", "port": 7000 }, { "host": "172.16.44.142", "port": 7001 }, { "host": "172.16.44.142", "port": 7002 }, { "host": "172.16.44.142", "port": 7003 }, { "host": "172.16.44.142", "port": 7004 }, { "host": "172.16.44.142", "port": 7005 }] cluster = rediscluster.RedisCluster(startup_nodes=nodes, decode_responses=True) # print(cluster.get('name')) print(cluster.set('name', 'test'))
def get_conn(nodes): """ get a connection to the redis cluster parameter 'nodes' is like [{'host':'xxx','port':yyy},{'host':'xx','port':yy}] """ return rediscluster.RedisCluster(startup_nodes=nodes)
def __init__(self, host, port, nodes): self.client = rediscluster.RedisCluster(host, port, startup_nodes=nodes, decode_responses=True) self.client.ping()