import time
def multiLoader(user_list, r):
for email, user in user_list:
r.set(email, user)
startup_nodes = [
{"host":"127.0.0.1", "port":"6379"},
{"host":"127.0.0.1", "port":"6380"},
{"host":"127.0.0.1", "port":"6381"}
]
rc = StrictRedisCluster(startup_nodes=startup_nodes)
set_count = 0
start_time = int(time.time())
rc.flushall()
with open('users.csv', 'rU') as fd:
csvreader = csv.reader(fd)
user_list = []
for line in csvreader:
if csvreader.line_num == 1:
continue
email = line[0].strip()
first_name = line[1].strip()
if email and first_name:
user_list.append((email, first_name))
set_count += 1
import pickle
import sys
from rediscluster import StrictRedisCluster
def wait_log(redis_client, n, log_name):
num_keys = 0
old_num_keys = 0
while num_keys < n:
keys = redis_client.keys(pattern="*"+log_name+"*")
num_keys = len(keys)
time.sleep(1)
if num_keys != old_num_keys:
print str(num_keys) + " lambdas out of " + str(n) + " finished"
old_num_keys = num_keys
if __name__ == '__main__':
n = int(sys.argv[1])
# connect to redis
redis_host = "rediscluster-log.a9ith3.clustercfg.usw2.cache.amazonaws.com"
startup_nodes = [{"host": redis_host, "port": "6379"}]
redis_client = StrictRedisCluster(startup_nodes=startup_nodes, skip_full_coverage_check=True)
wait_log(redis_client, n, "logs")
print str(n) + " lambdas all finished"
redis_client.flushall()
print ">>> redis cluster flushed"
class DB(object):
def __init__(self, config):
self.config = config
cache_backend = self.config_get('cache_backend') or 'redis'
if cache_backend == 'redis':
self.__backend = cache_backend
self.__host = self.config_get('cache_host') or '127.0.0.1'
self.__port = self.config_get('cache_port') or 6379
self.__db_name = self.config_get('cache_db_name') or 0
self.db = redis.StrictRedis(host=self.__host,
port=self.__port,
db=self.__db_name)
elif cache_backend == 'redis-cluster':
self.__backend = cache_backend
node1_host = self.config_get('node1_host')
node1_port = self.config_get('node1_port')
node2_host = self.config_get('node2_host')
node2_port = self.config_get('node2_port')
node3_host = self.config_get('node3_host')
node3_port = self.config_get('node3_port')
node4_host = self.config_get('node4_host')
node4_port = self.config_get('node4_port')
node5_host = self.config_get('node5_host')
node5_port = self.config_get('node5_port')
node6_host = self.config_get('node6_host')
node6_port = self.config_get('node6_port')
self.__host = (node1_host, node2_host, node3_host, node4_host,
node5_host, node6_host)
self.__port = (node1_port, node2_port, node3_port, node4_port,
node5_port, node6_port)
self.__db_name = 'cluster'
cluster_nodes = [{
'host': node1_host,
'port': node1_port
}, {
'host': node2_host,
'port': node2_port
}, {
'host': node3_host,
'port': node3_port
}, {
'host': node4_host,
'port': node4_port
}, {
'host': node5_host,
'port': node5_port
}, {
'host': node6_host,
'port': node6_port
}]
self.db = StrictRedisCluster(startup_nodes=cluster_nodes,
decode_responses=True)
self.set_value = self.db.set
self.has_value = self.db.exists
self.remove_value = self.db.delete
def config_get(self, name):
if isinstance(self.config, ConfigParser):
return self.config.get('app:api', name)
else:
return self.config.get('app:api').get(name)
def get(self, key):
return self.db.get(key)
def keys(self, prefix):
keys = self.db.keys(prefix)
return keys
def put(self, key, value, ex=90000):
self.set_value(key, value, ex)
def remove(self, key):
self.remove_value(key)
def has(self, key):
return self.has_value(key)
def scan_iter(self, prefix=None):
return [key for key in self.db.scan_iter(prefix)]
def remove_pattern(self, prefix):
for key in self.db.scan_iter(prefix):
self.remove(key)
def flushall(self):
self.db.flushall()
@property
def backend(self):
return self.__backend
@property
def host(self):
return self.__host
@property
def port(self):
return self.__port
@property
def db_name(self):
return self.__db_name
from rediscluster import StrictRedisCluster
startup_nodes = [{
"host": "rediscluster.a9ith3.clustercfg.usw2.cache.amazonaws.com",
"port": "6379"
}]
client = StrictRedisCluster(startup_nodes=startup_nodes,
skip_full_coverage_check=True)
client.flushall()
print "Redis cluster flushed"
startup_nodes = [{
"host": "127.0.0.1",
"port": "6379"
}, {
"host": "127.0.0.1",
"port": "6380"
}, {
"host": "127.0.0.1",
"port": "6381"
}]
rc = StrictRedisCluster(startup_nodes=startup_nodes)
set_count = 0
start_time = int(time.time())
rc.flushall()
with open('users.csv', 'rU') as fd:
csvreader = csv.reader(fd)
user_list = []
for line in csvreader:
if csvreader.line_num == 1:
continue
email = line[0].strip()
first_name = line[1].strip()
if email and first_name:
user_list.append((email, first_name))
set_count += 1
