def init_seeding(self, x):
init_nodes = random.sample(list(self.nodes.keys()), x)
cluster_can_add = list()
for i in range(x):
temp_cluster = Cluster(str(i + 1))
temp_cluster.add_node(self.nodes[init_nodes[i]], True)
self.clusters[str(i + 1)] = temp_cluster
cluster_can_add.append(temp_cluster)
n = len(cluster_can_add)
while n > 0:
j = int(random.random() * n)
temp_cluster = cluster_can_add[j]
node = self.find_to_add(temp_cluster)
if node is None:
cluster_can_add.pop(j)
n -= 1
self.total_population += temp_cluster.total_population
continue
temp_cluster.add_node(self.nodes[node], True)
def create_clusters(node_data):
for node_name, node_tags in node_data:
c = Cluster(ID=int(node_name))
c.add_node(node_name, node_tags)
clusterglobals.clusters_list.append(c)
class TestCluster(unittest.TestCase):
def setUp(self):
print "### setUp ###"
self.cluster = Cluster()
def test_make_key(self):
print "test_make_key"
text = "%s:%s" % (HOST, PORT)
self.assertEqual(self.cluster._make_key(HOST, PORT), text)
def test_set_role(self):
self.cluster.add_node(host=HOST,
port=PORT,
role=ROLE_MASTER,
status=REDIS_STATUS_OK)
self.cluster._set_role(HOST, PORT, ROLE_MASTER)
node = self.cluster.get_node(HOST, PORT)
self.assertTrue(node.is_master)
def test_filtered_list(self):
print "test_filtered_list"
self.cluster.add_node(host=HOST,
port=PORT,
role=ROLE_MASTER,
status=REDIS_STATUS_OK)
self.cluster.add_node(host=HOST,
port=PORT + 1,
role=ROLE_SLAVE,
status=REDIS_STATUS_KO)
self.cluster.add_node(host=HOST,
port=PORT + 2,
role=ROLE_SLAVE,
status=REDIS_STATUS_OK)
testlist = self.cluster.filtered_list(roles=(ROLE_MASTER))
self.assertEqual(len(testlist), 1)
self.assertTrue(testlist[0].is_master())
self.assertTrue(testlist[0].is_alive())
testlist = self.cluster.filtered_list(roles=(ROLE_SLAVE))
self.assertEqual(len(testlist), 2)
self.assertTrue(testlist[0].is_slave() and testlist[1].is_slave())
testlist = self.cluster.filtered_list()
self.assertEqual(len(testlist), 3)
testlist = self.cluster.filtered_list(status=REDIS_STATUS_KO)
self.assertEqual(len(testlist), 1)
self.assertTrue(testlist[0].is_slave() and not testlist[0].is_alive())
testlist = self.cluster.filtered_list(status=REDIS_STATUS_OK)
self.assertEqual(len(testlist), 2)
testlist = self.cluster.filtered_list(roles=ROLE_MASTER,
status=REDIS_STATUS_KO)
self.assertEqual(len(testlist), 0)
testlist = self.cluster.filtered_list(roles=(ROLE_MASTER, ROLE_SLAVE),
status=REDIS_STATUS_KO)
self.assertEqual(len(testlist), 1)
self.assertTrue(testlist[0].is_slave() and not testlist[0].is_alive())
def test_add_single_master(self):
print "test_add_single_master"
self.cluster.add_node(host=HOST,
port=PORT,
role=ROLE_MASTER,
status=REDIS_STATUS_OK)
self.assertEqual(len(self.cluster), 1)
node = self.cluster.get_master()
self.assertTrue(node.is_master() and node.is_alive())
def test_add_single_slave(self):
print "test_add_single_slave"
self.cluster.add_node(host=HOST,
port=PORT,
role=ROLE_SLAVE,
status=REDIS_STATUS_OK)
self.assertEqual(len(self.cluster), 1)
self.assertIsNone(self.cluster.get_master())
node = self.cluster.get_node(host=HOST, port=PORT)
self.assertTrue(node.is_slave() and node.is_alive())
def test_promote_new_master_single_node(self):
print "test_promote_new_master_single_node"
old_master = self.cluster.add_node(host=HOST,
port=PORT,
role=ROLE_MASTER,
status=REDIS_STATUS_OK)
self.assertEqual(len(self.cluster), 1)
self.cluster.promote_new_master(old_master)
node = self.cluster.get_master()
self.assertTrue(node is not None)
self.assertTrue(node.is_master())
self.assertFalse(node.is_alive())
def test_str(self):
print "test_str"
self.cluster.add_node(host=HOST,
port=PORT,
role=ROLE_MASTER,
status=REDIS_STATUS_OK)
self.cluster.add_node(host=HOST,
port=PORT + 1,
role=ROLE_SLAVE,
status=REDIS_STATUS_OK)
self.assertEqual(
"(localhost:1000,master,OK),(localhost:1001,slave,OK)",
str(self.cluster))
def _create_message(self, redis_host, redis_port, redis_status):
worker_name = '%s:%s' % (redis_host, redis_port)
text = "%s,%s" % (worker_name, redis_status)
return text
def tearDown(self):
print "### tearDown ###"
self.cluster = None
class RedisMonitor(object):
''' The main monitor object. It starts all the Workers and
changes the cluster configuration in ZooKeeper when a Worker
send an update message.'''
def __init__(self, zk, redis_hosts, sleep_time, zk_path):
self.redis_class = redis.Redis
self.zk = zk
self.cluster = Cluster()
self.queue = Queue()
self.redis_hosts = redis_hosts
self.zk_properties = None
self.list_of_workers = []
self.zk_path = zk_path
self.redis_servers_map = {}
self.sleep_time = sleep_time
def execute(self):
''' execute is the only public method. It starts the main loop.'''
logger.info("Enter")
logger.info("self.redis_class=%s", str(self.redis_class))
self._discover_redis()
while 1:
self._parse_message_from_queue()
self._check_all_workers()
logger.info("Exit")
def _discover_redis(self):
''' check all the redis instances and registers to zk host, port and status.
It also prepares all the worker processes for monitoring.'''
logger.info('Enter')
self.zk.create_recursive(self.zk_path, '', OPEN_ACL_UNSAFE)
self.zk_properties = self.zk.properties(self.zk_path)
self._build_list_of_workers()
(master_as_tuple, list_of_slaves) = self._get_cluster_representation()
logger.info('master: [%r]', master_as_tuple[0])
slaves_map = {}
for i, slave in enumerate(list_of_slaves):
slaves_map[i] = ('%s:%s' % (slave.host, slave.port), slave.status)
logger.info("slaves: [%r] ", slaves_map)
self._update_zk_properties(master = master_as_tuple, slaves = slaves_map)
# start all workers
for worker in self.list_of_workers:
worker.start()
logger.info("%d Workers created", len(self.list_of_workers))
logger.info("Exit")
def _get_cluster_representation(self):
master = self.cluster.get_master()
master_as_tuple = ('%s:%s' %(master.host, master.port), master.status)
list_of_slaves = self.cluster.filtered_list((ROLE_SLAVE, ))
return (master_as_tuple, list_of_slaves)
def _parse_message_from_queue(self):
''' Blocks on queue, receives and reads messages from workers,
execute update actions based on past and present cluster's status.'''
message = self.queue.get()
logger.debug("Received message from Worker: [%s]", message)
server, new_status = message.split(',')
host, port = server.split(':')
present_node = self.cluster.get_node(host, port)
old_status = present_node.status
if new_status == REDIS_STATUS_KO:
present_node.setKO()
if old_status == REDIS_STATUS_OK:
logger.warn("Node (%s) [%s] has DIED!", present_node.role, server)
if present_node.is_master():
logger.warn("Master is down: promoting a new master...")
self._promote_new_master(present_node)
self._update_zk()
elif new_status == REDIS_STATUS_OK:
present_node.setOK()
if old_status == REDIS_STATUS_KO:
logger.warn("Node (%s) [%s] has RESURRECTED!", present_node.role, server)
redis_server = self._get_redis_server(host=present_node.host, port=present_node.port)
master = self.cluster.get_master()
if master.is_alive():
redis_server.slaveof(master.host, master.port)
else:
self._promote_new_master(master)
self._update_zk()
else:
logger.critical("Worker sent an unknown status: [%r]", new_status)
def _check_all_workers(self):
''' check if all worker daemons are still alive,
if not then re-start.'''
for worker in self.list_of_workers:
if not worker.get_process().is_alive():
logger.error("Process [%s] has died, restarting it...", worker.get_process().name)
worker.start()
def _get_redis_server(self, host, port):
''' Loookup the redis_class from a map. If not there,
creates and put in the map.'''
key = "%s:%d" % (host, port)
conn = None
if key in self.redis_servers_map:
conn = self.redis_servers_map[key]
else:
conn = self.redis_class(host, port)
self.redis_servers_map[key] = conn
return conn
def _build_list_of_workers(self):
''' Add nodes to cluster, for each host:port received from configuration.'''
for i, r in enumerate(self.redis_hosts):
role=ROLE_SLAVE
host = r.split(':')[0]
port = int(r.split(':')[1])
try:
role = self._get_redis_server(host=host, port=port).info()['role']
self.cluster.add_node(host, port, role, REDIS_STATUS_OK)
except ConnectionError:
logger.error("Node [%r] not available !!", r)
self.cluster.add_node(host, port, role, REDIS_STATUS_KO)
finally:
redis_server = self._get_redis_server(host=host, port=port)
self.list_of_workers.append(Worker(redis_server, host, port, self.queue, self.sleep_time))
def _promote_new_master(self, old_master):
''' Promotes the first available node as new master,
if the old master died and if at least one node is still alive.
All the remaining nodes are set as slaves of the new master.'''
logger.info("Enter")
new_master = self.cluster.promote_new_master(old_master)
if new_master.is_alive():
redis_master_server = self._get_redis_server(host=new_master.host, port=new_master.port)
# set this node as the master
redis_master_server.slaveof()
# set all the other alive nods as slaves of the new master
list_of_OK_slaves = self.cluster.filtered_list(roles=(ROLE_SLAVE,), status=(REDIS_STATUS_OK,))
for slave in list_of_OK_slaves:
redis_slave_server = self._get_redis_server(host=slave.host, port=slave.port)
redis_slave_server.slaveof(host=new_master.host, port=new_master.port)
else:
logger.critical("*** Can't promote a new master: all nodes are down! ***")
logger.info("Exit")
def _update_zk(self):
''' Updates ZooKeeper with the new cluster's configuration.'''
logger.info("Enter")
(master_as_tuple, list_of_slaves) = self._get_cluster_representation()
logger.info("Master: [%r]", master_as_tuple)
slaves_map = {}
for i, slave in enumerate(list_of_slaves):
slaves_map[i] = ("%s:%s" % (slave.host, slave.port), slave.status)
logger.info("Slaves: [%r]", slaves_map)
logger.warn("Updating ZooKeeper...")
self._update_zk_properties(master = master_as_tuple, slaves = slaves_map)
logger.info("Exit")
def _update_zk_properties(self, master, slaves):
''' Calls the update action on the configured zk_path'''
self.zk_properties.update(master = master, slaves = slaves)
def __len__(self):
return len(self.cluster)
def __str__(self):
return str(self.cluster)
class TestCluster(unittest.TestCase):
def setUp(self):
print "### setUp ###"
self.cluster = Cluster()
def test_make_key(self):
print "test_make_key"
text = "%s:%s" % (HOST, PORT)
self.assertEqual(self.cluster._make_key(HOST, PORT), text)
def test_set_role(self):
self.cluster.add_node(host=HOST, port=PORT, role=ROLE_MASTER, status=REDIS_STATUS_OK)
self.cluster._set_role(HOST, PORT, ROLE_MASTER)
node = self.cluster.get_node(HOST, PORT)
self.assertTrue(node.is_master)
def test_filtered_list(self):
print "test_filtered_list"
self.cluster.add_node(host=HOST, port=PORT, role=ROLE_MASTER, status=REDIS_STATUS_OK)
self.cluster.add_node(host=HOST, port=PORT+1, role=ROLE_SLAVE, status=REDIS_STATUS_KO)
self.cluster.add_node(host=HOST, port=PORT+2, role=ROLE_SLAVE, status=REDIS_STATUS_OK)
testlist = self.cluster.filtered_list(roles=(ROLE_MASTER))
self.assertEqual(len(testlist), 1)
self.assertTrue(testlist[0].is_master())
self.assertTrue(testlist[0].is_alive())
testlist = self.cluster.filtered_list(roles=(ROLE_SLAVE))
self.assertEqual(len(testlist), 2)
self.assertTrue(testlist[0].is_slave() and testlist[1].is_slave())
testlist = self.cluster.filtered_list()
self.assertEqual(len(testlist), 3)
testlist = self.cluster.filtered_list(status=REDIS_STATUS_KO)
self.assertEqual(len(testlist), 1)
self.assertTrue(testlist[0].is_slave() and not testlist[0].is_alive())
testlist = self.cluster.filtered_list(status=REDIS_STATUS_OK)
self.assertEqual(len(testlist), 2)
testlist = self.cluster.filtered_list(roles=ROLE_MASTER, status=REDIS_STATUS_KO)
self.assertEqual(len(testlist), 0)
testlist = self.cluster.filtered_list(roles=(ROLE_MASTER,ROLE_SLAVE), status=REDIS_STATUS_KO)
self.assertEqual(len(testlist), 1)
self.assertTrue(testlist[0].is_slave() and not testlist[0].is_alive())
def test_add_single_master(self):
print "test_add_single_master"
self.cluster.add_node(host=HOST, port=PORT, role=ROLE_MASTER, status=REDIS_STATUS_OK)
self.assertEqual(len(self.cluster), 1)
node = self.cluster.get_master()
self.assertTrue(node.is_master() and node.is_alive())
def test_add_single_slave(self):
print "test_add_single_slave"
self.cluster.add_node(host=HOST, port=PORT, role=ROLE_SLAVE, status=REDIS_STATUS_OK)
self.assertEqual(len(self.cluster), 1)
self.assertIsNone(self.cluster.get_master())
node = self.cluster.get_node(host=HOST, port=PORT)
self.assertTrue(node.is_slave() and node.is_alive())
def test_promote_new_master_single_node(self):
print "test_promote_new_master_single_node"
old_master = self.cluster.add_node(host=HOST, port=PORT, role=ROLE_MASTER, status=REDIS_STATUS_OK)
self.assertEqual(len(self.cluster), 1)
self.cluster.promote_new_master(old_master)
node = self.cluster.get_master()
self.assertTrue(node is not None)
self.assertTrue(node.is_master())
self.assertFalse(node.is_alive())
def test_str(self):
print "test_str"
self.cluster.add_node(host=HOST, port=PORT, role=ROLE_MASTER, status=REDIS_STATUS_OK)
self.cluster.add_node(host=HOST, port=PORT+1, role=ROLE_SLAVE, status=REDIS_STATUS_OK)
self.assertEqual("(localhost:1000,master,OK),(localhost:1001,slave,OK)", str(self.cluster))
def _create_message(self, redis_host, redis_port, redis_status):
worker_name = '%s:%s' % (redis_host, redis_port)
text = "%s,%s" % (worker_name, redis_status)
return text
def tearDown(self):
print "### tearDown ###"
self.cluster = None
ec2 = EC2(key, access)
events = Events(key, access, cluster.name())
host = Host(cluster.name(), events)
node = host.get_node()
endpoint = host.get_endpoint()
component = os.path.basename(sys.argv[0])
def log(message, logging='info'):
events.log(node, component, message, logging)
if __name__ == '__main__':
log('joining the cluster', 'info')
log('adding the node to the cluster', 'info')
# now we are ready to be (added to) the cluster
cluster.add_node(node, endpoint)
log('creating a Route53 records', 'info')
r53_zone.create_record(node, endpoint)
log('setting the tag', 'info')
ec2.set_tag(node)
log('getting the master of the node', 'info')
master = cluster.get_master(node)
# if we don't have a master, we ARE the master
if master == None:
log('setting the main Route53 record for the cluster', 'info')
r53_zone.update_record(cluster.name(), endpoint)
# and make sure we 'run' correctly (no-slave, well-monitored)
log('set the host to run as master', 'info')
host.set_master()
def create_clusters(node_data):
for node_name, node_tags in node_data:
c = Cluster(ID = int(node_name))
c.add_node(node_name, node_tags)
clusterglobals.clusters_list.append(c)
