コード例 #1
0
ファイル: latency.py プロジェクト: adrianco/cassandgo
def getLatency(cmd,toIP):
    """
    Ping latency average between fromDC and toDC
    """
    try:
        res = cmd.run('sudo ping -c3 '+toIP)
        m = re.search(' = [^\/]+\/([^\/]+)\/.+ ms',res[1])
        if m:
            return "%.2f ms" % round(float(m.group(1)),2)
    except Exception as e:
        logError(e)
    return ''
コード例 #2
0
ファイル: cluster.py プロジェクト: adrianco/cassandgo
def createClusterDC(datacenter,region,zone,nodes,node_type,nb_seeds_per_dc,cluster_name,opscenter):
	cluster = {'cluster':cluster_name,'datacenter':datacenter,'region':region,'zone':zone,'nodes':nodes,'node_type':node_type,'nb_seeds_per_dc':nb_seeds_per_dc,'instances':[],'opscenter':opscenter}
	try:
		# Connection
		ec2 = boto.ec2.connect_to_region(region+'-'+zone)

		# AZ disponibles
		AZlist = ec2.get_all_zones()
		AZ = []
		for AZitem in AZlist:
			if AZitem.state == 'available':
				AZ.append(AZitem.name)
		if (len(AZ) == 0):
			# No available zone
			print "No available AZ !"
			return None

		# On récupère les infos pour l'AMI, key, SG
		info = filter(lambda x: ((x['region'] == region) and (x['zone'] == zone)), conf_HVM)[0]

		# Onlyne one AZ available ?
		if (len(AZ) == 1):
			# we create all instances in the same AZ
			createInstance(ec2,info['ami'],nodes,AZ[0],node_type,info['key'],info['sg'],'--clustername '+cluster_name+' --totalnodes '+str(nodes)+' --version community')
		else:
			# we cycle Availability Zones
			iAZ = 0
			for i in range(nodes):
				if ((i == 0) and (opscenter == True)):
					# On installe OpsCenter sur ce premier noeud Master
					instance = createInstance(ec2,info['ami'],1,AZ[iAZ],node_type,info['key'],info['sg'],'--clustername '+cluster_name+' --totalnodes 1 --version community')
				else:
					# Node without opscenter
					instance = createInstance(ec2,info['ami'],1,AZ[iAZ],node_type,info['key'],info['sg'],'--clustername '+cluster_name+' --opscenter no --totalnodes 1 --version community')
				node_name = "["+datacenter+"] "
				if i == 0:
					node_name += ' M '
				else:
					node_name += ' '+str(i+1)+' '
				node_name += cluster_name
				time.sleep(5)
				instance.add_tag('Name', node_name)
				instance.add_tag('Cluster', cluster_name) 
				cluster['instances'].append({'instance':instance,'index':str(i+1),'creation':datetime.datetime.now(),'region':region,'zone':zone,'datacenter':datacenter,'AZ':AZ[iAZ]})
				iAZ += 1
				if (iAZ == len(AZ)):	iAZ = 0

		return cluster
			
	except Exception as e:
		logError(e)
		return None
コード例 #3
0
def runCmd(cmd, command):
    """
        run command on the server via ssh (cmd : boto.manage.cmdshell)
        """

    try:
        res = cmd.run(command)
        if res[2] != '':
            raise Exception(res[2])
        return res
    except Exception as e:
        logError(e)
        return None
コード例 #4
0
ファイル: server.py プロジェクト: adrianco/cassandgo
def runCmd(cmd,command):
        """
        run command on the server via ssh (cmd : boto.manage.cmdshell)
        """

        try:
                res = cmd.run(command)
                if res[2] != '':
                        raise Exception(res[2])
                return res
        except Exception as e:
                logError(e)
                return None
コード例 #5
0
ファイル: cluster.py プロジェクト: thomas545/cassandgo
def createClusterDC(datacenter, region, zone, nodes, node_type,
                    nb_seeds_per_dc, cluster_name, opscenter):
    cluster = {
        'cluster': cluster_name,
        'datacenter': datacenter,
        'region': region,
        'zone': zone,
        'nodes': nodes,
        'node_type': node_type,
        'nb_seeds_per_dc': nb_seeds_per_dc,
        'instances': [],
        'opscenter': opscenter
    }
    try:
        # Connection
        ec2 = boto.ec2.connect_to_region(region + '-' + zone)

        # AZ disponibles
        AZlist = ec2.get_all_zones()
        AZ = []
        for AZitem in AZlist:
            if AZitem.state == 'available':
                AZ.append(AZitem.name)
        if (len(AZ) == 0):
            # No available zone
            print "No available AZ !"
            return None

        # On récupère les infos pour l'AMI, key, SG
        info = filter(
            lambda x: ((x['region'] == region) and (x['zone'] == zone)),
            conf_HVM)[0]

        # Onlyne one AZ available ?
        if (len(AZ) == 1):
            # we create all instances in the same AZ
            createInstance(
                ec2, info['ami'], nodes, AZ[0], node_type, info['key'],
                info['sg'], '--clustername ' + cluster_name +
                ' --totalnodes ' + str(nodes) + ' --version community')
        else:
            # we cycle Availability Zones
            iAZ = 0
            for i in range(nodes):
                if ((i == 0) and (opscenter == True)):
                    # On installe OpsCenter sur ce premier noeud Master
                    instance = createInstance(
                        ec2, info['ami'], 1, AZ[iAZ], node_type, info['key'],
                        info['sg'], '--clustername ' + cluster_name +
                        ' --totalnodes 1 --version community')
                else:
                    # Node without opscenter
                    instance = createInstance(
                        ec2, info['ami'], 1, AZ[iAZ], node_type, info['key'],
                        info['sg'], '--clustername ' + cluster_name +
                        ' --opscenter no --totalnodes 1 --version community')
                node_name = "[" + datacenter + "] "
                if i == 0:
                    node_name += ' M '
                else:
                    node_name += ' ' + str(i + 1) + ' '
                node_name += cluster_name
                time.sleep(5)
                instance.add_tag('Name', node_name)
                instance.add_tag('Cluster', cluster_name)
                cluster['instances'].append({
                    'instance': instance,
                    'index': str(i + 1),
                    'creation': datetime.datetime.now(),
                    'region': region,
                    'zone': zone,
                    'datacenter': datacenter,
                    'AZ': AZ[iAZ]
                })
                iAZ += 1
                if (iAZ == len(AZ)): iAZ = 0

        return cluster

    except Exception as e:
        logError(e)
        return None
コード例 #6
0
ファイル: cluster.py プロジェクト: thomas545/cassandgo
def createCluster(infosDC, cluster_name):
    """
	Create a Multi-DC Cassandra cluster
	"""
    clusters = []
    total_nodes = 0
    print
    print "Cluster creation..."
    print "-" * 80
    for infoDC in infosDC:
        total_nodes += infoDC['nodes']

        # Params by default
        node_type = infoDC[
            'node_type'] if 'node_type' in infoDC else default_node_type
        nb_seeds_per_dc = infoDC[
            'nb_seeds_per_dc'] if 'nb_seeds_per_dc' in infoDC else default_nb_seeds_per_dc
        opscenter = infoDC['opscenter'] if 'opscenter' in infoDC else True

        if opscenter:
            print "Cluster", cluster_name, " in ", (
                infoDC['region'] + '-' + str(infoDC['zone'])
            ), " :", infoDC[
                'nodes'], " nodes [", node_type, "] ", nb_seeds_per_dc, " seed(s) opsCenter:", opscenter
        else:
            print "Cluster", cluster_name, " in ", (
                infoDC['region'] + '-' + str(infoDC['zone'])
            ), " :", infoDC[
                'nodes'], " nodes [", node_type, "] ", nb_seeds_per_dc, " seed(s) "
        res = createClusterDC(infoDC['datacenter'], infoDC['region'],
                              str(infoDC['zone']), infoDC['nodes'], node_type,
                              nb_seeds_per_dc, cluster_name, opscenter)
        if res:
            clusters.append(res)
        else:
            logError("Error during cluster creation ! :" + str(infoDC))
            return None

    # Waiting for initalized and running nodes
    print
    print "Waiting for nodes..."
    print "-" * 80
    time.sleep(10)
    bUP = False
    while not bUP:
        for cluster in clusters:
            for instance in cluster['instances']:
                # Instance running ?
                if instance['instance'].state == 'running':
                    if 'Ready' not in instance['instance'].tags.keys():
                        if checkInstance(instance['instance']):
                            # Node is Ready
                            total_nodes -= 1
                            instance['instance'].add_tag('Ready', 1)
                            instance['running'] = datetime.datetime.now()
                            dt = instance['running'] - instance['creation']
                            duration = (dt.days * 24 * 60 * 60 + dt.seconds)
                            instance['creation_duration'] = duration
                            print cluster['datacenter'] + ' : Node ' + instance[
                                'index'] + ' in ' + instance[
                                    'AZ'] + ' -> OK [' + str(
                                        duration
                                    ) + 's] IP:' + instance[
                                        'instance'].ip_address + '/' + instance[
                                            'instance'].private_ip_address
                else:
                    instance['instance'].update()
        time.sleep(2)
        if total_nodes == 0:
            break
    print
    delay = 180
    print "Waiting for Cassandra warmup on all nodes (", delay, "s)..."
    print "-" * 80
    time.sleep(delay)

    print
    print "Stopping Cassandra, opsCenter and Datastax agent on all nodes..."
    print "-" * 80
    for cluster in clusters:
        for instance in cluster['instances']:
            print cluster['datacenter'] + ' : Node ' + instance[
                'index'] + ' in ' + instance['AZ']
            key_path = os.path.join(
                os.path.expanduser(keysDir),
                'Key-' + instance['region'] + '-' + instance['zone'] + '.pem')
            cmd = boto.manage.cmdshell.sshclient_from_instance(
                instance['instance'], key_path, user_name='ubuntu')
            bRunning = False
            while not bRunning:
                res = server.statusCassandra(cmd)
                if 'is running' in res[1]:
                    bRunning = True
                else:
                    time.sleep(5)
            server.stopCassandra(cmd)
            server.stopDataStaxAgent(cmd)
            if ((cluster['opscenter'] == True) and (instance['index'] == '1')):
                server.stopOpsCenter(cmd)

    print
    print "Cleaning cassandra data files on all nodes..."
    print "-" * 80
    time.sleep(30)
    for cluster in clusters:
        for instance in cluster['instances']:
            print cluster['datacenter'] + ' : Node ' + instance[
                'index'] + ' in ' + instance['AZ']
            key_path = os.path.join(
                os.path.expanduser(keysDir),
                'Key-' + instance['region'] + '-' + instance['zone'] + '.pem')
            cmd = boto.manage.cmdshell.sshclient_from_instance(
                instance['instance'], key_path, user_name='ubuntu')
            server.cleanCassandra(cmd)

    # Multiple Region ?
    snitch = "Ec2Snitch"
    regions = set([])
    for cluster in clusters:
        for instance in cluster['instances']:
            regions.add(instance['region'] + instance['zone'])
    if (len(regions) > 1):
        # Multiple
        snitch = "Ec2MultiRegionSnitch"

    # Enumerate seeds by DC
    bOpsCenterExists = False
    for cluster in clusters:
        cluster['opsCenterServer'] = None
        cluster['InternalSeeds'] = list()
        cluster['ExternalSeeds'] = list()
        for instance in cluster['instances']:
            if ((cluster['opscenter'] == True) and (instance['index'] == '1')):
                cluster['opsCenterServer'] = {
                    'publicIP': instance['instance'].ip_address,
                    'privateIP': instance['instance'].private_ip_address,
                    'instance': instance['instance'],
                    'regionZone': instance['region'] + '-' + instance['zone']
                }
                bOpsCenterExists = True
            if (len(cluster['InternalSeeds']) < cluster['nb_seeds_per_dc']):
                cluster['InternalSeeds'].append(
                    instance['instance'].private_ip_address)
                cluster['ExternalSeeds'].append(
                    instance['instance'].ip_address)

    # seeds merge
    for i in range(len(clusters)):
        clusters[i]['Seeds'] = list(clusters[i]['InternalSeeds'])
        for j in range(len(clusters)):
            if j != i:
                clusters[i]['Seeds'] += clusters[j]['ExternalSeeds']
    print
    print "Updating cassandra.yaml and DataStax agent conf on all nodes..."
    print "-" * 80
    for cluster in clusters:
        for instance in cluster['instances']:
            print cluster['datacenter'] + ' : Node ' + instance[
                'index'] + ' in ' + instance['AZ']
            key_path = os.path.join(
                os.path.expanduser(keysDir),
                'Key-' + instance['region'] + '-' + instance['zone'] + '.pem')
            cmd = boto.manage.cmdshell.sshclient_from_instance(
                instance['instance'], key_path, user_name='ubuntu')
            server.updateCassandraYaml(cmd, cluster['Seeds'],
                                       instance['instance'].ip_address, snitch)
            # Config DataStax agent
            if cluster['opsCenterServer']:
                # There is an OpsCenter Server on this DC, so we use the private ip address for agent stomp_interface
                server.updateDataStaxAgent(
                    cmd, cluster['opsCenterServer']['privateIP'])
            else:
                # no OpsCenter server on this DC, we search an available server in other DCs
                # Priority on opsCenter server on the same Region and Zone
                bStomp = False
                for cluster2 in clusters:
                    if cluster2 != cluster:
                        if cluster2['opsCenterServer']:
                            if (cluster2['opsCenterServer']['regionZone'] == (
                                    instance['region'] + '-' +
                                    instance['zone'])):
                                # same Region and Zone, we could use the private ip address for agent stomp_interface
                                server.updateDataStaxAgent(
                                    cmd,
                                    cluster2['opsCenterServer']['privateIP'])
                                bStomp = True
                                break
                # If not found, we searcg for an OpsCenter server in another Region Zone
                if not bStomp:
                    for cluster2 in clusters:
                        if cluster2 != cluster:
                            if cluster2['opsCenterServer']:
                                # we use the public ip address for agent stomp_interface
                                server.updateDataStaxAgent(
                                    cmd,
                                    cluster2['opsCenterServer']['publicIP'])
                                bStomp = True
                                break

    print
    print "Starting Cassandra Seeds nodes..."
    print "-" * 80
    for cluster in clusters:
        nb_seeds = 0
        for instance in cluster['instances']:
            if ((nb_seeds < cluster['nb_seeds_per_dc']) and
                (int(instance['index']) <= cluster['nb_seeds_per_dc'])):
                nb_seeds += 1
                print cluster['datacenter'] + ' : Node ' + instance[
                    'index'] + ' in ' + instance['AZ']
                key_path = os.path.join(
                    os.path.expanduser(keysDir), 'Key-' + instance['region'] +
                    '-' + instance['zone'] + '.pem')
                cmd = boto.manage.cmdshell.sshclient_from_instance(
                    instance['instance'], key_path, user_name='ubuntu')
                server.startCassandra(cmd)
                time.sleep(30)

    print
    print "Starting Cassandra on all other nodes..."
    print "-" * 80
    time.sleep(10)
    for cluster in clusters:
        nb_seeds = 0
        for instance in cluster['instances']:
            if ((nb_seeds < cluster['nb_seeds_per_dc']) and
                (int(instance['index']) <= cluster['nb_seeds_per_dc'])):
                nb_seeds += 1
                server.startDataStaxAgent(cmd)
                time.sleep(5)
            else:
                print cluster['datacenter'] + ' : Node ' + instance[
                    'index'] + ' in ' + instance['AZ']
                key_path = os.path.join(
                    os.path.expanduser(keysDir), 'Key-' + instance['region'] +
                    '-' + instance['zone'] + '.pem')
                cmd = boto.manage.cmdshell.sshclient_from_instance(
                    instance['instance'], key_path, user_name='ubuntu')
                server.startCassandra(cmd)
                server.startDataStaxAgent(cmd)
                time.sleep(5)

    if bOpsCenterExists:
        print
        print "Starting OpsCenter..."
        print "-" * 80
        time.sleep(10)
        for cluster in clusters:
            if cluster['opsCenterServer']:
                key_path = os.path.join(
                    os.path.expanduser(keysDir),
                    'Key-' + cluster['opsCenterServer']['regionZone'] + '.pem')
                cmd = boto.manage.cmdshell.sshclient_from_instance(
                    cluster['opsCenterServer']['instance'],
                    key_path,
                    user_name='ubuntu')
                server.startOpsCenter(cmd)
                print cluster['datacenter'] + ' : Node 1 in ' + cluster[
                    'opsCenterServer']['regionZone']

        print
        print "Starting DataStax Agents..."
        print "-" * 80
        time.sleep(10)
        for cluster in clusters:
            for instance in cluster['instances']:
                print cluster['datacenter'] + ' : Node ' + instance[
                    'index'] + ' in ' + instance['AZ']
                key_path = os.path.join(
                    os.path.expanduser(keysDir), 'Key-' + instance['region'] +
                    '-' + instance['zone'] + '.pem')
                cmd = boto.manage.cmdshell.sshclient_from_instance(
                    instance['instance'], key_path, user_name='ubuntu')
                server.startDataStaxAgent(cmd)
                time.sleep(5)

    print
    print "Cassandra cluster finalization..."
    print "-" * 80
    time.sleep(30)

    print
    print "Intra-Cluster latencies..."
    print "-" * 80
    for cluster in clusters:
        latency.getLatenciesRegion(cluster)

    print
    print "Inter-Cluster latencies..."
    print "-" * 80
    for i in range(len(clusters)):
        if (i == 0):
            dcs = []
            for j in range(len(clusters)):
                dcs.append(clusters[j]['datacenter'])
            latency.printLatency(clusters, dcs)
            print
        j = i + 2
        while (j < len(clusters)):
            dcs = [clusters[i]['datacenter'], clusters[j]['datacenter']]
            latency.printLatency(clusters, dcs)
            print
            j += 1

    print
    print "How to connect to " + cluster_name + "'s nodes..."
    print "-" * 80
    for cluster in clusters:
        print '[' + cluster['datacenter'] + '] ' + cluster[
            'region'] + '-' + cluster['zone']
        print "-" * 80
        for instance in cluster['instances']:
            print 'Node ' + instance[
                'index'] + ' : ssh -i ' + keysDir + '/Key-' + cluster[
                    'region'] + '-' + cluster[
                        'zone'] + '.pem ubuntu@' + instance[
                            'instance'].ip_address
        print

    if bOpsCenterExists:
        # Restart OpsCenterServer
        print
        print "Connect to OpsCenter"
        print "-" * 80
        for cluster in clusters:
            if cluster['opsCenterServer']:
                key_path = os.path.join(
                    os.path.expanduser(keysDir),
                    'Key-' + cluster['opsCenterServer']['regionZone'] + '.pem')
                cmd = boto.manage.cmdshell.sshclient_from_instance(
                    cluster['opsCenterServer']['instance'],
                    key_path,
                    user_name='ubuntu')
                server.restartOpsCenter(cmd)
                print 'http://' + cluster['opsCenterServer'][
                    'publicIP'] + ':8888'

    print
    print "Cluster status..."
    print "-" * 80
    instance = clusters[0]['instances'][0]
    key_path = os.path.join(
        os.path.expanduser(keysDir),
        'Key-' + instance['region'] + '-' + instance['zone'] + '.pem')
    cmd = boto.manage.cmdshell.sshclient_from_instance(instance['instance'],
                                                       key_path,
                                                       user_name='ubuntu')
    print server.clusterStatus(cmd)
    print

    return clusters
コード例 #7
0
ファイル: cluster.py プロジェクト: adrianco/cassandgo
def createCluster(infosDC,cluster_name):
	"""
	Create a Multi-DC Cassandra cluster
	"""
	clusters = []
	total_nodes = 0
	print
	print "Cluster creation..."
	print "-"*80
	for infoDC in infosDC:
		total_nodes += infoDC['nodes']

		# Params by default
		node_type = infoDC['node_type'] if 'node_type' in infoDC else default_node_type
		nb_seeds_per_dc = infoDC['nb_seeds_per_dc'] if 'nb_seeds_per_dc' in infoDC else default_nb_seeds_per_dc
		opscenter = infoDC['opscenter'] if 'opscenter' in infoDC else True

		if opscenter:
			print "Cluster",cluster_name," in ",(infoDC['region']+'-'+str(infoDC['zone']))," :",infoDC['nodes']," nodes [",node_type,"] ",nb_seeds_per_dc," seed(s) opsCenter:",opscenter
		else:
			print "Cluster",cluster_name," in ",(infoDC['region']+'-'+str(infoDC['zone']))," :",infoDC['nodes']," nodes [",node_type,"] ",nb_seeds_per_dc," seed(s) "
		res = createClusterDC(infoDC['datacenter'],infoDC['region'],str(infoDC['zone']),infoDC['nodes'],node_type,nb_seeds_per_dc,cluster_name,opscenter)
		if res:
			clusters.append(res)
		else:
			logError("Error during cluster creation ! :"+str(infoDC))
			return None

	# Waiting for initalized and running nodes
	print 
	print "Waiting for nodes..."
	print "-"*80
	time.sleep(10);
	bUP = False
	while not bUP:
		for cluster in clusters:
			for instance in cluster['instances']:
				# Instance running ?
				if instance['instance'].state == 'running':
					if 'Ready' not in instance['instance'].tags.keys():
						if checkInstance(instance['instance']):
							# Node is Ready
							total_nodes -= 1
							instance['instance'].add_tag('Ready', 1)
							instance['running'] = datetime.datetime.now()
							dt = instance['running'] - instance['creation']
							duration = (dt.days * 24 * 60 * 60 + dt.seconds)
							instance['creation_duration'] = duration
							print cluster['datacenter'] + ' : Node ' + instance['index'] + ' in ' + instance['AZ'] + ' -> OK [' + str(duration) + 's] IP:' + instance['instance'].ip_address + '/' + instance['instance'].private_ip_address
				else:
					instance['instance'].update()
		time.sleep(2)
		if total_nodes == 0:
			break
	print 
	delay = 180
	print "Waiting for Cassandra warmup on all nodes (",delay,"s)..."
	print "-"*80
	time.sleep(delay)

	print 
	print "Stopping Cassandra, opsCenter and Datastax agent on all nodes..."
	print "-"*80
	for cluster in clusters:
		for instance in cluster['instances']:
			print cluster['datacenter'] + ' : Node ' + instance['index'] + ' in ' + instance['AZ']
			key_path = os.path.join(os.path.expanduser(keysDir),'Key-'+instance['region']+'-'+instance['zone']+'.pem')
			cmd = boto.manage.cmdshell.sshclient_from_instance(instance['instance'],key_path,user_name='ubuntu')
			bRunning = False
			while not bRunning:
				res = server.statusCassandra(cmd)
				if 'is running' in res[1]:
					bRunning = True
				else:
					time.sleep(5)
			server.stopCassandra(cmd)
			server.stopDataStaxAgent(cmd)
			if ((cluster['opscenter'] == True) and (instance['index'] == '1')):
				server.stopOpsCenter(cmd)

	print 
	print "Cleaning cassandra data files on all nodes..."
	print "-"*80
	time.sleep(30)
	for cluster in clusters:
		for instance in cluster['instances']:
			print cluster['datacenter'] + ' : Node ' + instance['index'] + ' in ' + instance['AZ']
			key_path = os.path.join(os.path.expanduser(keysDir),'Key-'+instance['region']+'-'+instance['zone']+'.pem')
			cmd = boto.manage.cmdshell.sshclient_from_instance(instance['instance'],key_path,user_name='ubuntu')
			server.cleanCassandra(cmd)

	# Multiple Region ?
	snitch = "Ec2Snitch"
	regions = set([])
        for cluster in clusters:
        	for instance in cluster['instances']:
			regions.add(instance['region']+instance['zone'])
	if (len(regions) > 1):
		# Multiple
		snitch = "Ec2MultiRegionSnitch"

	# Enumerate seeds by DC
	bOpsCenterExists = False
	for cluster in clusters:
		cluster['opsCenterServer'] = None
		cluster['InternalSeeds'] = list()
		cluster['ExternalSeeds'] = list()
		for instance in cluster['instances']:
			if ((cluster['opscenter'] == True) and (instance['index'] == '1')):
				cluster['opsCenterServer'] = {'publicIP':instance['instance'].ip_address,'privateIP':instance['instance'].private_ip_address,'instance':instance['instance'],'regionZone':instance['region']+'-'+instance['zone']}
				bOpsCenterExists = True
			if (len(cluster['InternalSeeds']) < cluster['nb_seeds_per_dc']):
				cluster['InternalSeeds'].append(instance['instance'].private_ip_address)
				cluster['ExternalSeeds'].append(instance['instance'].ip_address)

	# seeds merge
	for i in range(len(clusters)):
		clusters[i]['Seeds'] = list(clusters[i]['InternalSeeds'])
		for j in range(len(clusters)):
			if j != i:
				clusters[i]['Seeds'] += clusters[j]['ExternalSeeds'] 
	print 
	print "Updating cassandra.yaml and DataStax agent conf on all nodes..."
	print "-"*80
	for cluster in clusters:
		for instance in cluster['instances']:
			print cluster['datacenter'] + ' : Node ' + instance['index'] + ' in ' + instance['AZ']
			key_path = os.path.join(os.path.expanduser(keysDir),'Key-'+instance['region']+'-'+instance['zone']+'.pem')
			cmd = boto.manage.cmdshell.sshclient_from_instance(instance['instance'],key_path,user_name='ubuntu')
			server.updateCassandraYaml(cmd,cluster['Seeds'],instance['instance'].ip_address,snitch)
			# Config DataStax agent
			if cluster['opsCenterServer']:
				# There is an OpsCenter Server on this DC, so we use the private ip address for agent stomp_interface
				server.updateDataStaxAgent(cmd,cluster['opsCenterServer']['privateIP'])
			else:
				# no OpsCenter server on this DC, we search an available server in other DCs
				# Priority on opsCenter server on the same Region and Zone
				bStomp = False
				for cluster2 in clusters:
					if cluster2 != cluster:
						if cluster2['opsCenterServer']:
							if (cluster2['opsCenterServer']['regionZone'] == (instance['region']+'-'+instance['zone'])):
								# same Region and Zone, we could use the private ip address for agent stomp_interface
								server.updateDataStaxAgent(cmd,cluster2['opsCenterServer']['privateIP'])
								bStomp = True
								break
				# If not found, we searcg for an OpsCenter server in another Region Zone
				if not bStomp:
					for cluster2 in clusters:
						if cluster2 != cluster:
							if cluster2['opsCenterServer']:
								# we use the public ip address for agent stomp_interface
								server.updateDataStaxAgent(cmd,cluster2['opsCenterServer']['publicIP'])
								bStomp = True
								break

	print 
	print "Starting Cassandra Seeds nodes..."
	print "-"*80
	for cluster in clusters:
		nb_seeds = 0
		for instance in cluster['instances']:
			if ((nb_seeds < cluster['nb_seeds_per_dc']) and (int(instance['index']) <= cluster['nb_seeds_per_dc'])):
				nb_seeds += 1
				print cluster['datacenter'] + ' : Node ' + instance['index'] + ' in ' + instance['AZ']
				key_path = os.path.join(os.path.expanduser(keysDir),'Key-'+instance['region']+'-'+instance['zone']+'.pem')
				cmd = boto.manage.cmdshell.sshclient_from_instance(instance['instance'],key_path,user_name='ubuntu')
				server.startCassandra(cmd)
				time.sleep(30)

	print 
	print "Starting Cassandra on all other nodes..."
	print "-"*80
	time.sleep(10)
	for cluster in clusters:
		nb_seeds = 0
		for instance in cluster['instances']:
			if ((nb_seeds < cluster['nb_seeds_per_dc']) and (int(instance['index']) <= cluster['nb_seeds_per_dc'])):
				nb_seeds += 1
				server.startDataStaxAgent(cmd)
				time.sleep(5)
			else:
				print cluster['datacenter'] + ' : Node ' + instance['index'] + ' in ' + instance['AZ']
				key_path = os.path.join(os.path.expanduser(keysDir),'Key-'+instance['region']+'-'+instance['zone']+'.pem')
				cmd = boto.manage.cmdshell.sshclient_from_instance(instance['instance'],key_path,user_name='ubuntu')
				server.startCassandra(cmd)
				server.startDataStaxAgent(cmd)
				time.sleep(5)

	if bOpsCenterExists:
		print 
		print "Starting OpsCenter..."
		print "-"*80
		time.sleep(10)
		for cluster in clusters:
			if cluster['opsCenterServer']:
				key_path = os.path.join(os.path.expanduser(keysDir),'Key-'+cluster['opsCenterServer']['regionZone']+'.pem')
				cmd = boto.manage.cmdshell.sshclient_from_instance(cluster['opsCenterServer']['instance'],key_path,user_name='ubuntu')
				server.startOpsCenter(cmd)
				print cluster['datacenter'] + ' : Node 1 in ' + cluster['opsCenterServer']['regionZone']

		print 
		print "Starting DataStax Agents..."
		print "-"*80
		time.sleep(10)
		for cluster in clusters:
			for instance in cluster['instances']:
				print cluster['datacenter'] + ' : Node ' + instance['index'] + ' in ' + instance['AZ']
				key_path = os.path.join(os.path.expanduser(keysDir),'Key-'+instance['region']+'-'+instance['zone']+'.pem')
				cmd = boto.manage.cmdshell.sshclient_from_instance(instance['instance'],key_path,user_name='ubuntu')
				server.startDataStaxAgent(cmd)
				time.sleep(5)

	print 
	print "Cassandra cluster finalization..."
	print "-"*80
	time.sleep(30)

	print 
	print "Intra-Cluster latencies..."
	print "-"*80
	for cluster in clusters:
		latency.getLatenciesRegion(cluster)

	print 
	print "Inter-Cluster latencies..."
	print "-"*80
	for i in range(len(clusters)):
		if (i == 0):
			dcs = []
			for j in range(len(clusters)):
				dcs.append(clusters[j]['datacenter'])
			latency.printLatency(clusters,dcs)
			print
		j = i+2
		while (j < len(clusters)):
			dcs = [clusters[i]['datacenter'],clusters[j]['datacenter']]
			latency.printLatency(clusters,dcs)
			print
			j += 1

	print 
	print "How to connect to "+cluster_name+"'s nodes..."
	print "-"*80
	for cluster in clusters:
		print '['+cluster['datacenter']+'] '+cluster['region']+'-'+cluster['zone']
		print "-"*80
		for instance in cluster['instances']:
			print 'Node '+instance['index']+' : ssh -i '+keysDir+'/Key-'+cluster['region']+'-'+cluster['zone']+'.pem ubuntu@'+instance['instance'].ip_address	
		print

	if bOpsCenterExists:
		# Restart OpsCenterServer
		print
		print "Connect to OpsCenter"
		print "-"*80
		for cluster in clusters:
			if cluster['opsCenterServer']:
				key_path = os.path.join(os.path.expanduser(keysDir),'Key-'+cluster['opsCenterServer']['regionZone']+'.pem')
				cmd = boto.manage.cmdshell.sshclient_from_instance(cluster['opsCenterServer']['instance'],key_path,user_name='ubuntu')
				server.restartOpsCenter(cmd)
				print 'http://'+cluster['opsCenterServer']['publicIP']+':8888'

	print
	print "Cluster status..."
	print "-"*80
	instance = clusters[0]['instances'][0]
	key_path = os.path.join(os.path.expanduser(keysDir),'Key-'+instance['region']+'-'+instance['zone']+'.pem')
	cmd = boto.manage.cmdshell.sshclient_from_instance(instance['instance'],key_path,user_name='ubuntu')
	print server.clusterStatus(cmd)
	print

	return clusters