Example #1
0
    def place_job(self, host, cpu, mem, io):
        node_dict = {}
        node_dict = GetNodeDict()
        print " DECISION MAKER "
        for key, value in node_dict.iteritems():
            print key, value.max_cpu, value.max_memory, value.max_io, value.avail_cpu, value.avail_memory, value.avail_io
        print " DECISION MAKER END "
        for key, value in node_dict.iteritems():
            if (key == host):
                print "Test Host" + str(value.hostname)
                print value
                print "avail CPU : " + str(value.avail_cpu)
                print ''
                pickleAddOrUpdateDictionary(
                    value.hostname, str(value.ip_address),
                    float(value.max_cpu), float(value.max_memory),
                    float(value.max_io),
                    float(value.avail_cpu) - float(cpu),
                    float(value.avail_memory) - float(mem),
                    float(value.avail_io) - float(io))
                #value.avail_cpu= int(value.avail_cpu) - int(cpu)
                #value.avail_memory = int(value.avail_memory) - int(mem)
                #value.avail_io = int(value.avail_io) -  int(io)

                #Code to update the dictionary again
                #with open('node_dict.pkl','w') as node_pickle_out:
                #	pickle.dump(node_dict,node_pickle_out)
                return value.hostname
        return None
Example #2
0
	def is_space_available_for_vm(self, cpu, mem, io):
        	node_dict={}
        	node_dict=GetNodeDict()
        	for key, value in node_dict.iteritems() :
            		if ( float(cpu) <= float(value.avail_cpu) and float(	mem) <= float(value.avail_memory) and float(io) <= float(value.avail_io)) :
                		return value.hostname
        	return None
Example #3
0
def setupAll():
	
	cmd= 'python ../setup/bridge_setup.py'
	uuid = subprocess.check_output(cmd, shell=True, stderr=subprocess.PIPE)
	username = '******'
	password = '******'
	
	cmd= '../setup/SSHPasswordless.sh -h '+'localhost'+' -u '+username+' -p '+password
	uuid = subprocess.check_output(cmd, shell=True, stderr=subprocess.PIPE)
	cmd= 'cp -f ../setup/SSHPasswordless.sh /root/'
	uuid = subprocess.check_output(cmd, shell=True, stderr=subprocess.PIPE)
	
	cmd= '../nfs_client_setup.sh'
	uuid = subprocess.check_output(cmd, shell=True, stderr=subprocess.PIPE)

	node_dict=GetNodeDict('../framework/node_dict.pkl')
	all_address = []
	for key, value in node_dict.iteritems() :
		all_address.append(key)
	for node in all_address:
		cmd= '../setup/SSHPasswordless.sh -h '+node+' -u '+username+' -p '+password
		uuid = subprocess.check_output(cmd, shell=True, stderr=subprocess.PIPE)
		cmd= 'scp ../setup/SSHPasswordless.sh root@'+node+':/root/'
		uuid = subprocess.check_output(cmd, shell=True, stderr=subprocess.PIPE)
		cmd= 'scp ../setup/ifcfg* root@'+node+':/root/'
		uuid = subprocess.check_output(cmd, shell=True, stderr=subprocess.PIPE)
		cmd= 'scp ../setup/bridge_setup.py root@'+node+':/root/'
		uuid = subprocess.check_output(cmd, shell=True, stderr=subprocess.PIPE)
	for n in itertools.permutations(all_address, 2):
		cmd= 'ssh root@'+n[0]+' \"/root/SSHPasswordless.sh -h '+n[0]+' -u '+username+' -p '+password+'\"'
		uuid = subprocess.check_output(cmd, shell=True, stderr=subprocess.PIPE)
Example #4
0
 def is_cpu_available_on_host(self, host, cpu):
     node_dict = {}
     node_dict = GetNodeDict()
     for key, value in node_dict.iteritems():
         if key == host:
             if (float(cpu) <= float(value.avail_cpu)):
                 return True
     return False
Example #5
0
	def is_cpu_available_on_host(self, host, cpu):
        	node_dict={}
        	node_dict=GetNodeDict()
        	for key, value in node_dict.iteritems() :
			if key == host:
            			if ( float(cpu) <= float(value.avail_cpu) ):
					return True
        	return False
Example #6
0
 def is_mem_available_on_host(self, host, memory):
     node_dict = {}
     node_dict = GetNodeDict()
     for key, value in node_dict.iteritems():
         if key == host:
             if (float(memory) < float(value.avail_memory)):
                 return True
     return False
Example #7
0
	def is_mem_available_on_host(self, host, memory):
        	node_dict={}
        	node_dict=GetNodeDict()
        	for key, value in node_dict.iteritems() :
			if key == host:
           			if ( float(memory) < float(value.avail_memory) ):
           				return True
        	return False
Example #8
0
 def is_space_available_for_vm(self, cpu, mem, io):
     node_dict = {}
     node_dict = GetNodeDict()
     for key, value in node_dict.iteritems():
         if (float(cpu) <= float(value.avail_cpu)
                 and float(mem) <= float(value.avail_memory)
                 and float(io) <= float(value.avail_io)):
             return value.hostname
     return None
def get_overall_pm_utilization(host):
    node_dict = GetNodeDict()
    for key, value in node_dict.iteritems() :
	if key == host:
	    if max(value.avail_cpu, value.avail_memory, value.avail_io)> HIGH:
		print 'HIGH'
	    elif max(value.avail_cpu, value.avail_memory, value.avail_io)> MEDIUM:
		print 'MEDIUM'
	    elif max(value.avail_cpu, value.avail_memory, value.avail_io)> LOW:
		print 'LOW'


                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             def start_server():
Example #10
0
def do_prereq_start_workload_host(hostname):
    try:
        node_dict = GetNodeDict()
        ip = node_dict[hostname].ip_address
        #print "IP RUN "+str(ip)

        update_host_in_config(hostname)

        # copy host configuration file to guest
        scpHostConfig = 'scp -q -o StrictHostKeyChecking=no /var/lib/virtdc/vmonere/hostinfo/' + hostname + '.txt root@' + ip + ':' + host_file_path
        subprocess.Popen(scpHostConfig, shell=True, stderr=subprocess.PIPE)

        scp_host_sender = 'scp -q -o StrictHostKeyChecking=no /var/lib/virtdc/vmonere/host/vmonere_host_sender.socket root@' + ip + ':' + host_sender_path + 'vmonere_host_sender.socket'
        print scp_host_sender
        subprocess.Popen(scp_host_sender, shell=True, stderr=subprocess.PIPE)

        scp_host_vmonere_util = 'scp -q -o StrictHostKeyChecking=no /var/lib/virtdc/vmonere/host/vmonere_utility.py root@' + ip + ':' + host_sender_path + 'vmonere_utility.py'
        print scp_host_vmonere_util
        subprocess.Popen(scp_host_vmonere_util,
                         shell=True,
                         stderr=subprocess.PIPE)

        scp_vmonere_timeout = 'scp -q -o StrictHostKeyChecking=no /var/lib/virtdc/vmonere/host/timeout.py root@' + ip + ':' + host_sender_path + 'timeout.py'
        print scp_vmonere_timeout
        subprocess.Popen(scp_vmonere_timeout,
                         shell=True,
                         stderr=subprocess.PIPE)

        start_monitor = 'ssh -n -q -o StrictHostKeyChecking=no root@' + ip + ' \"/bin/nohup sh -c \'/bin/python /var/lib/virtdc/vmonere/host/vmonere_host_sender.socket\' > /dev/null 2>&1 &' + '\"'
        print start_monitor
        subprocess.Popen(start_monitor, shell=True, stderr=subprocess.PIPE)

    except subprocess.CalledProcessError as e:
        print "error>", e.output, '<'
Example #11
0
def addOrUpdateDictionaryOfVM(hostName, vmid, guest):
    # code to add the dictionary elements
    host_vm_dict = getHostVMDict()
    node_dict = GetNodeDict()

    for key, value in node_dict.iteritems():
        if key not in host_vm_dict:
            host_vm_dict[key] = {}
            # For Testing
            # host_vm_dict[key]={"vm1":Guest("192.168.1.14","vm1", float(1), float(3),float(42424),float(424242),float(1))}
    if guest == None:
        del host_vm_dict[hostName][vmid]
        pickleNodeVMDictionary(host_vm_dict)
    else:
        host_vm_dict[hostName][vmid] = guest
        pickleNodeVMDictionary(host_vm_dict)
    print host_vm_dict
def show_host_info(host_name):
	node_dict = GetNodeDict()
	for node, value in node_dict.iteritems() :
		if node == host_name:
			value = node_dict[host_name]
			print '%s%s' %(str('Host Name:').ljust(30),host_name)
			print '--------------------------------------------'
			print '%s%s' %(str('Host IP:').ljust(30),value.ip_address)
			print '%s%s' %(str('Available CPU [core]:').ljust(30),value.avail_cpu)
			print '%s%s' %(str('Maximum CPU [core]:').ljust(30),value.max_cpu)
			print '%s%s' %(str('Available Memory [KiB]:').ljust(30),value.avail_memory)
			print '%s%s' %(str('Maximum Memory [KiB]:').ljust(30),value.max_memory)
			print '%s%s' %(str('Available Disk Space [KiB]:').ljust(30),value.avail_io)
			print '%s%s' %(str('Maximum Disk Space [KiB]:').ljust(30),value.max_io)
			#print '--------------------------------------------'
			return True
	return False
def list_host_domain_information():
	node_dict = GetNodeDict()
	print '----------------------------------------------------------'
	print '%s%s%s%s' %(str('Host Name').ljust(12), str(' Id').ljust(7),str('Domain Name').ljust(32),'Status')
	print '----------------------------------------------------------'
	for node, value in node_dict.iteritems() :
	    #print 'Host Name : '+str(node)
	    host = " "+str(node)+"     "
            #print '-------------------------------------------------'
            list_cmd="virsh --connect qemu+ssh://host_node/system list | tail -n +3 |  sed '/^$/d'  | sed \'s/^/ "+host+"/\'"
	    list_cmd = list_cmd.replace("host_node", node.strip());
	    #print list_cmd
	    #print list_cmd
	    running_domains = subprocess.check_output(list_cmd, shell=True, stderr=subprocess.PIPE)
	    #print running_domains
	    running_domains = "  "+str(running_domains.strip())
	    print(running_domains.decode(sys.stdout.encoding))
            list_cmd="virsh --connect qemu+ssh://host_node/system list | tail -n +3 |  sed '/^$/d'  | sed \'s/^/ "+host+"/\'"
Example #14
0
def addOrUpdateDictionaryOfVM(hostName, vmid, guest):
    #code to add the dictionary elements
    host_vm_dict = getHostVMDict()
    node_dict = GetNodeDict()

    for key, value in node_dict.iteritems():
        if key not in host_vm_dict:
            host_vm_dict[key] = {}

#For Testing
            #host_vm_dict[key]={"vm1":Guest("192.168.1.14","vm1", float(1), float(3),float(42424),float(424242),float(1))}
    if (guest == None):
        del host_vm_dict[hostName][vmid]
        pickleNodeVMDictionary(host_vm_dict)
    else:
        host_vm_dict[hostName][vmid] = guest
        pickleNodeVMDictionary(host_vm_dict)
    print host_vm_dict
def list_host_domain_information():
    node_dict = GetNodeDict()
    print '----------------------------------------------------------'
    print '%s%s%s%s' % (str('Host Name').ljust(12), str(' Id').ljust(7),
                        str('Domain Name').ljust(32), 'Status')
    print '----------------------------------------------------------'
    for node, value in node_dict.iteritems():
        #print 'Host Name : '+str(node)
        host = " " + str(node) + "     "
        #print '-------------------------------------------------'
        list_cmd = "virsh --connect qemu+ssh://host_node/system list | tail -n +3 |  sed '/^$/d'  | sed \'s/^/ " + host + "/\'"
        list_cmd = list_cmd.replace("host_node", node.strip())
        #print list_cmd
        #print list_cmd
        running_domains = subprocess.check_output(list_cmd,
                                                  shell=True,
                                                  stderr=subprocess.PIPE)
        #print running_domains
        running_domains = "  " + str(running_domains.strip())
        print(running_domains.decode(sys.stdout.encoding))
        list_cmd = "virsh --connect qemu+ssh://host_node/system list | tail -n +3 |  sed '/^$/d'  | sed \'s/^/ " + host + "/\'"
def show_host_info(host_name):
    node_dict = GetNodeDict()
    for node, value in node_dict.iteritems():
        if node == host_name:
            value = node_dict[host_name]
            print '%s%s' % (str('Host Name:').ljust(30), host_name)
            print '--------------------------------------------'
            print '%s%s' % (str('Host IP:').ljust(30), value.ip_address)
            print '%s%s' % (str('Available CPU [core]:').ljust(30),
                            value.avail_cpu)
            print '%s%s' % (str('Maximum CPU [core]:').ljust(30),
                            value.max_cpu)
            print '%s%s' % (str('Available Memory [KiB]:').ljust(30),
                            value.avail_memory)
            print '%s%s' % (str('Maximum Memory [KiB]:').ljust(30),
                            value.max_memory)
            print '%s%s' % (str('Available Disk Space [KiB]:').ljust(30),
                            value.avail_io)
            print '%s%s' % (str('Maximum Disk Space [KiB]:').ljust(30),
                            value.max_io)
            #print '--------------------------------------------'
            return True
    return False
Example #17
0
	def place_job(self, host, cpu, mem, io) :
		node_dict={}
		node_dict=GetNodeDict()	
		print " DECISION MAKER "
		for key, value in node_dict.iteritems() :
    			print key, value.max_cpu, value.max_memory, value.max_io, value.avail_cpu, value.avail_memory, value.avail_io
		print " DECISION MAKER END "
		for key, value in node_dict.iteritems() :
			if ( key == host ):
				print "Test Host"+str(value.hostname)
				print value
				print "avail CPU : "+str(value.avail_cpu)
				print ''
				pickleAddOrUpdateDictionary(value.hostname, str(value.ip_address), float(value.max_cpu), float(value.max_memory), float(value.max_io), float(value.avail_cpu) - float(cpu), float(value.avail_memory) - float(mem), float(value.avail_io) -  float(io))			
				#value.avail_cpu= int(value.avail_cpu) - int(cpu)
				#value.avail_memory = int(value.avail_memory) - int(mem)
				#value.avail_io = int(value.avail_io) -  int(io)

				#Code to update the dictionary again
				#with open('node_dict.pkl','w') as node_pickle_out:
    				#	pickle.dump(node_dict,node_pickle_out)
				return value.hostname
		return None
def start_host_monitor():
    master_node = 'node1'
    node_dict = GetNodeDict()
    for node in node_dict:
        if node == master_node:
            start_master_monitor_cmd = "/usr/bin/python /var/lib/virtdc/vmonere/host/vmonere_host_sender.socket &"
            print 'Test Master'
            update_host_in_config(node)
            # copy host configuration file to guest
            cpHostConfig = 'cp /var/lib/virtdc/vmonere/hostinfo/' + node + '.txt ' + host_file_path
            print cpHostConfig
            subprocess.Popen(cpHostConfig, shell=True, stderr=subprocess.PIPE)
            print start_master_monitor_cmd
            start_master_monitor = subprocess.Popen(start_master_monitor_cmd,
                                                    shell=True,
                                                    stderr=subprocess.PIPE)
        else:
            do_prereq_start_workload_host(node)
Example #19
0
def process_action_on_current_usage(host, vmid, value, cpu_usage, mem_usage,
                                    io_usage):
    node_dict = GetNodeDict()

    #for key, value in node_dict.iteritems() :
    #print key, value.hostname, value.ip_address, value.max_cpu, value.max_memory, value.max_io, value.avail_cpu, value.avail_memory, value.avail_io

    #Log activity
    manager_activity_log = open(
        '/var/lib/virtdc/logs/activity_logs/manager.log', 'a+')
    manager_activity_log.write(
        str(datetime.datetime.now()) + '::PLACEMENT MANAGER::MEMORY::' + host +
        ' :: ' + vmid + ' :: Alotted Memory ' + str(value.current_memory) +
        ' :: Current Memory ' + str(mem_usage) + '\n')
    manager_activity_log.write(
        str(datetime.datetime.now()) + '::PLACEMENT MANAGER::CPU::' + host +
        ' :: ' + vmid + ' :: Alotted CPU ' + str(value.current_cpu) +
        ' :: Current CPU ' + str(cpu_usage) + '\n')

    obj = NodeFinder()
    max_cpu = value.max_cpu

    #print 'Max CPU '+str(max_cpu)
    allotted_cpu = float(value.current_cpu)
    #print 'Allocate CPU : '+str(allotted_cpu)

    #Base OS should not go below the minimum memory
    mem_usage = float(mem_usage) + float(_base_mem_size)

    allotted_memory = float(value.current_memory)
    max_memory = float(value.max_memory)

    #Check CPU usage, regarding a 0.1 margin as eligible to scale up
    if ((cpu_usage + 0.1 > allotted_cpu) and (cpu_usage < max_cpu)):
        if (obj.is_cpu_available_on_host(host, 1)):
            #print 'Test 2'
            new_cpu_value = value.current_cpu + 1
            print "New CPU: %s" % new_cpu_value
            vm_cpu_scaling(host, vmid, value.vmip, new_cpu_value)
            manager_activity_log.write(
                str(datetime.datetime.now()) +
                '::PLACEMENT MANAGER::CPU::Scaling ::' + host + ' :: ' + vmid +
                ' :: Memory scaled from ' + str(value.current_cpu) + ' to ' +
                str(cpu_usage) + '\n')
        else:
            print 'Test 3'
            new_host = obj.is_space_available_for_vm(cpu_usage, mem_usage,
                                                     io_usage)
            if new_host is None:
                print "Cant migrate guest"
            else:
                print 'Dest Node : ' + new_host
                #Initiate vm migration
                migrate_flag = vm_migrate_guest(host, new_host, vmid)
                if (migrate_flag):
                    manager_activity_log.write(
                        str(datetime.datetime.now()) +
                        '::PLACEMENT MANAGER::CPU::Migration ::' + host +
                        ' :: ' + vmid + ' :: Domain migrated from ' +
                        str(host) + ' to ' + str(new_host) +
                        ' for CPU Scaling from' + str(value.current_cpu) +
                        ' to ' + str(cpu_usage) + '\n')
                else:
                    manager_activity_log.write(
                        str(datetime.datetime.now()) +
                        '::PLACEMENT MANAGER::CPU::Migration ::' + host +
                        ' :: ' + vmid + ' :: Cannot migrate from ' +
                        str(host) + ' to ' + str(new_host) +
                        ' for CPU Scaling from' + str(value.current_cpu) +
                        ' to ' + str(cpu_usage) + '\n')

    #if(	(cpu_usage>current_cpu) and 	(cpu_usage<max_cpu)	):  -- CPU scaling down is not implemented
    #Check Memory Usage - Memory scale up will be initiated when usage is greater than usage+scaleup_threshold
    #print "MEM USAGE: " + str(mem_usage)
    #print "Alloc mem: " + str(allotted_memory)
    #print "threadhold: " + str(float(mem_scale_up_threshold))
    #print "Max mem: " + str(max_memory)
    if (((mem_usage > (allotted_memory + float(mem_scale_up_threshold))))
            and (mem_usage < max_memory)):
        required_extra_memory = mem_usage - allotted_memory
        print "Required Mem: " + str(required_extra_memory)
        if (obj.is_mem_available_on_host(host, required_extra_memory)):
            vm_memory_scaling(host, vmid, float(mem_usage))
            manager_activity_log.write(
                str(datetime.datetime.now()) +
                '::PLACEMENT MANAGER::MEMORY::Scaling ::' + str(host) +
                ' :: ' + str(vmid) + ' :: Memory scaled from ' +
                str(allotted_memory) + ' to ' + str(mem_usage) + '\n')
        else:
            new_host = obj.is_space_available_for_vm(cpu_usage, mem_usage,
                                                     io_usage)
            if new_host is None:
                print "Cant migrate Guest"
            else:
                #Initiate vm migration
                migrate_flag = vm_migrate_guest(host, new_host, vmid)
                if (migrate_flag):
                    manager_activity_log.write(
                        str(datetime.datetime.now()) +
                        '::PLACEMENT MANAGER::MEMORY::Migration ::' + host +
                        ' :: ' + vmid + ' :: Domain migrated from ' +
                        str(host) + ' to ' + str(new_host) +
                        ' for Memory Scaling from' +
                        str(value.current_memory) + ' to ' + str(mem_usage) +
                        '\n')
                else:
                    manager_activity_log.write(
                        str(datetime.datetime.now()) +
                        '::PLACEMENT MANAGER::MEMORY::Migration ::' + host +
                        ' :: ' + vmid + ' :: Cannot migrate from ' +
                        str(host) + ' to ' + str(new_host) +
                        ' for Memory Scaling from' +
                        str(value.current_memory) + ' to ' + str(mem_usage) +
                        '\n')

    #To scale down memory - Memory scale down will be initiated when usage is lower than usage-scaledown_threshold or
    elif ((mem_usage < (allotted_memory - float(mem_scale_down_threshold)))
          and (mem_usage < max_memory)):
        vm_memory_scaling(host, vmid, float(mem_usage))
        manager_activity_log.write(
            str(datetime.datetime.now()) +
            '::PLACEMENT MANAGER::MEMORY::Scaling ::' + str(host) + ' :: ' +
            str(vmid) + ' :: Memory scaled from ' + str(allotted_memory) +
            ' to ' + str(mem_usage) + '\n')