def makeListHost(self, l): h = Host() for e in range(0, len(self.hostOrder)): h.__dict__[self.hostOrder[e]] = l[e] h.up = boolean(h.up) h.decayed = boolean(h.decayed) h.state = int(h.state) if h.reserved is not None: h.reserved = eval(h.reserved) else: h.reserved = [] return h
def __init__(self, config, client): self.config = config self.client = client self.hooks = [] self.log = logging.getLogger(__file__) self.scheduleDelay = float(self.config.get("Primitive", "scheduleDelay")) self.densePack = boolean(self.config.get("Primitive", "densePack")) self.hosts = {} # Zoni self.minServersOn = 3 self.shutdownDelay = 300 self.pcm = zoni.services.rpycservices.client("zoni", 12345).createConn() self.zoniStateFile = "/var/tmp/zoniStateFile" if os.path.exists(self.zoniStateFile): self.zoniState = self.__loadZoniState(self.zoniStateFile) else: self.zoniState = {} self.__initState() items = self.config.items("Primitive") items.sort() for item in items: (name, value) = item name = name.lower() if (name.startswith("hook")): try: self.hooks.append(instantiateImplementation(value, config, client, False)) except: self.log.exception("Failed to load hook %s" % (value))
def makeListInstance(self, l): i = Instance() for e in range(0, len(self.instanceOrder)): i.__dict__[self.instanceOrder[e]] = l[e] i.state = int(i.state) i.decayed = boolean(i.decayed) i.disks = map(lambda x: DiskConfiguration(d=x), eval(i.disks)) i.nics = map(lambda x: NetworkConfiguration(d=x), eval(i.nics)) i.hints = eval(i.hints) return i
def startClusterManager(config): global service, data dfs = instantiateImplementation(config.get("ClusterManager", "dfs"), config) data = instantiateImplementation(config.get("ClusterManager", "data"), config) service = instantiateImplementation( config.get("ClusterManager", "service"), config, data, dfs) if boolean(config.get("Security", "authAndEncrypt")): users = {} userDatabase = data.getUsers() for user in userDatabase.values(): if user.passwd != None: users[user.name] = user.passwd users[config.get('AllowedUsers', 'nodeManagerUser')] = config.get( 'AllowedUsers', 'nodeManagerPassword') users[config.get('AllowedUsers', 'agentUser')] = config.get('AllowedUsers', 'agentPassword') authenticator = TlsliteVdbAuthenticator.from_dict(users) # XXXstroucki ThreadedServer is liable to have exceptions # occur within if an endpoint is lost. t = ThreadedServer(service=rpycservices.ManagerService, hostname='0.0.0.0', port=int(config.get('ClusterManagerService', 'port')), auto_register=False, authenticator=authenticator) else: t = ThreadedServer(service=rpycservices.ManagerService, hostname='0.0.0.0', port=int(config.get('ClusterManagerService', 'port')), auto_register=False) t.logger.setLevel(logging.ERROR) t.service.service = service t.service._type = 'ClusterManagerService' debugConsole(globals()) t.start() # shouldn't exit by itself return
def start(self): oldInstances = {} muffle = {} while True: try: # Generate a list of VMs/host hosts = {} load = {} for h in self.client.getHosts(): hosts[h.id] = h load[h.id] = [] load[None] = [] _instances = self.client.getInstances() instances = {} for i in _instances: instances[i.id] = i for i in instances.itervalues(): if (i.hostId or i.state == InstanceState.Pending): load[i.hostId] = load[i.hostId] + [i.id] # Check for VMs that have exited for i in oldInstances: if (i not in instances and oldInstances[i].state != InstanceState.Pending): for hook in self.hooks: hook.postDestroy(oldInstances[i]) # Schedule new VMs oldInstances = instances if (len(load.get(None, [])) > 0): load[None].sort() for i in load[None]: inst = instances[i] try: minMax = None minMaxHost = None targetHost = inst.hints.get("targetHost", None) try: allowElsewhere = boolean(inst.hints.get("allowElsewhere", "False")) except Exception, e: allowElsewhere = False # TargetHost specified if (targetHost != None): for h in hosts.values(): if ((str(h.id) == targetHost or h.name == targetHost)): # make sure that host is up, in a normal state and is not reserved if (h.up == True and h.state == HostState.Normal and len(h.reserved) == 0): memUsage = reduce(lambda x, y: x + instances[y].memory, load[h.id], inst.memory) coreUsage = reduce(lambda x, y: x + instances[y].cores, load[h.id], inst.cores) if (memUsage <= h.memory and coreUsage <= h.cores): minMax = len(load[h.id]) minMaxHost = h # If a host machine is reserved, only allow if userid is in reserved list if ((len(h.reserved) > 0) and inst.userId in h.reserved): memUsage = reduce(lambda x, y: x + instances[y].memory, load[h.id], inst.memory) coreUsage = reduce(lambda x, y: x + instances[y].cores, load[h.id], inst.cores) if (memUsage <= h.memory and coreUsage <= h.cores): minMax = len(load[h.id]) minMaxHost = h if ((targetHost == None or allowElsewhere) and minMaxHost == None): for h in hosts.values(): if (h.up == True and h.state == HostState.Normal and len(h.reserved) == 0): if (minMax is None or (self.densePack and len(load[h.id]) > minMax) or (not self.densePack and len(load[h.id]) < minMax)): memUsage = reduce(lambda x, y: x + instances[y].memory, load[h.id], inst.memory) coreUsage = reduce(lambda x, y: x + instances[y].cores, load[h.id], inst.cores) if (memUsage <= h.memory and coreUsage <= h.cores): minMax = len(load[h.id]) minMaxHost = h if (minMaxHost): if (not inst.hints.get("__resume_source", None)): for hook in self.hooks: hook.preCreate(inst) self.log.info("Scheduling instance %s (%d mem, %d cores, %d uid) on host %s" % (inst.name, inst.memory, inst.cores, inst.userId, minMaxHost.name)) self.client.activateVm(i, minMaxHost) load[minMaxHost.id] = load[minMaxHost.id] + [i] muffle.clear() else: if (inst.name not in muffle): self.log.info("Failed to find a suitable place to schedule %s" % (inst.name)) muffle[inst.name] = True except Exception, e: if (inst.name not in muffle): self.log.exception("Failed to schedule or activate %s" % (inst.name)) muffle[inst.name] = True
def __init__(self, config, dfs, nm): VmControlInterface.__init__(self, config, dfs, nm) self.QEMU_BIN = self.config.get("Qemu", "qemuBin", default="/usr/bin/kvm") self.INFO_DIR = self.config.get("Qemu", "infoDir", default="/var/tmp/VmControlQemu/") self.POLL_DELAY = float(self.config.get("Qemu", "pollDelay", default=1)) self.migrationRetries = int( self.config.get("Qemu", "migrationRetries", default=10)) self.monitorTimeout = float( self.config.get("Qemu", "monitorTimeout", default=60)) self.migrateTimeout = float( self.config.get("Qemu", "migrateTimeout", default=300)) self.useMigrateArgument = boolean( self.config.get("Qemu", "useMigrateArgument", default=False)) self.statsInterval = float( self.config.get("Qemu", "statsInterval", default=0)) reservedMem = self.config.get("Qemu", "reservedMem", default=512) reservedMem = int(reservedMem) self.reservedMem = reservedMem self.log = logging.getLogger(__file__) self.ifPrefix = "tashi" # keep a handle to my NM service self.service = None self.controlledVMs = {} self.hostname = socket.gethostname() self.usedPorts = [] self.usedPortsLock = threading.Lock() self.vncPorts = [] self.vncPortLock = threading.Lock() self.consolePort = 10000 self.consolePortLock = threading.Lock() maxParallelMigrations = self.config.get("Qemu", "maxParallelMigrations") maxParallelMigrations = int(maxParallelMigrations) if maxParallelMigrations < 1: maxParallelMigrations = 1 self.migrationSemaphore = threading.Semaphore(maxParallelMigrations) self.stats = {} self.suspendHandler = self.config.get("Qemu", "suspendHandler", default="gzip") self.resumeHandler = self.config.get("Qemu", "resumeHandler", default="zcat") self.scratchVg = self.config.get("Qemu", "scratchVg") self.scratchDir = self.config.get("Qemu", "scratchDir", default="/tmp") try: os.mkdir(self.INFO_DIR) except: pass self.__scanInfoDir() threading.Thread(target=self.__pollVMsLoop).start() if (self.statsInterval > 0): threading.Thread(target=self.statsThread).start()
def __startVm(self, instance, source): """Universal function to start a VM -- used by instantiateVM, resumeVM, and prepReceiveVM""" # Capture __startVm Hints # CPU hints cpuModel = instance.hints.get("cpumodel") cpuString = "" if cpuModel: # clean off whitespace cpuModel = self.__stripSpace(cpuModel) cpuString = "-cpu " + cpuModel # Clock hints clockString = instance.hints.get("clock", "dynticks") # clean off whitespace clockString = self.__stripSpace(clockString) # Disk hints # XXXstroucki: insert commentary on jcipar's performance # measurements # virtio is recommended, but linux will name devices # vdX instead of sdX. This adds a trap for someone who # converts a physical machine or other virtualization # layer's image to run under Tashi. diskInterface = instance.hints.get("diskInterface", "ide") # clean off whitespace diskInterface = self.__stripSpace(diskInterface) cachePolicy = instance.hints.get("cache", "off") cachePolicy = self.__stripSpace(cachePolicy) diskString = "" for index in range(0, len(instance.disks)): disk = instance.disks[index] uri = scrubString(disk.uri) imageLocal = self.dfs.getLocalHandle("images/" + uri) imageLocal = self.__dereferenceLink(imageLocal) thisDiskList = ["file=%s" % imageLocal] thisDiskList.append("if=%s" % diskInterface) thisDiskList.append("index=%d" % index) if (index == 0 and diskInterface == "virtio"): thisDiskList.append("boot=on") if (disk.persistent): snapshot = "off" migrate = "off" else: snapshot = "on" migrate = "on" thisDiskList.append("cache=%s" % cachePolicy) thisDiskList.append("snapshot=%s" % snapshot) if (self.useMigrateArgument): thisDiskList.append("migrate=%s" % migrate) diskString = diskString + "-drive " + ",".join(thisDiskList) + " " # scratch disk scratchSize = instance.hints.get("scratchSpace", "0") scratchSize = int(scratchSize) scratchName = None try: if scratchSize > 0: if self.scratchVg is None: raise Exception, "No scratch volume group defined" # create scratch disk # XXXstroucki: needs to be cleaned somewhere # XXXstroucki: clean user provided instance name scratchName = "lv%s" % instance.name # XXXstroucki hold lock # XXXstroucki check for capacity cmd = "/sbin/lvcreate --quiet -n%s -L %dG %s" % ( scratchName, scratchSize, self.scratchVg) # XXXstroucki check result __result = subprocess.Popen(cmd.split(), executable=cmd.split()[0], stdout=subprocess.PIPE).wait() index += 1 thisDiskList = [ "file=/dev/%s/%s" % (self.scratchVg, scratchName) ] thisDiskList.append("if=%s" % diskInterface) thisDiskList.append("index=%d" % index) thisDiskList.append("cache=%s" % cachePolicy) # XXXstroucki force scratch disk to be # persistent if (True or disk.persistent): snapshot = "off" migrate = "off" else: snapshot = "on" migrate = "on" thisDiskList.append("snapshot=%s" % snapshot) if (self.useMigrateArgument): thisDiskList.append("migrate=%s" % migrate) diskString = "%s-drive %s " % (diskString, ",".join(thisDiskList)) except: self.log.exception('caught exception in scratch disk formation') raise # Nic hints nicModel = instance.hints.get("nicModel", "virtio") # clean off whitespace nicModel = self.__stripSpace(nicModel) nicString = "" nicNetworks = {} for i in range(0, len(instance.nics)): # Don't allow more than one interface per vlan nic = instance.nics[i] if nicNetworks.has_key(nic.network): continue nicNetworks[nic.network] = True nicString = nicString + "-net nic,macaddr=%s,model=%s,vlan=%d -net tap,ifname=%s%d.%d,vlan=%d,script=/etc/qemu-ifup.%d " % ( nic.mac, nicModel, nic.network, self.ifPrefix, instance.id, i, nic.network, nic.network) # ACPI if (boolean(instance.hints.get("noAcpi", False))): noAcpiString = "-no-acpi" else: noAcpiString = "" # Construct the qemu command strCmd = "%s %s %s -clock %s %s %s -m %d -smp %d -serial null -vnc none -monitor pty -balloon virtio" % ( self.QEMU_BIN, noAcpiString, cpuString, clockString, diskString, nicString, instance.memory, instance.cores) if (source): strCmd = '%s -incoming "%s"' % (strCmd, source) # XXXstroucki perhaps we're doing it backwards cmd = shlex.split(strCmd) self.log.info("Executing command: %s" % (strCmd)) (pipe_r, pipe_w) = os.pipe() pid = os.fork() if (pid == 0): # child process pid = os.getpid() os.setpgid(pid, pid) os.close(pipe_r) os.dup2(pipe_w, sys.stderr.fileno()) for i in [sys.stdin.fileno(), sys.stdout.fileno()]: try: os.close(i) except: pass for i in xrange(3, os.sysconf("SC_OPEN_MAX")): try: os.close(i) except: pass # XXXstroucki unfortunately no kvm option yet # to direct COW differences elsewhere, so change # this process' TMPDIR, which kvm will honour os.environ['TMPDIR'] = self.scratchDir os.execl(self.QEMU_BIN, *cmd) sys.exit(-1) # parent process os.close(pipe_w) # enforce the new instance to have our hostId! # otherwise, a migrated VM will have its previous hostId. instance.hostId = self.service.id child = self.anonClass(pid=pid, instance=instance, stderr=os.fdopen(pipe_r, 'r'), migratingOut=False, monitorHistory=[], errorBit=True, OSchild=True) child.ptyFile = None child.vncPort = -1 child.instance.vmId = child.pid # XXXstroucki what about our hostId? # we need to make sure we don't report up a VM # with an inaccurate hostId. # Add a token to this new child object so that # we don't mistakenly clean up when matchHostPids # runs and the child process hasn't exec'ed yet. child.startTime = time.time() self.__saveChildInfo(child) self.log.info("Adding vmId %d" % (child.pid)) self.controlledVMs[child.pid] = child return (child.pid, cmd)