def fetchData(self, inputMomentum, injectVariance=False): """ Fetches measurement data from the server in the form of a VMMeasurement instance. @param inputMomentum: The measurement momentum. If None - no momentum will be considered. @param injectVariance: Whether to "inject" a change in the workload pattern. Used for experiment purposes. @return: measurement data from the server in the form of a VMMeasurement instance. May return None if no data is available. """ data = filterEmptyStrings( map(extractVal, self.fetchRawData().split(";"))) if (data and not AppServer._isHeader(data) and len(data) == 10): assert len( data ) == 10, "Measurement data %s does not have proper size" % (data) log.debug("Measurement %s: %s", self.readableName, data) varianceActiveMem = data[6] + convertMem( 1, fromCode="GB", toCode="KB") + data[9] * convertMem( 1, fromCode="MB", toCode="KB") varianceIdlePerc = data[4] * 0.9 measurement = VMMeasurement( readableName=self._measName(), vmAddress=self.address, serverTime=data[0], cpuCapacityMhz=data[1], cpuIOWaitPerc=data[2], cpuStealPerc=data[3], cpuIdlePerc=data[4] if not injectVariance else varianceIdlePerc, ramInKb=data[5], activeMemInKb=data[6] if not injectVariance else varianceActiveMem, diskUtilPerc=data[7], nicUtilPerc=data[8], numUsers=data[9]) measurement.considerMomentum(self.lastMeasurement, inputMomentum) self.lastMeasurement = measurement self.vmType.addMeasurement(measurement) self.htm.train(measurement) self._line(measurement) return measurement else: return None
def fetchData(self, inputMomentum, injectVariance = False): """ Fetches measurement data from the server in the form of a VMMeasurement instance. @param inputMomentum: The measurement momentum. If None - no momentum will be considered. @param injectVariance: Whether to "inject" a change in the workload pattern. Used for experiment purposes. @return: measurement data from the server in the form of a VMMeasurement instance. May return None if no data is available. """ data = filterEmptyStrings( map(extractVal, self.fetchRawData().split(";")) ) if(data and not AppServer._isHeader(data) and len(data) == 10): assert len(data) == 10, "Measurement data %s does not have proper size" % (data) log.debug("Measurement %s: %s", self.readableName, data) varianceActiveMem = data[6] + convertMem(1, fromCode="GB", toCode="KB") + data[9] * convertMem(1, fromCode="MB", toCode="KB") varianceIdlePerc = data[4] * 0.9 measurement = VMMeasurement(readableName = self._measName(), vmAddress=self.address, serverTime = data[0], cpuCapacityMhz = data[1], cpuIOWaitPerc = data[2], cpuStealPerc = data[3], cpuIdlePerc = data[4] if not injectVariance else varianceIdlePerc, ramInKb = data[5], activeMemInKb = data[6] if not injectVariance else varianceActiveMem, diskUtilPerc = data[7], nicUtilPerc = data[8], numUsers = data[9]) measurement.considerMomentum(self.lastMeasurement, inputMomentum) self.lastMeasurement = measurement self.vmType.addMeasurement(measurement) self.htm.train(measurement) self._line(measurement) return measurement else: return None
class VMMeasurement(BaseAutoscalingClass): """Performance measurements taken from a VM.""" freqMaxInMhz = 3500 maxNumCores = 8 maxCpuCapacity = freqMaxInMhz * maxNumCores maxRAMInKb = convertMem(8, fromCode="GB", toCode="KB") def __init__(self, readableName, vmAddress, serverTime, cpuCapacityMhz, cpuIOWaitPerc, cpuStealPerc, cpuIdlePerc, ramInKb, activeMemInKb, diskUtilPerc, nicUtilPerc, numUsers): """ Constr. @param readableName: see superclass. @param vmAddress: the address of the VM. Must not be None. @param serverTime: the time of the measurement on the VM. Must not be None. @param cpuCapacityMhz: The CPU capacity in Mhz. Must not be None. Must be positive. @param cpuIOWaitPerc: Percentage of CPU I/O waiting. Must not be None. Must be non-negative. @param cpuStealPerc: Percentage of stolen CPU time by the hypervisor. Must not be None. Must be non-negative. @param cpuIdlePerc: Percentage of CPU idle time. Must not be None. Must be non-negative. @param ramInKb: RAM capacity in Kb. Must not be None. Must be positive. @param activeMemInKb: Actively used RAM in Kb. Must not be None. Must be positive and less than the capacity. @param diskUtilPerc: Percentage of disk utilisation. Must not be None. @param nicUtilPerc: Percentage of NIC utilisation. Must not be None. @param numUsers: Number of users served on the system. Must not be None. Must be non-negative. """ super(VMMeasurement, self).__init__(readableName=readableName) assert 0 <= cpuCapacityMhz <= VMMeasurement.maxCpuCapacity, "CPU capacity {0} is not in the range[{1}, {2}]".format( cpuCapacityMhz, 0, VMMeasurement.maxCpuCapacity) assert 0 <= cpuIOWaitPerc <= 100, "cpuIOWait% {0} is not in the range[{1}, {2}]".format( cpuIOWaitPerc, 0, 100) assert 0 <= cpuStealPerc <= 100, "cpuSteal% {0} is not in the range[{1}, {2}]".format( cpuStealPerc, 0, 100) assert 0 <= cpuIdlePerc <= 100, "cpuIdle% {0} is not in the range[{1}, {2}]".format( cpuIdlePerc, 0, 100) assert cpuIOWaitPerc + cpuStealPerc + cpuIdlePerc <= 100, "cpuIOWait% + cpuSteal + cpuIdle% ({0}) is greater than 100".format( cpuIOWaitPerc + cpuStealPerc + cpuIdlePerc) assert 0 <= ramInKb <= VMMeasurement.maxRAMInKb, "RAM capacity {0} is not in the range[{1}, {2}]".format( ramInKb, 0, VMMeasurement.maxRAMInKb) if activeMemInKb > ramInKb: activeMemInKb = ramInKb assert 0 <= activeMemInKb <= ramInKb, "Active RAM {0} is not in the range[{1}, {2}]".format( activeMemInKb, 0, ramInKb) assert 0 <= diskUtilPerc <= 100, "diskUtil% {0} is not in the range[{1}, {2}]".format( diskUtilPerc, 0, 100) assert 0 <= nicUtilPerc <= 100, "nicUtil% {0} is not in the range[{1}, {2}]".format( nicUtilPerc, 0, 100) assert 0 <= numUsers, "numUsers {0} is not positive".format(numUsers) self.timestamp = datetime.datetime.now() self.vmAddress = vmAddress self.serverTime = serverTime self.cpuCapacityMhz = cpuCapacityMhz self.cpuIOWaitPerc = cpuIOWaitPerc self.cpuStealPerc = cpuStealPerc self.cpuIdlePerc = cpuIdlePerc self.ramInKb = ramInKb self.activeMemInKb = activeMemInKb self.diskUtilPerc = diskUtilPerc self.nicUtilPerc = nicUtilPerc self.numUsers = numUsers self.anomaly = 0 self.prevNormalisedCpuUtil = None self.prevNormalisedRAMUtil = None self.prevNormalisedNICUtil = None self.prevNormalisedDiskUtil = None self.prevNumUsers = 0 self.prevCpuUtil = None self.prevRamUtil = None self.prevAnomaly = None self.prevNormCpus = collections.deque(maxlen=3) self.prevUsers = collections.deque(maxlen=3) self.momentum = None def normaliseCpuCapacity(self): """ Returns the normalised CPU capacity of the VM in the range [0,1]. Considers momentum if set. @return: The normalised CPU capacity. """ return VMMeasurement._assertRatio( (100.0 - self.cpuStealPerc) * self.cpuCapacityMhz / (100 * VMMeasurement.maxCpuCapacity)) def normaliseRAMCapacity(self): """ Returns the normalised RAM capacity of the VM in the range [0,1]. Considers momentum if set. @return: The normalised RAM capacity. """ return VMMeasurement._assertRatio( float(self.ramInKb) / VMMeasurement.maxRAMInKb) def normaliseCpuUtil(self): """ Returns the normalised CPU utilisation of the VM in the range [0,1]. Considers momentum if set. @return: The normalised CPU utilisation. """ result = (100.0 - self.cpuStealPerc - self.cpuIdlePerc) * self.cpuCapacityMhz / ( 100 * VMMeasurement.maxCpuCapacity) return VMMeasurement._assertRatio( self._momentum(result, self.prevNormalisedCpuUtil)) def cpuUtil(self): """ Returns the CPU utilisation of the VM in the range [0,1]. Considers momentum if set. @return: The CPU utilisation. """ result = (100.0 - self.cpuStealPerc - self.cpuIdlePerc) / (100.0 - self.cpuStealPerc) return VMMeasurement._assertRatio( self._momentum(result, self.prevCpuUtil)) def normaliseRAMUtil(self): """ Returns the normalised RAM utilisation of the VM in the range [0,1]. Considers momentum if set. @return: The normalised RAM utilisation. """ result = float(self.activeMemInKb) / VMMeasurement.maxRAMInKb return VMMeasurement._assertRatio( self._momentum(result, self.prevNormalisedRAMUtil)) def ramUtil(self): """ Returns the RAM utilisation of the VM in the range [0,1]. Considers momentum if set. @return: The RAM utilisation. """ result = self.activeMemInKb / float(self.ramInKb) return VMMeasurement._assertRatio( self._momentum(result, self.prevRamUtil)) def getAnomaly(self): """ Returns the anomaly score of this VM measurment in the range [0,1]. Considers momentum if set. @return: The anomaly score of this measurement. """ return VMMeasurement._assertRatio( self._momentum(self.anomaly, self.prevAnomaly)) def normaliseNICUtil(self): """ Returns the normalised NIC utilisation of the VM in the range [0,1]. Considers momentum if set. @return: The normalised NIC utilisation. """ result = self.nicUtilPerc / 100.0 return VMMeasurement._assertRatio( self._momentum(result, self.prevNormalisedNICUtil)) def normaliseDiskUtil(self): """ Returns the normalised disk utilisation of the VM in the range [0,1]. Considers momentum if set. @return: The normalised disk utilisation. """ result = self.diskUtilPerc / 100.0 return VMMeasurement._assertRatio( self._momentum(result, self.prevNormalisedDiskUtil)) def numberOfUsers(self): """ Returns the number of users served in the VM. @return: The number of users served in the VM. """ result = self.numUsers return int(self._momentum(result, self.prevNumUsers)) def _momentum(self, value, prevValue): if self.momentum is not None: return (1 - self.momentum) * value + self.momentum * prevValue else: return value def considerMomentum(self, prevMeasurement, inputMomentum): """ Modifies this measurment in the light of the previous measurement by using the specified momentum. @param prevMeasurement: the previous measurement. If None - no momentum is considered. An instance of VMMeasurement. @param inputMomentum: the momentum. If None - no momentum is considered. If not None must in the range [0,1). """ if prevMeasurement is not None and inputMomentum is not None: assert isinstance( prevMeasurement, VMMeasurement), "Prev measurement %s is invalid" % ( prevMeasurement) assert 0 <= inputMomentum < 1, "Momentum is %s, not in the range [0;1)" % ( inputMomentum) # Do not keep a reference to the previous measure, to avoid a memory leak. self.prevNormalisedCpuUtil = prevMeasurement.normaliseCpuUtil() self.prevNormalisedRAMUtil = prevMeasurement.normaliseRAMUtil() self.prevNormalisedNICUtil = prevMeasurement.normaliseNICUtil() self.prevNormalisedDiskUtil = prevMeasurement.normaliseDiskUtil() self.prevNumUsers = prevMeasurement.numberOfUsers() self.prevCpuUtil = prevMeasurement.cpuUtil() self.prevRamUtil = prevMeasurement.ramUtil() self.prevAnomaly = prevMeasurement.getAnomaly() self.prevNormCpus.extend(prevMeasurement.prevNormCpus) self.prevNormCpus.append(prevMeasurement.normaliseCpuUtil()) self.prevUsers.extend(prevMeasurement.prevUsers) self.prevUsers.append(prevMeasurement.numberOfUsers()) self.momentum = inputMomentum else: self.momentum = None def isValid(self, err=[]): """ Returns if this measurement is valid - i.e. there are no concerns for mismeasurments. @param param: an optional output list parameter, where to store all error messages if the measurement is invalid. @return: if this measurement is valid - i.e. there are no concerns for mismeasurments. """ overloadRatio = 0.7 isOverloaded = (self.cpuUtil() > overloadRatio or self.ramUtil() > overloadRatio or self.normaliseDiskUtil() > overloadRatio or self.normaliseNICUtil() > overloadRatio) isUnderloaded = (self.cpuUtil() < 0.1 or self.numberOfUsers() < 25) #prevUsers = self.prevNumUsers if self.prevNumUsers != 0 else 1 #currUsers = self.numUsers if self.numUsers != 0 else 1 #usersRatio = prevUsers / float(currUsers) #if currUsers > prevUsers else currUsers / float(prevUsers) #cpuRatio = 0 if self.prevCpuUtil is None or self.cpuUtil() == 0 else self.prevCpuUtil / float(self.cpuUtil()) #numUsersSwitch = usersRatio < 0.5 or usersRatio > 2 or cpuRatio < 0.5 or cpuRatio > 2 avgPrevCPU = self.getAvgPrevNormCPU() avgNumUsers = sum(self.prevUsers) / len(self.prevUsers) if len( self.prevUsers) > 0 else self.numberOfUsers() cpuAvgRatio = self.normaliseCpuUtil( ) / avgPrevCPU if avgPrevCPU > 0 else 0 usrsAvgRatio = self.numberOfUsers( ) / avgNumUsers if avgNumUsers > 0 else 0 avgSwitch = cpuAvgRatio < 0.5 or cpuAvgRatio > 2 or usrsAvgRatio < 0.5 or usrsAvgRatio > 2 if isOverloaded: log.info("Invalid because it's overloaded") err.append("overloaded") if isUnderloaded: log.info("Invalid because it's underloaded") err.append("underloaded") if avgSwitch: log.info("Invalid because it's an AVG switch") err.append("AVG Switch") # if numUsersSwitch: # log.info("Invalid because it's a switch") # err.append("switch") return not (isOverloaded or isUnderloaded or avgSwitch) def getAvgPrevNormCPU(self): """ Returns the average normalised CPU of the last 3 CPU measurements. @return: the average normalised CPU of the last 3 CPU measurements. """ return sum(self.prevNormCpus) / len(self.prevNormCpus) if len( self.prevNormCpus) > 0 else self.normaliseCpuUtil() @staticmethod def _assertRatio(value): assert 0 <= value <= 1, "Value {0} is not in the range [0, 1]" % ( value) return value
from autoscale.AWSBillingPolicy import AWSBillingPolicy ##== Configure the level, handler and format for the loggers rootLogger = logging.getLogger() rootLogger.setLevel(logging.DEBUG) ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s', datefmt="%H:%M:%S") ch.setFormatter(formatter) rootLogger.addHandler(ch) logging.getLogger('paramiko').setLevel(logging.ERROR) small = VMType(code="m1.small", declaredCpuCapacity=0.5, declaredRAMCapacityKB=convertMem(0.615, "GB", "KB"), costPerTimeUnit=0.02) f = VMFactory(providerId="openstack-nova", \ endPoint="https://keystone.rc.nectar.org.au:5000/v2.0/", \ accesskeyid="pt-697:[email protected]", \ secretkey="M2M0ZDZiOTIyNDI4MjQ1", \ imageOwnerId=None, \ locationId = "Melbourne", \ imageId = "Melbourne/b40a036d-3911-4533-84f5-ad565b8376dc", \ securityGroupName = "AllOpen", \ keyPairName = "MCCloud", \ groupName = "test-mccloud", \ vmManagerJar = vmManagerJarPath(), \ pemFile = "/home/nikolay/Dropbox/mccloud.pem", \ monitoringScript = scriptPath("monitor.sh"), \ userName = "******", \
runConfig = "/home/nikolay/Dropbox/CloudStoneSetupOnUbuntuAdvanced/AutoScaler/run.xml" userName = "******" loadBalancerAddress = "ec2-54-79-203-150.ap-southeast-2.compute.amazonaws.com" firstAppServerAddress = "ec2-54-253-205-116.ap-southeast-2.compute.amazonaws.com" #"ec2-54-253-144-28.ap-southeast-2.compute.amazonaws.com" clientAddress = "ec2-54-79-149-247.ap-southeast-2.compute.amazonaws.com" ##== Factory for creating objects that manage VMs, Load Balancer and Clients factory = VMFactory(providerId, endPoint, accesskeyid, secretkey, imageOwnerId, locationId, imageId, securityGroupName, keyPairName, groupName, vmManagerJar, pemFile, monitoringScript, userName, runConfig, billingPolicy) ##== VM types t1Micro = VMType(code="t1.micro", declaredCpuCapacity=0.5, declaredRAMCapacityKB=convertMem(0.615, "GB", "KB"), costPerTimeUnit=0.02) m1Small = VMType(code="m1.small", declaredCpuCapacity=1, declaredRAMCapacityKB=convertMem(1.7, "GB", "KB"), costPerTimeUnit=0.058) m1Medium = VMType(code="m1.medium", declaredCpuCapacity=2, declaredRAMCapacityKB=convertMem(3.75, "GB", "KB"), costPerTimeUnit=0.117) m3Medium = VMType(code="m3.medium", declaredCpuCapacity=3, declaredRAMCapacityKB=convertMem(3.75, "GB", "KB"), costPerTimeUnit=0.098) types = [m1Small, m1Medium, m3Medium]
##== Initialising common variables - addresses, access keys pemFile = "/home/nikolay/Dropbox/CloudStoneSetupOnUbuntuAdvanced/CloudStone.pem" monitoringScript = "/home/nikolay/Dropbox/CloudStoneSetupOnUbuntuAdvanced/AutoScaler/monitor.sh" runConfig = "/home/nikolay/Dropbox/CloudStoneSetupOnUbuntuAdvanced/AutoScaler/run.xml" userName = "******" loadBalancerAddress = "ec2-54-79-203-150.ap-southeast-2.compute.amazonaws.com" firstAppServerAddress = "ec2-54-253-205-116.ap-southeast-2.compute.amazonaws.com"#"ec2-54-253-144-28.ap-southeast-2.compute.amazonaws.com" clientAddress = "ec2-54-79-149-247.ap-southeast-2.compute.amazonaws.com" ##== Factory for creating objects that manage VMs, Load Balancer and Clients factory = ClientFactory(providerId, accesskeyid, secretkey, imageOwnerId, locationId, imageId, securityGroupName, keyPairName,\ groupName, mavenPrjPath, pemFile, monitoringScript, userName, runConfig) ##== VM types t1Micro = VMType(code="t1.micro", declaredCpuCapacity=0.5, declaredRAMCapacityKB=convertMem(0.615, "GB", "KB"), costPerTimeUnit=0.02) m1Small = VMType(code="m1.small", declaredCpuCapacity=1, declaredRAMCapacityKB=convertMem(1.7, "GB", "KB"), costPerTimeUnit=0.058) m1Medium = VMType(code="m1.medium", declaredCpuCapacity=2, declaredRAMCapacityKB=convertMem(3.75, "GB", "KB"), costPerTimeUnit=0.117) m3Medium = VMType(code="m3.medium", declaredCpuCapacity=3, declaredRAMCapacityKB=convertMem(3.75, "GB", "KB"), costPerTimeUnit=0.098) types = [m1Small, m1Medium, m3Medium] types.sort(key=lambda t: t.declaredCpuCapacity) ##== Initialise client, first AS servers and Load Balancers firstAppServer = factory.createVM(readableName="App 1", vmType=m3Medium, address=firstAppServerAddress) serverFarm = factory.createServerFarm(address=loadBalancerAddress) serverFarm.addServers(firstAppServer) ##== fann = FANNWrapper(topology=(inputUnits, hiddenUnits, 2))