def fetchData(self, inputMomentum, injectVariance=False):
"""
Fetches measurement data from the server in the form of a VMMeasurement instance.
@param inputMomentum: The measurement momentum. If None - no momentum will be considered.
@param injectVariance: Whether to "inject" a change in the workload pattern. Used for experiment purposes.
@return: measurement data from the server in the form of a VMMeasurement instance. May return None if no data is available.
"""
data = filterEmptyStrings(
map(extractVal,
self.fetchRawData().split(";")))
if (data and not AppServer._isHeader(data) and len(data) == 10):
assert len(
data
) == 10, "Measurement data %s does not have proper size" % (data)
log.debug("Measurement %s: %s", self.readableName, data)
varianceActiveMem = data[6] + convertMem(
1, fromCode="GB", toCode="KB") + data[9] * convertMem(
1, fromCode="MB", toCode="KB")
varianceIdlePerc = data[4] * 0.9
measurement = VMMeasurement(
readableName=self._measName(),
vmAddress=self.address,
serverTime=data[0],
cpuCapacityMhz=data[1],
cpuIOWaitPerc=data[2],
cpuStealPerc=data[3],
cpuIdlePerc=data[4]
if not injectVariance else varianceIdlePerc,
ramInKb=data[5],
activeMemInKb=data[6]
if not injectVariance else varianceActiveMem,
diskUtilPerc=data[7],
nicUtilPerc=data[8],
numUsers=data[9])
measurement.considerMomentum(self.lastMeasurement, inputMomentum)
self.lastMeasurement = measurement
self.vmType.addMeasurement(measurement)
self.htm.train(measurement)
self._line(measurement)
return measurement
else:
return None
def fetchData(self, inputMomentum, injectVariance = False):
"""
Fetches measurement data from the server in the form of a VMMeasurement instance.
@param inputMomentum: The measurement momentum. If None - no momentum will be considered.
@param injectVariance: Whether to "inject" a change in the workload pattern. Used for experiment purposes.
@return: measurement data from the server in the form of a VMMeasurement instance. May return None if no data is available.
"""
data = filterEmptyStrings( map(extractVal, self.fetchRawData().split(";")) )
if(data and not AppServer._isHeader(data) and len(data) == 10):
assert len(data) == 10, "Measurement data %s does not have proper size" % (data)
log.debug("Measurement %s: %s", self.readableName, data)
varianceActiveMem = data[6] + convertMem(1, fromCode="GB", toCode="KB") + data[9] * convertMem(1, fromCode="MB", toCode="KB")
varianceIdlePerc = data[4] * 0.9
measurement = VMMeasurement(readableName = self._measName(),
vmAddress=self.address,
serverTime = data[0],
cpuCapacityMhz = data[1],
cpuIOWaitPerc = data[2],
cpuStealPerc = data[3],
cpuIdlePerc = data[4] if not injectVariance else varianceIdlePerc,
ramInKb = data[5],
activeMemInKb = data[6] if not injectVariance else varianceActiveMem,
diskUtilPerc = data[7],
nicUtilPerc = data[8],
numUsers = data[9])
measurement.considerMomentum(self.lastMeasurement, inputMomentum)
self.lastMeasurement = measurement
self.vmType.addMeasurement(measurement)
self.htm.train(measurement)
self._line(measurement)
return measurement
else:
return None
class VMMeasurement(BaseAutoscalingClass):
"""Performance measurements taken from a VM."""
freqMaxInMhz = 3500
maxNumCores = 8
maxCpuCapacity = freqMaxInMhz * maxNumCores
maxRAMInKb = convertMem(8, fromCode="GB", toCode="KB")
def __init__(self, readableName, vmAddress, serverTime, cpuCapacityMhz,
cpuIOWaitPerc, cpuStealPerc, cpuIdlePerc, ramInKb,
activeMemInKb, diskUtilPerc, nicUtilPerc, numUsers):
"""
Constr.
@param readableName: see superclass.
@param vmAddress: the address of the VM. Must not be None.
@param serverTime: the time of the measurement on the VM. Must not be None.
@param cpuCapacityMhz: The CPU capacity in Mhz. Must not be None. Must be positive.
@param cpuIOWaitPerc: Percentage of CPU I/O waiting. Must not be None. Must be non-negative.
@param cpuStealPerc: Percentage of stolen CPU time by the hypervisor. Must not be None. Must be non-negative.
@param cpuIdlePerc: Percentage of CPU idle time. Must not be None. Must be non-negative.
@param ramInKb: RAM capacity in Kb. Must not be None. Must be positive.
@param activeMemInKb: Actively used RAM in Kb. Must not be None. Must be positive and less than the capacity.
@param diskUtilPerc: Percentage of disk utilisation. Must not be None.
@param nicUtilPerc: Percentage of NIC utilisation. Must not be None.
@param numUsers: Number of users served on the system. Must not be None. Must be non-negative.
"""
super(VMMeasurement, self).__init__(readableName=readableName)
assert 0 <= cpuCapacityMhz <= VMMeasurement.maxCpuCapacity, "CPU capacity {0} is not in the range[{1}, {2}]".format(
cpuCapacityMhz, 0, VMMeasurement.maxCpuCapacity)
assert 0 <= cpuIOWaitPerc <= 100, "cpuIOWait% {0} is not in the range[{1}, {2}]".format(
cpuIOWaitPerc, 0, 100)
assert 0 <= cpuStealPerc <= 100, "cpuSteal% {0} is not in the range[{1}, {2}]".format(
cpuStealPerc, 0, 100)
assert 0 <= cpuIdlePerc <= 100, "cpuIdle% {0} is not in the range[{1}, {2}]".format(
cpuIdlePerc, 0, 100)
assert cpuIOWaitPerc + cpuStealPerc + cpuIdlePerc <= 100, "cpuIOWait% + cpuSteal + cpuIdle% ({0}) is greater than 100".format(
cpuIOWaitPerc + cpuStealPerc + cpuIdlePerc)
assert 0 <= ramInKb <= VMMeasurement.maxRAMInKb, "RAM capacity {0} is not in the range[{1}, {2}]".format(
ramInKb, 0, VMMeasurement.maxRAMInKb)
if activeMemInKb > ramInKb:
activeMemInKb = ramInKb
assert 0 <= activeMemInKb <= ramInKb, "Active RAM {0} is not in the range[{1}, {2}]".format(
activeMemInKb, 0, ramInKb)
assert 0 <= diskUtilPerc <= 100, "diskUtil% {0} is not in the range[{1}, {2}]".format(
diskUtilPerc, 0, 100)
assert 0 <= nicUtilPerc <= 100, "nicUtil% {0} is not in the range[{1}, {2}]".format(
nicUtilPerc, 0, 100)
assert 0 <= numUsers, "numUsers {0} is not positive".format(numUsers)
self.timestamp = datetime.datetime.now()
self.vmAddress = vmAddress
self.serverTime = serverTime
self.cpuCapacityMhz = cpuCapacityMhz
self.cpuIOWaitPerc = cpuIOWaitPerc
self.cpuStealPerc = cpuStealPerc
self.cpuIdlePerc = cpuIdlePerc
self.ramInKb = ramInKb
self.activeMemInKb = activeMemInKb
self.diskUtilPerc = diskUtilPerc
self.nicUtilPerc = nicUtilPerc
self.numUsers = numUsers
self.anomaly = 0
self.prevNormalisedCpuUtil = None
self.prevNormalisedRAMUtil = None
self.prevNormalisedNICUtil = None
self.prevNormalisedDiskUtil = None
self.prevNumUsers = 0
self.prevCpuUtil = None
self.prevRamUtil = None
self.prevAnomaly = None
self.prevNormCpus = collections.deque(maxlen=3)
self.prevUsers = collections.deque(maxlen=3)
self.momentum = None
def normaliseCpuCapacity(self):
"""
Returns the normalised CPU capacity of the VM in the range [0,1]. Considers momentum if set.
@return: The normalised CPU capacity.
"""
return VMMeasurement._assertRatio(
(100.0 - self.cpuStealPerc) * self.cpuCapacityMhz /
(100 * VMMeasurement.maxCpuCapacity))
def normaliseRAMCapacity(self):
"""
Returns the normalised RAM capacity of the VM in the range [0,1]. Considers momentum if set.
@return: The normalised RAM capacity.
"""
return VMMeasurement._assertRatio(
float(self.ramInKb) / VMMeasurement.maxRAMInKb)
def normaliseCpuUtil(self):
"""
Returns the normalised CPU utilisation of the VM in the range [0,1]. Considers momentum if set.
@return: The normalised CPU utilisation.
"""
result = (100.0 - self.cpuStealPerc -
self.cpuIdlePerc) * self.cpuCapacityMhz / (
100 * VMMeasurement.maxCpuCapacity)
return VMMeasurement._assertRatio(
self._momentum(result, self.prevNormalisedCpuUtil))
def cpuUtil(self):
"""
Returns the CPU utilisation of the VM in the range [0,1]. Considers momentum if set.
@return: The CPU utilisation.
"""
result = (100.0 - self.cpuStealPerc -
self.cpuIdlePerc) / (100.0 - self.cpuStealPerc)
return VMMeasurement._assertRatio(
self._momentum(result, self.prevCpuUtil))
def normaliseRAMUtil(self):
"""
Returns the normalised RAM utilisation of the VM in the range [0,1]. Considers momentum if set.
@return: The normalised RAM utilisation.
"""
result = float(self.activeMemInKb) / VMMeasurement.maxRAMInKb
return VMMeasurement._assertRatio(
self._momentum(result, self.prevNormalisedRAMUtil))
def ramUtil(self):
"""
Returns the RAM utilisation of the VM in the range [0,1]. Considers momentum if set.
@return: The RAM utilisation.
"""
result = self.activeMemInKb / float(self.ramInKb)
return VMMeasurement._assertRatio(
self._momentum(result, self.prevRamUtil))
def getAnomaly(self):
"""
Returns the anomaly score of this VM measurment in the range [0,1]. Considers momentum if set.
@return: The anomaly score of this measurement.
"""
return VMMeasurement._assertRatio(
self._momentum(self.anomaly, self.prevAnomaly))
def normaliseNICUtil(self):
"""
Returns the normalised NIC utilisation of the VM in the range [0,1]. Considers momentum if set.
@return: The normalised NIC utilisation.
"""
result = self.nicUtilPerc / 100.0
return VMMeasurement._assertRatio(
self._momentum(result, self.prevNormalisedNICUtil))
def normaliseDiskUtil(self):
"""
Returns the normalised disk utilisation of the VM in the range [0,1]. Considers momentum if set.
@return: The normalised disk utilisation.
"""
result = self.diskUtilPerc / 100.0
return VMMeasurement._assertRatio(
self._momentum(result, self.prevNormalisedDiskUtil))
def numberOfUsers(self):
"""
Returns the number of users served in the VM.
@return: The number of users served in the VM.
"""
result = self.numUsers
return int(self._momentum(result, self.prevNumUsers))
def _momentum(self, value, prevValue):
if self.momentum is not None:
return (1 - self.momentum) * value + self.momentum * prevValue
else:
return value
def considerMomentum(self, prevMeasurement, inputMomentum):
"""
Modifies this measurment in the light of the previous measurement by using the specified momentum.
@param prevMeasurement: the previous measurement. If None - no momentum is considered. An instance of VMMeasurement.
@param inputMomentum: the momentum. If None - no momentum is considered. If not None must in the range [0,1).
"""
if prevMeasurement is not None and inputMomentum is not None:
assert isinstance(
prevMeasurement,
VMMeasurement), "Prev measurement %s is invalid" % (
prevMeasurement)
assert 0 <= inputMomentum < 1, "Momentum is %s, not in the range [0;1)" % (
inputMomentum)
# Do not keep a reference to the previous measure, to avoid a memory leak.
self.prevNormalisedCpuUtil = prevMeasurement.normaliseCpuUtil()
self.prevNormalisedRAMUtil = prevMeasurement.normaliseRAMUtil()
self.prevNormalisedNICUtil = prevMeasurement.normaliseNICUtil()
self.prevNormalisedDiskUtil = prevMeasurement.normaliseDiskUtil()
self.prevNumUsers = prevMeasurement.numberOfUsers()
self.prevCpuUtil = prevMeasurement.cpuUtil()
self.prevRamUtil = prevMeasurement.ramUtil()
self.prevAnomaly = prevMeasurement.getAnomaly()
self.prevNormCpus.extend(prevMeasurement.prevNormCpus)
self.prevNormCpus.append(prevMeasurement.normaliseCpuUtil())
self.prevUsers.extend(prevMeasurement.prevUsers)
self.prevUsers.append(prevMeasurement.numberOfUsers())
self.momentum = inputMomentum
else:
self.momentum = None
def isValid(self, err=[]):
"""
Returns if this measurement is valid - i.e. there are no concerns for mismeasurments.
@param param: an optional output list parameter, where to store all error messages if the measurement is invalid.
@return: if this measurement is valid - i.e. there are no concerns for mismeasurments.
"""
overloadRatio = 0.7
isOverloaded = (self.cpuUtil() > overloadRatio
or self.ramUtil() > overloadRatio
or self.normaliseDiskUtil() > overloadRatio
or self.normaliseNICUtil() > overloadRatio)
isUnderloaded = (self.cpuUtil() < 0.1 or self.numberOfUsers() < 25)
#prevUsers = self.prevNumUsers if self.prevNumUsers != 0 else 1
#currUsers = self.numUsers if self.numUsers != 0 else 1
#usersRatio = prevUsers / float(currUsers) #if currUsers > prevUsers else currUsers / float(prevUsers)
#cpuRatio = 0 if self.prevCpuUtil is None or self.cpuUtil() == 0 else self.prevCpuUtil / float(self.cpuUtil())
#numUsersSwitch = usersRatio < 0.5 or usersRatio > 2 or cpuRatio < 0.5 or cpuRatio > 2
avgPrevCPU = self.getAvgPrevNormCPU()
avgNumUsers = sum(self.prevUsers) / len(self.prevUsers) if len(
self.prevUsers) > 0 else self.numberOfUsers()
cpuAvgRatio = self.normaliseCpuUtil(
) / avgPrevCPU if avgPrevCPU > 0 else 0
usrsAvgRatio = self.numberOfUsers(
) / avgNumUsers if avgNumUsers > 0 else 0
avgSwitch = cpuAvgRatio < 0.5 or cpuAvgRatio > 2 or usrsAvgRatio < 0.5 or usrsAvgRatio > 2
if isOverloaded:
log.info("Invalid because it's overloaded")
err.append("overloaded")
if isUnderloaded:
log.info("Invalid because it's underloaded")
err.append("underloaded")
if avgSwitch:
log.info("Invalid because it's an AVG switch")
err.append("AVG Switch")
# if numUsersSwitch:
# log.info("Invalid because it's a switch")
# err.append("switch")
return not (isOverloaded or isUnderloaded or avgSwitch)
def getAvgPrevNormCPU(self):
"""
Returns the average normalised CPU of the last 3 CPU measurements.
@return: the average normalised CPU of the last 3 CPU measurements.
"""
return sum(self.prevNormCpus) / len(self.prevNormCpus) if len(
self.prevNormCpus) > 0 else self.normaliseCpuUtil()
@staticmethod
def _assertRatio(value):
assert 0 <= value <= 1, "Value {0} is not in the range [0, 1]" % (
value)
return value
from autoscale.AWSBillingPolicy import AWSBillingPolicy
##== Configure the level, handler and format for the loggers
rootLogger = logging.getLogger()
rootLogger.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s', datefmt="%H:%M:%S")
ch.setFormatter(formatter)
rootLogger.addHandler(ch)
logging.getLogger('paramiko').setLevel(logging.ERROR)
small = VMType(code="m1.small", declaredCpuCapacity=0.5, declaredRAMCapacityKB=convertMem(0.615, "GB", "KB"), costPerTimeUnit=0.02)
f = VMFactory(providerId="openstack-nova", \
endPoint="https://keystone.rc.nectar.org.au:5000/v2.0/", \
accesskeyid="pt-697:[email protected]", \
secretkey="M2M0ZDZiOTIyNDI4MjQ1", \
imageOwnerId=None, \
locationId = "Melbourne", \
imageId = "Melbourne/b40a036d-3911-4533-84f5-ad565b8376dc", \
securityGroupName = "AllOpen", \
keyPairName = "MCCloud", \
groupName = "test-mccloud", \
vmManagerJar = vmManagerJarPath(), \
pemFile = "/home/nikolay/Dropbox/mccloud.pem", \
monitoringScript = scriptPath("monitor.sh"), \
userName = "******", \
runConfig = "/home/nikolay/Dropbox/CloudStoneSetupOnUbuntuAdvanced/AutoScaler/run.xml"
userName = "******"
loadBalancerAddress = "ec2-54-79-203-150.ap-southeast-2.compute.amazonaws.com"
firstAppServerAddress = "ec2-54-253-205-116.ap-southeast-2.compute.amazonaws.com" #"ec2-54-253-144-28.ap-southeast-2.compute.amazonaws.com"
clientAddress = "ec2-54-79-149-247.ap-southeast-2.compute.amazonaws.com"
##== Factory for creating objects that manage VMs, Load Balancer and Clients
factory = VMFactory(providerId, endPoint, accesskeyid, secretkey, imageOwnerId,
locationId, imageId, securityGroupName, keyPairName,
groupName, vmManagerJar, pemFile, monitoringScript,
userName, runConfig, billingPolicy)
##== VM types
t1Micro = VMType(code="t1.micro",
declaredCpuCapacity=0.5,
declaredRAMCapacityKB=convertMem(0.615, "GB", "KB"),
costPerTimeUnit=0.02)
m1Small = VMType(code="m1.small",
declaredCpuCapacity=1,
declaredRAMCapacityKB=convertMem(1.7, "GB", "KB"),
costPerTimeUnit=0.058)
m1Medium = VMType(code="m1.medium",
declaredCpuCapacity=2,
declaredRAMCapacityKB=convertMem(3.75, "GB", "KB"),
costPerTimeUnit=0.117)
m3Medium = VMType(code="m3.medium",
declaredCpuCapacity=3,
declaredRAMCapacityKB=convertMem(3.75, "GB", "KB"),
costPerTimeUnit=0.098)
types = [m1Small, m1Medium, m3Medium]
##== Initialising common variables - addresses, access keys
pemFile = "/home/nikolay/Dropbox/CloudStoneSetupOnUbuntuAdvanced/CloudStone.pem"
monitoringScript = "/home/nikolay/Dropbox/CloudStoneSetupOnUbuntuAdvanced/AutoScaler/monitor.sh"
runConfig = "/home/nikolay/Dropbox/CloudStoneSetupOnUbuntuAdvanced/AutoScaler/run.xml"
userName = "******"
loadBalancerAddress = "ec2-54-79-203-150.ap-southeast-2.compute.amazonaws.com"
firstAppServerAddress = "ec2-54-253-205-116.ap-southeast-2.compute.amazonaws.com"#"ec2-54-253-144-28.ap-southeast-2.compute.amazonaws.com"
clientAddress = "ec2-54-79-149-247.ap-southeast-2.compute.amazonaws.com"
##== Factory for creating objects that manage VMs, Load Balancer and Clients
factory = ClientFactory(providerId, accesskeyid, secretkey, imageOwnerId, locationId, imageId, securityGroupName, keyPairName,\
groupName, mavenPrjPath, pemFile, monitoringScript, userName, runConfig)
##== VM types
t1Micro = VMType(code="t1.micro", declaredCpuCapacity=0.5, declaredRAMCapacityKB=convertMem(0.615, "GB", "KB"), costPerTimeUnit=0.02)
m1Small = VMType(code="m1.small", declaredCpuCapacity=1, declaredRAMCapacityKB=convertMem(1.7, "GB", "KB"), costPerTimeUnit=0.058)
m1Medium = VMType(code="m1.medium", declaredCpuCapacity=2, declaredRAMCapacityKB=convertMem(3.75, "GB", "KB"), costPerTimeUnit=0.117)
m3Medium = VMType(code="m3.medium", declaredCpuCapacity=3, declaredRAMCapacityKB=convertMem(3.75, "GB", "KB"), costPerTimeUnit=0.098)
types = [m1Small, m1Medium, m3Medium]
types.sort(key=lambda t: t.declaredCpuCapacity)
##== Initialise client, first AS servers and Load Balancers
firstAppServer = factory.createVM(readableName="App 1", vmType=m3Medium, address=firstAppServerAddress)
serverFarm = factory.createServerFarm(address=loadBalancerAddress)
serverFarm.addServers(firstAppServer)
##==
fann = FANNWrapper(topology=(inputUnits, hiddenUnits, 2))
