def create(self, req):
rc = req.params.get('requireCount')
resetFlag = req.params.get('reset')
if rc and isNumber(rc) and resetFlag and isNumber(resetFlag):
rc = int(rc)
resetFlag = int(resetFlag)
if resetFlag:
TomcatInstanceUtil.resetAllUsingInstances()
uic = UsingInstancesDBUtil.getUsingInstancesCount()
if rc > uic:
needC = rc - uic
while needC > 0:
TomcatInstanceUtil.createTomcatInstance()
needC -= 1
elif rc < uic:
deleteC = uic - rc
TomcatInstanceUtil.deleteSpecifyNumberInstances(deleteC)
TomcatInstanceUtil.ensureAllUsingInstancesActive()
return UsingInstancesDBUtil.getAllUsingInstancesInfo()
else:
result = errorResultJson('The Post Body Must be {requireCount:x, reset:y} (ps:x must be number, reset must be 0 or 1)')
return result
def migrate(vmId, azName):
if not vmId or not azName:
return False
nova = GetClientUtil.getNovaClient()
targetInstance = InstanceUtil.findInstanceById(vmId)
ips = nova.servers.ips(targetInstance.id)
innerIP = TomcatInstanceUtil.getInnerIPFromIPInfo(ips)
targetInstance.delete()
oldStdout = sys.stdout
nullFile = open('/dev/null', 'w')
sys.stdout = nullFile
while InstanceUtil.findInstanceById(vmId):
time.sleep(2)
sys.stdout = oldStdout
nullFile.close()
UsingInstancesDBUtil.deleteUsingInstanceByResourceId(targetInstance.id)
newInstance = TomcatInstanceUtil.createTomcatInstance(azName, innerIP)
if newInstance:
while not InstanceUtil.isInstanceStatusActice(newInstance.id):
time.sleep(2)
return True
else:
return False
def deleteAllTestingInstance():
nova = GetClientUtil.getNovaClient()
instancesList = nova.servers.list()
for instance in instancesList:
if len(instance.name) > 20:
instance.delete()
UsingInstancesDBUtil.deleteUsingInstanceByResourceId(
instance.id)
def getInitialScheme(self):
periodDB = shelve.open(periodRecoderFile)
periodDB[periodRecoder] = None
periodDB.close()
fiboDB = shelve.open(fiboDataFile)
fiboDB[fiboDataName] = None
fiboDB.close()
clearAllData()
PerformanceDBUtil.clearPerformanceDataTable()
WorkloadDBUtil.clearWorkloadTable()
uiCount = UsingInstancesDBUtil.getUsingInstancesCount()
if uiCount < initUsingInstancesNumbers:
TomcatInstanceUtil.resetAllUsingInstances()
self.acrcPlacementComponent.getPlacementScheme(
initUsingInstancesNumbers - uiCount, True)
elif uiCount > initUsingInstancesNumbers:
self.acrcPlacementComponent.getPlacementScheme(
uiCount - initUsingInstancesNumbers, False)
TomcatInstanceUtil.resetAllUsingInstances()
else:
TomcatInstanceUtil.resetAllUsingInstances()
def ensureAllUsingInstancesActive():
usingInstancesIds = UsingInstancesDBUtil.getAllUsingInstancesIds()
while usingInstancesIds:
for uiid in usingInstancesIds:
if InstanceUtil.isInstanceStatusActice(uiid):
usingInstancesIds.remove(uiid)
time.sleep(5)
def deleteSpecifyNumberInstances(no):
if not no:
return
nova = GetClientUtil.getNovaClient()
instancesList = nova.servers.list()
count = 0
for instance in instancesList:
if len(instance.name) > 20:
instance.delete()
UsingInstancesDBUtil.deleteUsingInstanceByResourceId(
instance.id)
count += 1
if count == no:
break
def initExperimentThree(self, req):
vmNum = req.params.get('requireCount')
if vmNum and isNumber(vmNum):
rc = int(vmNum)
TomcatInstanceUtil.deleteAllTestingInstance()
azList = ['az1', 'az2', 'az3', 'az4']
azLen = len(azList)
for i in range(rc):
TomcatInstanceUtil.createTomcatInstance(azName=azList[i % azLen])
TomcatInstanceUtil.ensureAllUsingInstancesActive()
for ipEnd in ipEndOfComputes:
os.system('/home/sk/cloudEx/shellScript/initAllPMUtil.sh ' + str(ipEnd) + ' > /dev/null')
# 清空与预测相关的数据
clearAllData()
PMCPUDBUtil.clearPMCPUTable()
VMCPUDBUtil.clearVMCPUTable()
return UsingInstancesDBUtil.getAllUsingInstancesInfo()
else:
result = errorResultJson('The Post Body Must be {requireCount:x} (ps:x must be number)')
return result
def getVMList(self, req):
pmId = req.params.get('id')
azName = PMAndAZDBUtil.getAZNameByResourceId(pmId)
vmList = UsingInstancesDBUtil.getUsingInstancesByAZName(azName)
if not vmList:
return []
else:
return [vm['id'] for vm in vmList]
def getThreadInfoByResourceId(resourceId, jumper_server):
innerIP = UsingInstancesDBUtil.getUsingInstanceInnerIPById(resourceId)
url = jumper_server + "/getSpecificThreadInfo?ip=" + str(innerIP)
req = urllib2.Request(url)
# TODO: retry mechanism might be added later
infos = eval(urllib2.urlopen(req).read())
# here simplify the model since our instance must has single cpu
freq = infos[0][2] # format => [[thread, freqStandard, freqReal]...]
util = SampleUtil.getCpuUtilPeriodAVGByResourceId(resourceId)
return {"cal": 1.0 * freq * util, "util": util}
def getAllUsingInstancesPeriodAVGMemoryUtil():
allUiIds = UsingInstancesDBUtil.getAllUsingInstancesIds()
uiAvgMemoryList = []
for uiId in allUiIds:
uiAvgMemoryList.append(
SampleUtil.getMemoryUtilPeriodAVGByResourceId(uiId))
if uiAvgMemoryList:
return avgNumberList(uiAvgMemoryList)
else:
return None
def getNumberOfVMsShouldBeScaled(self):
if not self.predictor or not self.ruleChecker:
raise Exception('no predictor or ruleChecker')
newestHD = WorkloadDBUtil.getNewestWorkload()
if not newestHD:
raise Exception('Can not get newest workload')
addWLToACRCWindow(newestHD['realWL'])
predictWL = self.predictor.getNextPeriodWorkload()
addPWLToACRCWindow(predictWL)
#预测写入Workload表
periodNoDB = shelve.open(periodRecoderFile)
periodNo = periodNoDB.get(periodRecoder, None)
if not periodNo:
raise Exception('Can not get periodNo')
WorkloadDBUtil.addPredictWorkloadToSpecificPeriod(periodNo, predictWL)
#记录供给的虚拟机数目的分布
provisionInfoDB = shelve.open(provisionInfoFile)
#查询需要的VM
predictVMNumbers = WorkloadVMMapDBUtil.getTargetVMsToSpecificWorkload(predictWL)
provisionInfoDB[predictProvisionVMNumbers] = predictVMNumbers
logger.info('PredictWorkload:' + str(predictWL) + ' predictVMNumbers:' + str(predictVMNumbers))
addedVMNumbers = self.ruleChecker.getNextAddedVMs()
provisionInfoDB[reactiveProvisionVMNumbers] = addedVMNumbers
logger.info('addedVMNumbers:' + str(addedVMNumbers))
provisionInfoDB.close()
nextPeriodVMNumbers = predictVMNumbers + addedVMNumbers
usingInstancesCount = UsingInstancesDBUtil.getUsingInstancesCount()
gab = abs(usingInstancesCount - nextPeriodVMNumbers)
if usingInstancesCount > nextPeriodVMNumbers:
return {'vmNumbers':gab, 'isUp':False}
elif usingInstancesCount < nextPeriodVMNumbers:
return {'vmNumbers':gab, 'isUp':True}
else:
return {'vmNumbers':gab}
def deleteSpecifyNumberInstancesWithSpecifyAZ(no, azName):
if not no:
return
holdVms = UsingInstancesDBUtil.getUsingInstancesByAZName(azName)
holdVmsLen = len(holdVms)
if no > holdVmsLen:
raise Exception('Could not down ' + str(no) + ' vms in ' +
str(azName) + ', it only has ' + str(holdVmsLen) +
' vms!')
nova = GetClientUtil.getNovaClient()
count = 0
for vm in holdVms:
targetInstance = nova.servers.get(vm['id'])
targetInstance.delete()
UsingInstancesDBUtil.deleteUsingInstanceByResourceId(
targetInstance.id)
count += 1
if count == no:
break
def getThreadInfosOverAllUsingInstances(jump_server):
allUiIds = UsingInstancesDBUtil.getAllUsingInstancesIds()
uiAvgCPUList = []
totalCalculation = 0.0
for uiId in allUiIds:
tmp = SampleUtil.getThreadInfoByResourceId(uiId, jump_server)
uiAvgCPUList.append(tmp["util"])
totalCalculation += tmp["cal"]
if uiAvgCPUList:
return avgNumberList(uiAvgCPUList), totalCalculation
else:
return None
def initExperiment(self, req):
check = 1
try:
token = req.headers['X-Auth-Token']
if token != 'sk':
check = 0
except KeyError:
check = 0
if check:
ExperimentInit().getInitialScheme()
return UsingInstancesDBUtil.getAllUsingInstancesInfo()
else:
return errorResultJson('You are not allowed to do this!')
def updateCloudInfo(self, topoFilePath):
if not topoFilePath:
raise Exception('no topoFile')
topoDic = analyseJsonFormatConfigFile(topoFilePath)
assert topoDic, 'no cloud topo infomation'
#获取Region和AZ的availability
nodeAvailabilityFilePath = '/home/sk/openstackPythonSDKTest/ConfigDic/NodeAvailability'
naf = open(nodeAvailabilityFilePath)
exec naf
naf.close()
cloudRoot = TopoTreeNode('cloudRoot', TopoTreeNodeAvailability)
azList = []
for regionName in topoDic:
regionTreeNode = TopoTreeNode(regionName, RegionAvailability)
cloudRoot.children.append(regionTreeNode)
regionAZs = topoDic[regionName]
for az in regionAZs:
vmsInAz = UsingInstancesDBUtil.getUsingInstancesByAZName(
az['azName'])
if vmsInAz:
azTreeNode = TopoTreeNode(az['azName'], AZAvailability)
regionTreeNode.children.append(azTreeNode)
newAZ = AvailabilityZone(az['azName'], az['distance'],
regionName, azTreeNode,
regionTreeNode)
newAZ.holdVMs = vmsInAz
else:
newAZ = AvailabilityZone(az['azName'], az['distance'],
regionName)
azList.append(newAZ)
#如果region下面的az都没有vm，则说明本application还没有部署到本region下的az里，本region对于本application的可用性为0
if not regionTreeNode.children:
regionTreeNode.availability = 0
self.cloudRoot = cloudRoot
self.azList = sorted(azList, key=attrgetter('distance'))
def getThreadInfo(self, req):
tarId = req.params.get('id')
if not tarId:
return errorResultJson('Please pass id!')
if PMAndAZDBUtil.isPMId(tarId):
tarIP = PMAndAZDBUtil.getInnerIPByPMId(tarId)
if not tarIP:
return errorResultJson(tarId + ' cannot be found')
os.system('/home/sk/cloudEx/shellScript/getThreadInfo.sh ' + tarIP.split('.')[-1] + ' > /dev/null')
df = open('/home/sk/cloudEx/tmpData/threadInfo.data')
info = df.read()
df.close()
else:
tarIP = UsingInstancesDBUtil.getUsingInstanceInnerIPById(tarId)
if not tarIP:
return errorResultJson(tarId + ' cannot be found')
params = urllib.urlencode({'ip': tarIP})
headers = {"Content-type": "application/x-www-form-urlencoded",
"Accept": "text/plain"}
httpClient = httplib.HTTPConnection("202.120.40.20", 50020, timeout=300)
httpClient.request("POST", "/getSpecificThreadInfo", params, headers)
response = httpClient.getresponse()
info = response.read()
httpClient.close()
infoList = eval(info)
res = []
for item in infoList:
res.append({'thread_cnt': item[0], 'standard_freq': item[1], 'current_freq': item[2]})
return res
def resetAllUsingInstances():
usingInstancesIds = UsingInstancesDBUtil.getAllUsingInstancesIds()
for uiid in usingInstancesIds:
InstanceUtil.rebootInstanceById(uiid)
def createTomcatInstance(azName='', fixedIp=''):
nova = GetClientUtil.getNovaClient()
targetInstanceName = instanceNamePrefix + TomcatInstanceUtil.getUUIDStr(
)
targetImage = ImageUtil.findImageByName(imageName)
targetFlavor = FlavorUtil.findFlavorByName(flavorName)
if azName:
if fixedIp:
targetInstance = nova.servers.create(targetInstanceName,
targetImage,
targetFlavor,
availability_zone=azName,
nics=[{
'net-id':
netId,
'v4-fixed-ip':
fixedIp
}])
else:
targetInstance = nova.servers.create(targetInstanceName,
targetImage,
targetFlavor,
availability_zone=azName,
nics=[{
'net-id': netId
}])
else:
if fixedIp:
targetInstance = nova.servers.create(targetInstanceName,
targetImage,
targetFlavor,
nics=[{
'net-id':
netId,
'v4-fixed-ip':
fixedIp
}])
else:
targetInstance = nova.servers.create(targetInstanceName,
targetImage,
targetFlavor,
nics=[{
'net-id': netId
}])
# print targetInstance.name
ips = nova.servers.ips(targetInstance.id)
while not ips:
ips = nova.servers.ips(targetInstance.id)
# print ips
innerIP = TomcatInstanceUtil.getInnerIPFromIPInfo(ips)
count = 0
while True:
try:
targetInstance = nova.servers.get(targetInstance.id)
realAZName = targetInstance.__dict__[
'OS-EXT-AZ:availability_zone']
except KeyError:
count += 1
if count >= 50:
raise Exception('get az info failure!')
else:
continue
else:
break
UsingInstancesDBUtil.addUsingInstance(targetInstance.id,
targetInstance.name,
1,
innerIP,
azName=realAZName)
return targetInstance
#!/usr/bin/env python
# encoding: utf-8
from ACRCUtil.ACRCPlacementComponent import ACRCPlacementComponent
from DBUtil.UsingInstancesDBUtil import UsingInstancesDBUtil
a = ACRCPlacementComponent()
countList = range(20)
for i in countList:
a.getPlacementScheme(1, True)
uic = UsingInstancesDBUtil.getUsingInstancesCount()
print 'vm numbers:' + str(uic) + ' av:' + str(a.calculateAvailability())
for i in countList:
a.getPlacementScheme(1, False)
uic = UsingInstancesDBUtil.getUsingInstancesCount()
print 'vm numbers:' + str(uic) + ' av:' + str(a.calculateAvailability())
def getPMOrVMUtil(self, req):
tarId = req.params.get('id')
periodNo = req.params.get('periodNo')
if tarId and periodNo and isNumber(periodNo):
periodNo = int(periodNo)
# 是物理机
if PMAndAZDBUtil.isPMId(tarId):
# log
cpuLogger.info('pmId:' + tarId + ' periodNo:' + str(periodNo))
tarIP = PMAndAZDBUtil.getInnerIPByPMId(tarId)
if not tarIP:
return errorResultJson(tarId + ' cannot be found')
os.system('/home/sk/cloudEx/shellScript/getPMUtil.sh ' + tarIP.split('.')[-1] + ' > /dev/null')
df = open('/home/sk/cloudEx/tmpData/result.data')
line = df.readline()
pmCPUUtil = round(float(line), 4) * 100
df.close()
if periodNo == 1:
PMCPUDBUtil.addFirstPeriodRealPMCPU(pmCPUUtil, tarId)
else:
PMCPUDBUtil.addRealPMCPUToSpecificPeriod(periodNo, tarId, pmCPUUtil)
addPMCPUUtilToPeriodicWindow(tarId, pmCPUUtil)
ppi = PMCPUPredictUtil()
pv = ppi.getNextPeriodWorkload(tarId)
addPredictPMCPUUtilToPeriodicWindow(tarId, pv)
PMCPUDBUtil.addPredictPMCPUToSpecificPeriod(periodNo + 1, pv, tarId)
# 是虚拟机
else:
tarIP = UsingInstancesDBUtil.getUsingInstanceInnerIPById(tarId)
# log
cpuLogger.info('vmIP:' + tarIP + ' periodNo:' + str(periodNo))
if not tarIP:
return errorResultJson(tarId + ' cannot be found')
rv = round(SampleUtil.getCpuUtilPeriodAVGByResourceId(tarId), 2)
if periodNo == 1:
VMCPUDBUtil.addFirstPeriodRealVMCPU(rv, tarIP)
else:
VMCPUDBUtil.addRealVMCPUToSpecificPeriod(periodNo, rv, tarIP)
addVMCPUUtilToPeriodicWindow(tarIP, rv)
pvi = VMCPUPredictUtil()
pv = pvi.getNextPeriodWorkload(tarIP)
addPredictVMCPUUtilToPeriodicWindow(tarIP, pv)
VMCPUDBUtil.addPredictVMCPUToSpecificPeriod(periodNo + 1, pv, tarIP)
return {'util': pv}
else:
return errorResultJson('Please pass the vmId or pmId!')
def getNumberOfVMsShouldBeScaled(self):
if not self.predictor or not self.ruleChecker:
raise Exception('no predictor or ruleChecker')
newestHD = WorkloadDBUtil.getNewestWorkload()
if not newestHD:
raise Exception('Can not get newest workload')
addWLToACRCWindow(newestHD['realWL'])
predictWL = self.predictor.getNextPeriodWorkload()
addPWLToACRCWindow(predictWL)
# 预测写入Workload表
periodNoDB = shelve.open(periodRecoderFile)
periodNo = periodNoDB.get(periodRecoder, None)
if not periodNo:
raise Exception('Can not get periodNo')
# do linear regression in given windowSize
# TODO: temporarily use default windowSize = sys.maxint
pairs = WorkloadDBUtil.getRealWorkloadAndRealTotalCalculationInPairs()
wl = pairs[0]
tc = pairs[1]
# TODO: do not fix the interception temporarily, might let the line cross the origin later
regression = np.polyfit(wl, tc, 1)
k = float(regression[0])
b = float(regression[1])
if k < 0 or wl.__len__() == 1:
# actually k should be positive, here to eliminate corner case when there's not much log
logger.warning("k = " + str(k) +
" is negative or length of log is 0!")
k = tc[-1] / wl[-1]
b = 0
logger.warning("using k = " + str(k) + " b = 0 instead")
predictVMNumbers = math.ceil(
k * predictWL /
(TomcatInstanceUtil.getCalculationCapacityPerInstance() - b))
predictTC = k * predictWL + predictVMNumbers * b
WorkloadDBUtil.addPredictWorkloadAndPredictTotalCalculationToSpecificPeriod(
periodNo, predictWL, predictTC)
# 记录供给的虚拟机数目的分布
provisionInfoDB = shelve.open(provisionInfoFile)
provisionInfoDB[predictProvisionVMNumbers] = predictVMNumbers
logger.info('PredictWorkload:' + str(predictWL) +
' predictVMNumbers:' + str(predictVMNumbers) +
' predictTC: ' + str(predictTC))
addedVMNumbers = self.ruleChecker.getNextAddedVMs()
provisionInfoDB[reactiveProvisionVMNumbers] = addedVMNumbers
logger.info('addedVMNumbers:' + str(addedVMNumbers))
provisionInfoDB.close()
nextPeriodVMNumbers = predictVMNumbers + addedVMNumbers
usingInstancesCount = UsingInstancesDBUtil.getUsingInstancesCount()
gab = abs(usingInstancesCount - nextPeriodVMNumbers)
if usingInstancesCount > nextPeriodVMNumbers:
rt = {'vmNumbers': gab, 'isUp': False}
elif usingInstancesCount < nextPeriodVMNumbers:
rt = {'vmNumbers': gab, 'isUp': True}
else:
rt = {'vmNumbers': gab}
logger.info('return with:' + str(rt))
return rt
def periodPerformanceDataHandler2(self, req):
minResponseTime = req.params.get('minResponseTime')
avgResponseTime = req.params.get('avgResponseTime')
maxResponseTime = req.params.get('maxResponseTime')
totalRequestCount = req.params.get('totalRequestCount')
breakSLACount = req.params.get('breakSLACount')
jumpServer = req.params.get('jumpServer')
infos = SampleUtil.getThreadInfosOverAllUsingInstances(jumpServer)
avgCpuUtil = round(infos[0] / 100.0, 4)
totalCalculation = round(infos[1] / 100.0, 4)
avgMemoryUtil = round(SampleUtil.getAllUsingInstancesPeriodAVGMemoryUtil() / 100.0, 4)
if not (isDecimal(minResponseTime) or isNumber(minResponseTime)) \
or not (isDecimal(maxResponseTime) or isNumber(maxResponseTime)) \
or not (isDecimal(avgResponseTime) or isNumber(avgResponseTime)) \
or not isNumber(totalRequestCount) \
or not isNumber(breakSLACount):
return errorResultJson('Please pass the params correctly')
elif avgCpuUtil == None or avgMemoryUtil == None or totalCalculation == None:
raise Exception("can not get avgCpuUtil or avgMemoryUtil or totalCalculation data")
else:
minResponseTime = float(minResponseTime)
avgResponseTime = float(avgResponseTime)
maxResponseTime = float(maxResponseTime)
totalRequestCount = int(totalRequestCount)
breakSLACount = int(breakSLACount)
# 确认periodNo
periodNoDB = shelve.open(periodRecoderFile)
periodNo = periodNoDB.get(periodRecoder, None)
if not periodNo:
periodNo = 1
periodNoDB[periodRecoder] = periodNo + 1
periodNoDB.close()
# 计算breakSLAPercent
breakSLAPercent = float(breakSLACount) / totalRequestCount
breakSLAPercent = round(breakSLAPercent, 4)
# 计算刚刚过去的这个周期的可用性
placementTool = ACRCPlacementComponent()
availabilityData = placementTool.calculateAvailability()
# 得到刚刚过去这个周期的虚拟机数目
vmNumbers = UsingInstancesDBUtil.getUsingInstancesCount()
# 添加上个周期应该提供的虚拟机数目
shouldVMNumbers = WorkloadVMMapDBUtil.getTargetVMsToSpecificWorkload(totalRequestCount)
if periodNo == 1:
ppVMNumbers = vmNumbers
rpVMNumbers = 0
else:
provisionInfoDB = shelve.open(provisionInfoFile)
ppVMNumbers = provisionInfoDB.get(predictProvisionVMNumbers, None)
rpVMNumbers = provisionInfoDB.get(reactiveProvisionVMNumbers, None)
# 添加performanceData
performanceData = {'minResponseTime': minResponseTime, 'maxResponseTime': maxResponseTime,
'avgResponseTime': avgResponseTime, 'breakSLAPercent': breakSLAPercent,
'avgCpuUtil': avgCpuUtil, 'avgMemoryUtil': avgMemoryUtil,
'availability': availabilityData, 'vmNumbers': vmNumbers,
'shouldVMNumbers': shouldVMNumbers, 'predictProvisionVMNumbers': ppVMNumbers,
'reactiveProvisionVMNumbers': rpVMNumbers}
PerformanceDBUtil.addPerformanceDataToSpecificPeriod(periodNo, performanceData)
# 向数据库中添加workload信息
if periodNo == 1:
WorkloadDBUtil.addFirstPeriodRealWorkloadAndRealTotalCalculation(totalRequestCount, totalCalculation)
else:
WorkloadDBUtil.addRealWorkloadAndRealTotalCalculationToSpecificPeriod(periodNo, totalRequestCount,
totalCalculation)
acrCtl = ACRController()
acrCtl.autonomicPeriodHandler()
TomcatInstanceUtil.ensureAllUsingInstancesActive()
return UsingInstancesDBUtil.getAllUsingInstancesInfo()