def run(self):
self.closeConnEvent.wait()
zeroPoint = getCurrentTime() + 5.0
pid = os.fork()
if pid == 0:
st = 0.0
else:
myPID = os.getpid()
childPID = pid
os.system("taskset -p 0x00000001 %d" % (myPID))
os.system("taskset -p 0x00000002 %d" % (childPID))
#subprocess.Popen(shlex.split("taskset -p 0x00000001 %d"%(myPID)))
#subprocess.Popen(shlex.split("taskset -p 0x00000004 %d"%(childPID)))
st = 0.0 + self.frequency
plan = TaskPlan(self.workType, st, self.usageTime, self.frequency * 2,
self.argv)
Timer(0.0, plan.runScenario, (zeroPoint, )).start()
Timer(zeroPoint + plan.getLastTime() + 10 - getCurrentTime(),
plan.dummy, ()).start()
if pid != 0: #parent
os.wait()
def runNextGroup(self, startIndex):
for i in range(startIndex, startIndex + 10): #10=concurrent_size
if i < len(self.workList):
Timer(self.workList[i][0] + self.zeroPoint - getCurrentTime(),
self.do_work, (i, )).start()
lastIndex = i
if lastIndex + 1 < len(self.workList):
Timer(
self.workList[lastIndex][0] - 2 + self.zeroPoint -
getCurrentTime(), self.runNextGroup,
(lastIndex + 1, )).start()
def finish(self):
self.finishTime = getCurrentTime()
aFile = open(setting.TIME_LOG_FILE, 'a')
aFile.write("%s %.7f %.7f %.7f %s\n" %
(self.workType, self.startTime, self.finishTime,
self.finishTime - self.startTime, json.dumps(self.argv)))
aFile.close()
def do_work(self, i):
#real work from self.workList[i]
print self.ip, "%.2f" % (getCurrentTime() -
self.zeroPoint), self.workList[i]
work = self.workList[i]
if work[1] == 'start':
globalIP = cacheFile.getGlobalControllerIP()
if fixHost and work[2] != None:
result = requestTo(
globalIP, setting.API_PORT,
'/guest/startWithIP?guestIP=%s&targetHostID=%s' %
(str(self.ip), work[2]))
else:
result = requestTo(
globalIP, setting.API_PORT,
'/guest/startWithIP?guestIP=%s' % (str(self.ip)))
elif work[1] == 'stop':
globalIP = cacheFile.getGlobalControllerIP()
result = requestTo(
globalIP, setting.API_PORT,
'/guest/forceOffWithIP?guestIP=%s' % (str(self.ip)))
else:
#result=connection.socketCall(self.ip,setting.LOCAL_PORT,work[1])
result = connection.socketCall(
self.ip, setting.LOCAL_PORT, 'set_work', [
json.dumps([work[1], work[3], work[0], work[2]]),
json.dumps([]), '{socket_connection}'
])
self.totalWork += 1
if result == 'OK':
pass
else: #error
self.errorWork += 1
def logNetworkUsage():
result = network.getInterfaceUsage('eth0')
rx = result['rx']
tx = result['tx']
currentTime = getCurrentTime()
logFile = open('/maekin/var/networkLog.txt', 'a')
logFile.write('%.7f %s %s\n' % (currentTime, rx, tx))
logFile.close()
print 'log network data at', currentTime
def printFinishTime():
print "finishTime(+20) = %.7f" % (getCurrentTime())
print "Conclusion"
for guest in guestList:
if guest.totalWork == 0:
print "%s error=%d total=%d" % (guest.ip, guest.errorWork,
guest.totalWork)
else:
print "%s error=%d total=%d (%f percent error)" % (
guest.ip, guest.errorWork, guest.totalWork,
float(guest.errorWork) * 100.0 / float(guest.totalWork))
def run(self):
#config time
result = subprocess.Popen(shlex.split("hwclock -s -u"))
result.wait()
#put vm start time log file
logfile = open(setting.TIME_LOG_FILE, 'a')
logfile.write("open %.7f\n" % (getCurrentTime()))
logfile.close()
#open socket and wait for command
connection.runServer(mainPort=setting.LOCAL_PORT,
commandDict=getCommandDict(
loadMethod, transferMethod, manageMethod))
def runMonitor():
"""
conn = libvirt.open(None)
if conn == None:
print 'Failed to open connection to the hypervisor'
return
"""
cpu_count=multiprocessing.cpu_count()
conn=None #process in loop will fix it later
pre={}
counter=0
try:
while True:
print counter
counter+=1
try:
domains = conn.listDomainsID()
except:
print "listDomainsID error"
time.sleep(1)
#loop until i can get new connection
print "Try to connect to hypervisor"
while True:
try:
conn = libvirt.open(None)
if conn!=None:
print "Can connect to hypervisor"
break
else:
print '.'
time.sleep(1)
except:
print "an error happen"
time.sleep(1)
continue
freshUUID=[]
for id in domains:
try:
dom=conn.lookupByID(id)
except:
print "lookupByID error"
break
#print dom.name(),dom.UUIDString()
infoString=''
######TIME ZONE######
if not (id in pre.keys()):
firstTime=True
pre[id]={'timestamp':getCurrentTime()}
else:
firstTime=False
newTimestamp=getCurrentTime()
timeRange=newTimestamp-pre[id]['timestamp']
pre[id]['timestamp']=newTimestamp
infoString+="UPDATE:%s\n"%(str(newTimestamp))
#######CPU ZONE#########
try:
if not ('cpuTime' in pre[id].keys()):
pre[id]['cpuTime']=dom.info()[4]
else:
newCpuTime=dom.info()[4]
cpuTimeRange=newCpuTime-pre[id]['cpuTime']
#percentCPU=(cpuTimeRange*100.0)/(timeRange*1000.0*1000.0*1000.0*dom.info()[3])
percentCPU=(cpuTimeRange*100.0)/(timeRange*1000.0*1000.0*1000.0*cpu_count)
pre[id]['cpuTime']=newCpuTime
#print "cpuUsage",max(0.0, min(100.0, percentCPU))
infoString+="CPU_USAGE:%s\n"%(str(max(0.0, min(100.0, percentCPU))))
infoString+="CPU_TIME:%s\n"%(str(newCpuTime))
except:
print 'ERROR at CPU ZONE'
#######IF ZONE#########
try:
ifPath=getInterfacePath(dom)
if ifPath:
ifStat=dom.interfaceStats(ifPath)
rxBytes=ifStat[0]
txBytes=ifStat[4]
if not ('rxBytes' in pre[id].keys()):
pre[id]['rxBytes']=rxBytes
pre[id]['txBytes']=txBytes
else:
rxRange=rxBytes-pre[id]['rxBytes']
txRange=txBytes-pre[id]['txBytes']
rateRx=rxRange/timeRange #Bytes per sec
rateTx=txRange/timeRange
pre[id]['rxBytes']=rxBytes
pre[id]['txBytes']=txBytes
#print "rateRx",rateRx,"rateTx",rateTx
infoString+="INTERFACE_RX_RATE:%s\n"%(str(rateRx))
infoString+="INTERFACE_TX_RATE:%s\n"%(str(rateTx))
infoString+="INTERFACE_RX_USED:%s\n"%(str(rxBytes))
infoString+="INTERFACE_TX_USED:%s\n"%(str(txBytes))
else:
pass #do nothing about interface
except:
print 'ERROR at INTERFACE ZONE'
#######IO ZONE########
try:
diskPath=getDiskPath(dom)
if diskPath:
diskStat=dom.blockStats(diskPath)
readBytes=diskStat[1]
writeBytes=diskStat[3]
if not ('readBytes' in pre[id].keys()):
pre[id]['readBytes']=readBytes
pre[id]['writeBytes']=writeBytes
else:
readRange=readBytes-pre[id]['readBytes']
writeRange=writeBytes-pre[id]['writeBytes']
rateRead=readRange/timeRange
rateWrite=writeRange/timeRange
pre[id]['readBytes']=readBytes
pre[id]['writeBytes']=writeBytes
#print "rateRead",rateRead,"rateWrite",rateWrite
infoString+="DISK_READ_RATE:%s\n"%(str(rateRead))
infoString+="DISK_WRITE_RATE:%s\n"%(str(rateWrite))
infoString+="DISK_READ_USED:%s\n"%(str(readBytes))
infoString+="DISK_WRITE_USED:%s\n"%(str(writeBytes))
else:
pass #do nothing about disk io
except:
print 'ERROR at DISK_IO ZONE'
#print "================================================="
#write data to the file
if firstTime==False:
targetFile=open(setting.CURRENT_INFO_PATH+dom.UUIDString().replace('-','')+'.info','w')
targetFile.write(infoString)
targetFile.close()
freshUUID.append(dom.UUIDString().replace('-','')+'.info')
#clear old file
for afile in os.listdir(setting.CURRENT_INFO_PATH):
if afile.endswith(".info") and (afile not in freshUUID):
try:
os.remove(setting.CURRENT_INFO_PATH+afile)
except:
continue
time.sleep(setting.GUEST_MONITOR_PERIOD)
except:
print "errorerrorerrorerrorerror"
traceback.print_exc()
def runCollector():
jiffyPerSec = float(os.sysconf(os.sysconf_names['SC_CLK_TCK']))
cpu_count = int(multiprocessing.cpu_count())
myMAC = network.getMyMACAddr()
myIP = network.getMyIPAddr()
while myIP == None:
time.sleep(1.5)
myIP = network.getMyIPAddr()
i_am_global_controller = False #it will be changed during working
hostID = None #will be know while working
isHost = None #will be know while working
timeCounter = -5 #start at 1 so everything will start slowly
while True:
#new system no need to check ip all the time
#if timeCounter%setting.IP_CHECKING_PERIOD == 0:
# myIP=network.getMyIPAddr()
# if myIP==None:
# myIP=network.renewIPAddr(allowStatic=True)
if timeCounter % setting.PLANNER_COLLECTING_PERIOD == 0:
#get host data
timestamp = getCurrentTime()
f = open('/proc/stat', 'r')
while True:
data = f.readline().split(' ')
if data[0] == 'cpu':
if '' in data:
data.remove('')
idleTime = float(data[4]) / jiffyPerSec
elif data[0] == 'btime':
btime = int(data[1])
break
f.close()
#get guest data (like mmond)
guestError = False
try:
conn = libvirt.open(None)
if conn == None:
print "cannot connect to hypervisor"
guestError = True
else:
domains = conn.listDomainsID()
guestData = []
for id in domains:
dom = conn.lookupByID(id)
guestData.append({
'UUID':
dom.UUIDString().replace('-', ''),
'timestamp':
getCurrentTime(),
'cpuTime':
float(dom.info()[4]) / float(1000000000),
})
except:
print "have error while getting guest load information"
guestError = True
try:
db = MySQLdb.connect(cacheFile.getDatabaseIP(),
setting.DB_USERNAME, setting.DB_PASSWORD,
setting.DB_NAME)
cursor = db.cursor()
if hostID == None:
cursor.execute(
"SELECT `hostID`,`isHost` FROM `hosts` WHERE `MACAddress`='%s'"
% (myMAC))
hostID, isHost = cursor.fetchone()
#put host data in database
cursor.execute(
"INSERT INTO `hostLoad` (`hostID`,`timestamp`,`idleTime`,`btime`) VALUES ('%s','%s','%s','%s')"
% (str(hostID), str(timestamp), str(idleTime), str(btime)))
#put guest data in database
haveActiveGuest = False
if guestError == False:
for element in guestData:
cursor.execute(
"SELECT `guestID` FROM `guests` WHERE `lastUUID`='%s'"
% (element['UUID']))
result = cursor.fetchone()
if result == None:
continue
else:
guestID = result[0]
cursor.execute(
"INSERT INTO `guestLoad` (`guestID`,`timestamp`,`cpuTime`) VALUES ('%s','%s','%s')"
% (str(guestID), str(element['timestamp']),
str(element['cpuTime'])))
haveActiveGuest = True
if haveActiveGuest and isHost == 0:
#should evacuate
allowEvacuate = True
cursor.execute(
"SELECT `detail` FROM `tasks` WHERE `opcode`=2005 AND `status`<>2"
)
tmpDetail = cursor.fetchone()
while tmpDetail != None:
detailDict = json.loads(tmpDetail[0])
if str(detailDict['hostID']) == str(hostID):
allowEvacuate = False
break
else:
tmpDetail = cursor.fetchone()
if allowEvacuate:
taskID = queue.enqueue({
'command': 'host_evacuate_mission',
'hostID': hostID
})
if taskID == None:
print "enqueue host_evacuate_mission error"
db.close()
except:
print "error: timeCounter=", timeCounter
traceback.print_exc()
if timeCounter % setting.LOG_CLEANING_PERIOD == 0 and i_am_global_controller:
#delete log in database that older than setting.LOG_CLEANING_PERIOD
thresholdTime = getCurrentTime() - float(
setting.LOG_CLEANING_PERIOD)
try:
db = MySQLdb.connect(cacheFile.getDatabaseIP(),
setting.DB_USERNAME, setting.DB_PASSWORD,
setting.DB_NAME)
cursor = db.cursor()
#check permission of moving old log to another table
cursor.execute(
"SELECT `value` FROM `cloud_variables` WHERE `key`='hold_all_log'"
)
if cursor.fetchone()[0] == '0':
#just delete it
cursor.execute(
"DELETE FROM `hostLoad` WHERE `timestamp`<'%s'" %
(str(thresholdTime)))
cursor.execute(
"DELETE FROM `guestLoad` WHERE `timestamp`<'%s'" %
(str(thresholdTime)))
else:
#must move log to another 2 tables before delete
cursor.execute(
"SELECT `hostLoadID`,`hostID`,`timestamp`,`idleTime`,`btime` FROM `hostLoad` WHERE `timestamp`<'%s'"
% (str(thresholdTime)))
maxID = 0
tmpData = cursor.fetchall()
valueList = []
for row in tmpData:
valueList.append("('%s','%s','%s','%s','%s')" % row)
if row[0] > maxID:
maxID = row[0]
if len(valueList) > 0:
cursor.execute(
"INSERT INTO `oldHostLoad` (`hostLoadID`,`hostID`,`timestamp`,`idleTime`,`btime`) VALUES %s"
% (','.join(valueList)))
cursor.execute(
"DELETE FROM `hostLoad` WHERE `hostLoadID`<=%s" %
(maxID))
###############
cursor.execute(
"SELECT `guestLoadID`,`guestID`,`timestamp`,`cpuTime` FROM `guestLoad` WHERE `timestamp`<'%s'"
% (str(thresholdTime)))
maxID = 0
tmpData = cursor.fetchall()
valueList = []
for row in tmpData:
valueList.append("('%s','%s','%s','%s')" % row)
if row[0] > maxID:
maxID = row[0]
if len(valueList) > 0:
cursor.execute(
"INSERT INTO `oldGuestLoad` (`guestLoadID`,`guestID`,`timestamp`,`cpuTime`) VALUES %s"
% (','.join(valueList)))
cursor.execute(
"DELETE FROM `guestLoad` WHERE `guestLoadID`<=%s" %
(maxID))
db.close()
except:
print "error: timeCounter=", timeCounter
traceback.print_exc()
if timeCounter % setting.PLANNER_ACTION_PERIOD in [
setting.PLANNER_ACTION_PERIOD / 2, 0
]:
#update i_am_global_controller
i_am_global_controller = (
str(myIP) == cacheFile.getGlobalControllerIP())
if timeCounter % setting.PLANNER_ACTION_PERIOD == setting.PLANNER_ACTION_PERIOD / 2 and i_am_global_controller:
#guest action here
#1.check auto_mode in database (can do in mode 1 and 2)
#2.check that old mission was end
#3.put guest balance mission in queue
canDo = False
try:
db = MySQLdb.connect(cacheFile.getDatabaseIP(),
setting.DB_USERNAME, setting.DB_PASSWORD,
setting.DB_NAME)
cursor = db.cursor()
cursor.execute(
"SELECT `value` FROM `cloud_variables` WHERE `key`='auto_mode'"
)
currentMode = cursor.fetchone()[0]
if currentMode in ['1', '2']:
#check old mission
cursor.execute(
"SELECT `taskID` FROM `tasks` WHERE `opcode`='4001' AND (`status`=0 OR `status`=1)"
)
if cursor.fetchone() == None:
canDo = True
db.close()
except:
print "error: timeCounter=", timeCounter
traceback.print_exc()
if canDo:
taskID = queue.enqueue({
'command': 'guest_balance_mission',
})
if taskID == None:
print "enqueue error"
if timeCounter % setting.PLANNER_ACTION_PERIOD == 0 and i_am_global_controller:
#host action here
#1.check auto_mode in database (can do in mode 2 only)
#2.check that old mission was end
#3.put host balance mission in queue
canDo = False
try:
db = MySQLdb.connect(cacheFile.getDatabaseIP(),
setting.DB_USERNAME, setting.DB_PASSWORD,
setting.DB_NAME)
cursor = db.cursor()
cursor.execute(
"SELECT `value` FROM `cloud_variables` WHERE `key`='auto_mode'"
)
currentMode = cursor.fetchone()[0]
if currentMode == '2':
#check old mission
cursor.execute(
"SELECT `taskID` FROM `tasks` WHERE `opcode`='4002' AND (`status`=0 OR `status`=1)"
)
if cursor.fetchone() == None:
canDo = True
db.close()
except:
print "error: timeCounter=", timeCounter
traceback.print_exc()
if canDo:
taskID = queue.enqueue({
'command': 'host_balance_mission',
})
if taskID == None:
print "enqueue error"
#print "\033[A\r"+"end:",timeCounter
#print "end:",timeCounter
timeCounter += 1
time.sleep(1)
def __init__(self, argv):
self.startTime = getCurrentTime()
self.workType = getOuterFunctionName()
self.argv = argv
elif part[0] == 'work':
currentGuest.addWork(part[1],
float(part[3]) * 60,
float(part[4]) * 60, float(part[2]))
elif part[0] == 'stop':
currentGuest.addStop(float(part[1]) * 60)
elif part[0] == 'end':
guestList.append(currentGuest)
currentGuest = None
else:
pass
maxLastTime = 0
for guest in guestList:
guest.sortTime()
if not guest.haveRightSequence():
print "please check", guest.ip
sys.exit()
if guest.getLastTime() > maxLastTime:
maxLastTime = guest.getLastTime()
# work will start now
print "finish reading, shooting is starting in 5 seconds..."
zeroPoint = getCurrentTime() + 5.0
print "startTime = %.7f" % (zeroPoint)
for guest in guestList:
Timer(0.1, guest.runScenario, (zeroPoint, )).start()
Timer(zeroPoint + maxLastTime + 20 - getCurrentTime(), printFinishTime,
()).start()
def getWeightData(hostList, timeRange=None):
'''
if hostList==None this method will replace hostList with all active hosts (with isHost=1)
'''
if timeRange == None:
timeRange = setting.WEIGHT_RANDOM_TIME
thresholdTimestamp = getCurrentTime() - timeRange
db = MySQLdb.connect(cacheFile.getDatabaseIP(), setting.DB_USERNAME,
setting.DB_PASSWORD, setting.DB_NAME)
cursor = db.cursor()
if hostList == None:
#must find all active host
cursor.execute(
"SELECT `hostID` FROM `hosts` WHERE `status`=1 AND `isHost`=1")
hostList = []
while True:
tmpHostID = cursor.fetchone()
if tmpHostID == None:
break
else:
hostList.append(tmpHostID[0])
hostList = list(hostList) #this line is very important, it's deep copy
strHostList = []
for tmpHostID in hostList:
strHostList.append(str(tmpHostID))
cursor.execute('''
SELECT `hostID`,`timestamp`,`idleTime`,`btime` FROM `hostLoad` WHERE `hostLoadID` IN
( SELECT MAX(`hostLoadID`)
FROM `hostLoad`
GROUP BY `hostID`
HAVING `hostID` IN (%s))''' % (','.join(strHostList)))
maxData = cursor.fetchall()
conclude = []
for element in maxData:
cursor.execute(
"SELECT `cpuCore`,`cpuSpeed` FROM `hosts` WHERE `hostID`=%s" %
(element[0]))
hostSpec = cursor.fetchone()
newDict = {
'hostID': element[0],
'maxTimestamp': element[1],
'maxIdleTime': element[2],
'cpuCore': hostSpec[0],
'cpuSpeed': hostSpec[1],
}
cursor.execute('''
SELECT `timestamp`,`idleTime` FROM `hostLoad` WHERE `hostLoadID` IN
( SELECT MIN(`hostLoadID`)
FROM `hostLoad`
WHERE `timestamp`>'%s' AND `btime`='%s'
GROUP BY `hostID`
HAVING `hostID`='%s')''' % (thresholdTimestamp, element[3], element[0]))
minData = cursor.fetchone()
if minData == None or minData[0] >= newDict['maxTimestamp']:
#this host is newly open (should send fake data out)
newDict['fake'] = True
newDict['minTimestamp'] = None
newDict['minIdleTime'] = None
newDict['percentRest'] = 0.0
newDict['resourceRest'] = 0.0
newDict['resourceFull'] = newDict['cpuSpeed'] * newDict['cpuCore']
else:
#real data
newDict['fake'] = False
newDict['minTimestamp'] = minData[0]
newDict['minIdleTime'] = minData[1]
newDict['percentRest'] = (
newDict['maxIdleTime'] - newDict['minIdleTime']) * 100.0 / (
(newDict['maxTimestamp'] - newDict['minTimestamp']) *
newDict['cpuCore'])
newDict['resourceRest'] = newDict['percentRest'] * newDict[
'cpuSpeed'] * newDict['cpuCore'] / 100.0
newDict['resourceFull'] = newDict['cpuSpeed'] * newDict['cpuCore']
conclude.append(newDict)
hostList.remove(newDict['hostID']
) #because this line, hostList should do deep copy
#fake the rest that have not any load data
for hostID in hostList: #rest from remove
cursor.execute(
"SELECT `cpuCore`,`cpuSpeed` FROM `hosts` WHERE `hostID`=%s" %
(hostID))
hostSpec = cursor.fetchone()
conclude.append({
'hostID': hostID,
'fake': True,
'cpuCore': hostSpec[0],
'cpuSpeed': hostSpec[1],
'percentRest': 0.0,
'resourceRest': 0.0,
'resourceFull': hostSpec[0] * hostSpec[1],
})
db.close()
print "@@@@@@@@@@@"
print conclude
return conclude
def getGuestLoadData(timeRange=None):
'''
for all active guest
'''
if timeRange == None:
timeRange = setting.GUEST_BALANCE_TIME
timeThreshold = getCurrentTime() - timeRange
infoHost = cacheFile.getDatabaseIP()
db = MySQLdb.connect(infoHost, setting.DB_USERNAME, setting.DB_PASSWORD,
setting.DB_NAME)
cursor = db.cursor()
cursor.execute(
"SELECT `guests`.`guestID`,`guests`.`lastHostID`,`hosts`.`cpuCore`,`hosts`.`cpuSpeed` FROM `guests` INNER JOIN `hosts` ON `guests`.`lastHostID`=`hosts`.`hostID` WHERE `guests`.`status`=1"
)
guestTmpData = cursor.fetchall()
conclude = []
for element in guestTmpData:
guestDict = {
'guestID': element[0],
'lastHostID': element[1],
'cpuCore': element[2],
'cpuSpeed': element[3],
}
cursor.execute(
"SELECT `timestamp`,`cpuTime` FROM `guestLoad` WHERE `timestamp`>'%s' AND `guestID`=%s"
% (timeThreshold, element[0]))
guestDict['minTimestamp'] = None
guestDict['maxTimestamp'] = None
guestDict['minCpuTime'] = None
guestDict['maxCpuTime'] = None
while True:
tmpData = cursor.fetchone()
if tmpData == None:
break
if guestDict['minTimestamp'] == None or tmpData[0] < guestDict[
'minTimestamp']:
guestDict['minTimestamp'] = tmpData[0]
guestDict['minCpuTime'] = tmpData[1]
if guestDict['maxTimestamp'] == None or tmpData[0] > guestDict[
'maxTimestamp']:
guestDict['maxTimestamp'] = tmpData[0]
guestDict['maxCpuTime'] = tmpData[1]
if guestDict['minTimestamp'] == guestDict['maxTimestamp'] or guestDict[
'minCpuTime'] > guestDict['maxCpuTime']:
#this is newly open guest (have not enough data to calculate)
#this guest will not be add in return list
continue
#calculate % and real load
guestDict['percentLoad'] = (
guestDict['maxCpuTime'] - guestDict['minCpuTime']) * 100.0 / (
guestDict['cpuCore'] *
(guestDict['maxTimestamp'] - guestDict['minTimestamp']))
guestDict['realLoad'] = guestDict['percentLoad'] * guestDict[
'cpuSpeed'] * guestDict['cpuCore']
conclude.append(guestDict)
db.close()
print "&&&&&&&&&&&&"
print conclude
return conclude