def cmd_add_queue(interface, queueId, rate):
if rate < 100: # if < 100K bits per second, just let it go with 100Kbps
rate = 100
command1 = "tc class add dev {0} parent 1:fffe classid 1:{1} htb rate {2}kbit".format(interface, queueId, rate)
command2 = "tc filter add dev {0} protocol ip parent 1: prio 0 handle {1} fw flowid 1:{1}".format(interface, queueId)
call(command1, shell=True, executable='/bin/bash')
call(command2, shell=True, executable='/bin/bash')
LogUtil.DebugLog('control', 'done cmd_add_queue', command1, command2)
def _initTrafficControl():
dataPlaneIp = _getDataPlaneIp()
command = "ifconfig | grep -B1 {0} | head -n1 | cut -b1-6".format(dataPlaneIp)
global _interface
_interface = check_output(command, shell=True, executable='/bin/bash')
_interface = _interface.split(' ')[0]
LogUtil.DebugLog('control', 'the data plane interface', _interface)
cmd_clear_rule(_interface)
cmd_add_qdisc(_interface)
cmd_add_root(_interface)
global _isInitialized
_isInitialized = True
def main():
if len(sys.argv) < 3:
print 'Provide controller IP address and port'
print 'Usage: python agentRun.py controllerIP controllerPort'
sys.exit()
LogUtil.InitLogging()
ctrlAddress = sys.argv[1]
ctrlPort = sys.argv[2]
if ctrlAddress != 'localhost':
try:
struct.unpack('I', socket.inet_pton(socket.AF_INET, ctrlAddress))
except socket.error, msg:
print 'Controller IP Address Invalid: ' + ctrlAddress
sys.exit()
def AddConnToJob(connCriterion, jobId):
app = None
saddr = None
sport = None
daddr = None
dport = None
for (key, value) in connCriterion.iteritems():
if key == 'app':
app = value
elif key == 'srcIP':
saddr = value
elif key == 'srcPort':
sport = value
elif key == 'dstIP':
daddr = value
elif key == 'dstPort':
dport = value
connKey = str((app, saddr, sport, daddr, dport))
if connKey not in _connsInJob:
_connsInJob[connKey] = jobId
(queueId, _) = JobAndQueueRate[jobId]
iptablesRule = 'OUTPUT '
if saddr:
iptablesRule += '-s {0} '.format(saddr)
if daddr:
iptablesRule += '-d {0} '.format(daddr)
iptablesRule += '-p tcp '
if sport:
iptablesRule += '--sport {0} '.format(sport)
if dport:
iptablesRule += '--dport {0} '.format(dport)
#if app:
#pid = check_output('ps aux | grep '+app+' | head -n 1 | cut -b 10-14', shell=True, executable='/bin/bash')
#pid = pid.lstrip().rstrip('\n')
#iptablesRule.append('-m owner --pid-owner '+pid)
iptablesRule += '-j MARK --set-mark {0}'.format(queueId)
LogUtil.DebugLog('control', 'new iptables rule', iptablesRule)
cmd_add_rule(iptablesRule)
def cmd_show_class(interface, className):
command = "tc -s class show dev {0} | grep {1}".format(interface, className)
call(command, shell=True, executable='/bin/bash')
LogUtil.DebugLog('control', 'done cmd_show_class', command)
def cmd_del_queue(interface, queueId):
command = "tc class del dev {0} parent 1:fffe classid 1:{1} &> /dev/null".format(interface, queueId)
call(command, shell=True, executable='/bin/bash')
LogUtil.DebugLog('control', 'done cmd_del_queue', command)
def cmd_add_rule(rule):
command = "iptables -A {0}".format(rule)
call(command, shell=True, executable='/bin/bash')
LogUtil.DebugLog('control', 'done cmd_add_rule', command)
def cmd_modify_queue(interface, queueId, rate):
if rate < 100:
rate = 100
command = "tc class change dev {0} parent 1:fffe classid 1:{1} htb rate {2}kbit".format(interface, queueId, rate)
call(command, shell=True, executable='/bin/bash')
LogUtil.DebugLog('control', 'done cmd_modify_queue', command)
def cmd_add_root(interface):
command = "tc class add dev {0} parent 1: classid 1:fffe htb rate 1000mbit".format(interface)
call(command, shell=True, executable='/bin/bash')
LogUtil.DebugLog('control', 'done cmd_add_root', command)
def cmd_add_qdisc(interface):
command = "tc qdisc add dev {0} root handle 1: htb".format(interface)
call(command, shell=True, executable='/bin/bash')
LogUtil.DebugLog('control', 'done cmd_add_gdisc', command)
def cmd_clear_rule(interface):
call("tc qdisc del dev {0} root &> /dev/null".format(interface), shell=True, executable='/bin/bash')
call("iptables --flush &> /dev/null", shell=True, executable='/bin/bash')
LogUtil.DebugLog('control', 'done cmd_clear_rule')
def agentManagerRun(ctrlAddress, ctrlPort):
LogUtil.DebugLog('manager', 'start to run agent manager')
#EvalLog('{0:6f},50,agentManager starts in pid {1}'.format(time.time(), os.getpid()))
global CtrlAddress
CtrlAddress = ctrlAddress
global sourceJobTable
sourceJobTable = {}
global socketTable
socketTable = {}
global sourceJobSkList
sourceJobSkList = {}
global minRunInterval
sourceJobQueue = Queue()
socketCriteriaQueue = Queue()
rcvModuleProcess = RcvModuleProcess(sourceJobQueue, socketCriteriaQueue)
dirServiceProcess = DirServiceProcess(socketCriteriaQueue, sourceJobQueue)
try:
#EvalLog('{0:6f},51,start rcvModule and dirService processes'.format(time.time()))
rcvModuleProcess.start()
dirServiceProcess.start()
# wait for rcv module to boot
#EvalLog('{0:6f},52,wait {1} seconds for rcvModule to boot'.format(time.time(), minRunInterval))
time.sleep(minRunInterval)
global sndToCtrl
sndToCtrl = HostAgentSndSocket(ctrlAddress, ctrlPort)
scheduleLoop = Timer(minRunInterval, scheduleLoopRun)
scheduleLoop.start()
time.sleep(minRunInterval / 2.0)
while True:
#EvalLog('{0:6f},53,enter main loop to process new source job'.format(time.time()))
currentTime = time.time()
while not sourceJobQueue.empty():
#EvalLog('{0:6f},54,got one new item from IPC queue'.format(time.time()))
(itemType, itemContent) = sourceJobQueue.get_nowait()
if itemType == IPCType['NewSourceJob']:
newSourceJob = itemContent
#debugLog('job', 'new source job', newSourceJob.debug())
registerComputePart(newSourceJob)
key = composeKey(newSourceJob.jobId, newSourceJob.flowId)
if (len(sourceJobTable)
== 0) or (minRunInterval >
(newSourceJob.period / 1000.0)):
minRunInterval = newSourceJob.period / 1000.0
alreadyPassedPeriods = int(
float(currentTime - newSourceJob.createTime) /
float(newSourceJob.period / 1000.0))
newSourceJob.deadline = newSourceJob.createTime + newSourceJob.period / 1000.0 * (
alreadyPassedPeriods + 1)
newSourceJob.lastSequence = alreadyPassedPeriods
jobFlowQueue.push(newSourceJob.deadline, key)
sourceJobTable[key] = newSourceJob
#EvalLog('{0:6f},55,process one new source job done'.format(time.time()))
elif itemType == IPCType['UpdateSourceJob']:
(jobId, middleAddress) = itemContent
for key in sourceJobTable.iterkeys():
if keyContainJobId(key, jobId):
sourceJobTable[key].middleAddress = middleAddress
break
#EvalLog('{0:6f},112,update middle address {1} for source job {2}'.format(time.time(), middleAddress, jobId))
elif itemType == IPCType['NewSocket']:
(sockfd, theSock) = itemContent
socketTable[sockfd] = theSock
elif itemType == IPCType['DeleteSocket']:
if itemContent in socketTable:
del socketTable[itemContent]
elif itemType == IPCType['AddSkToJobFlow']:
(jobFlow, sockfd) = itemContent
if jobFlow not in sourceJobSkList:
sourceJobSkList[jobFlow] = []
if sockfd not in sourceJobSkList[jobFlow]:
sourceJobSkList[jobFlow].append(sockfd)
elif itemType == IPCType['RemoveSkFromJobFlow']:
(jobFlow, sockfd) = itemContent
if (jobFlow in sourceJobSkList) and (
sockfd in sourceJobSkList[jobFlow]):
sourceJobSkList[jobFlow].remove(sockfd)
elif itemType == IPCType['UpdateControlJob']:
(jobId, newAction) = itemContent
if newAction[0][0] == 'RateLimit':
if jobId in agentRateLimitModule.JobAndQueueRate:
newRate = int(newAction[0][1])
agentRateLimitModule.UpdateRateLimitJob(
jobId, newRate)
#debugLog('manager', 'one round of main loop')
#for id in sourceJobTable.keys():
#debugLog('manager', 'jobId@flowId: ', id, 'sourceJob: ', sourceJobTable[id].debug())
#debugLog('manager', 'socketTable', socketTable)
#debugLog('manager', 'sourceJobSkList', sourceJobSkList)
#debugLog('manager', 'time: ', time.time())
#EvalLog('{0:6f},56,done one round of main loop. start to sleep {1} seconds'.format(time.time(), minRunInterval))
time.sleep(minRunInterval)
except KeyboardInterrupt:
logging.info('Catch KeyboardInterrupt in agent manager main loop')
print 'Catch KeyboardInterrupt in agent manager main loop'
#EvalLog('{0:6f},57,catch keyboard interrupt. start to clean up'.format(time.time()))
finally:
#EvalLog('{0:6f},58,ready to exit agent manager. wait {1} seconds for other processes to end'.format(time.time(), minRunInterval))
global stopSchedule
stopSchedule = True
agentProcMeasure.cleanup()
dirServiceProcess.terminate()
dirServiceProcess.join()
rcvModuleProcess.terminate()
rcvModuleProcess.join()
logging.info(
'Agent manager cleans everything up, and is ready to stop.')
print 'Exit from agent manager'
def scheduleLoopRun():
#debugLog('schedule', 'schedule loop', time.time())
#EvalLog('{0:6f},59,start one round of schedule loop'.format(time.time()))
global evalTimestamp
global measureLatency
LogUtil.EvalLog('JobExecutionLoop', evalTimestamp)
LogUtil.EvalLog('MeasureLatency', measureLatency)
evalTimestamp = 'Begin${0:6f}'.format(time.time())
measureLatency = 'Begin${0:6f}'.format(time.time())
if stopSchedule:
#debugLog('schedule', 'catch stop signal')
#EvalLog('{0:6f},60,schedule loop catches stop signal and exit'.format(time.time()))
logging.info('schedule loop should stop now.')
return
# next loop
currentTime = time.time()
if jobFlowQueue.minDeadline():
delta = currentTime - jobFlowQueue.minDeadline() - 0.1
else:
delta = 0.0
if delta < minRunInterval:
nextLoop = Timer((minRunInterval - delta), scheduleLoopRun)
else:
nextLoop = Timer(minRunInterval, scheduleLoopRun)
nextLoop.start()
# schedule jobs to run
jobFlowToRun = []
connMeasureJobFlow = []
procMeasureJobFlow = []
machineMeasureJobFlow = []
#debugLog('schedule', 'current time:', currentTime, 'jobFlowQueue:', jobFlowQueue.debug())
while (not jobFlowQueue.isEmpty()) and (jobFlowQueue.minDeadline() <
currentTime):
jobFlowKey = jobFlowQueue.pop()
jobFlowToRun.append(jobFlowKey)
measureType = sourceJobTable[jobFlowKey].measureType
if measureType == 'conn':
connMeasureJobFlow.append(jobFlowKey)
elif measureType == 'proc':
procMeasureJobFlow.append(jobFlowKey)
elif measureType == 'machine':
machineMeasureJobFlow.append(jobFlowKey)
sourceJobTable[jobFlowKey].updateDeadline()
jobFlowQueue.push(sourceJobTable[jobFlowKey].deadline, jobFlowKey)
#EvalLog('{0:6f},61,done preparing jobFlows to run: {1}'.format(time.time(), jobFlowToRun))
#debugLog('schedule', 'job flow to run: ', jobFlowToRun, \
# 'connMeasureJobFlow:', connMeasureJobFlow, \
# 'procMeasureJobFlow:', procMeasureJobFlow, \
# 'machineMeasureJobFlow:', machineMeasureJobFlow)
evalTimestamp += '#ScheduleDone${0:6f}'.format(time.time())
if IsLazyTableEnabled():
if connMeasureJobFlow:
connThread = Thread(target=agentConnMeasure.connMeasureRun,
args=(connMeasureJobFlow, None))
connThread.daemon = True
connThread.start()
if procMeasureJobFlow:
procThread = Thread(target=agentProcMeasure.procMeasureRun,
args=(procMeasureJobFlow, None))
procThread.daemon = True
procThread.start()
if machineMeasureJobFlow:
machineThread = Thread(target=agentProcMeasure.machineMeasureRun,
args=(machineMeasureJobFlow, None))
machineThread.daemon = True
machineThread.start()
else:
#EvalLog('{0:6f},109,start no-lazy-m operation'.format(time.time()))
connThread = Thread(target=agentConnMeasure.connMeasureRun,
args=(connMeasureJobFlow, None))
connThread.daemon = True
procThread = Thread(target=agentProcMeasure.procMeasureRunAll,
args=(procMeasureJobFlow, None))
procThread.daemon = True
machineThread = Thread(target=agentProcMeasure.machineMeasureRun,
args=(machineMeasureJobFlow, None))
machineThread.daemon = True
connThread.start()
procThread.start()
machineThread.start()
connThread.join()
procThread.join()
machineThread.join()