def handleTimeoutRule(timeoutRule):
printTimeoutRule(timeoutRule)
for i, targetRule in enumerate(Globals.TARGETRULES):
childRules = IPRules.findChildRules(Globals.TRANSITRULES, targetRule['ip'], targetRule['wild'])
allDone = True
for j, childRule in enumerate(childRules):
if childRule['ip'] == timeoutRule['ip'] and childRule['wild'] == timeoutRule['wild']:
Globals.TRANSITRULES.remove(timeoutRule)
timeoutRule['replica'] = targetRule['replica']
Globals.TRANSITRULES.append(timeoutRule)
elif childRule['replica'] != targetRule['replica']:
allDone = False
if allDone and childRules != []:
Globals.ASSIGNLOG.write("Moving to Installed: " + str(targetRule['ip']) + "/" + str(targetRule['wild']) + ":" + str(targetRule['replica']) + "\n")
childRules = IPRules.findChildRules(Globals.TRANSITRULES, targetRule['ip'], targetRule['wild'])
for j, childRule in enumerate(childRules):
Globals.TRANSITRULES.remove(childRule)
actionDeleteRule(childRule)
newChildren = IPRules.genChildrenRules(targetRule)
newChildren[0]['replica'] = targetRule['replica']
newChildren[1]['replica'] = targetRule['replica']
newChildren[0]['traffic'] = long(targetRule['traffic'] / 2)
newChildren[1]['traffic'] = long(targetRule['traffic'] / 2)
Globals.INSTALLEDRULES.append(newChildren[0])
Globals.INSTALLEDRULES.append(newChildren[1])
actionInstallRule(newChildren[0])
actionInstallRule(newChildren[1])
def printTargetInstallPairs(targetList, installList, transitList):
Globals.TARGETLOG.write('Installed:\n')
for i, target in enumerate(targetList):
childRules = IPRules.findChildRules(installList, target['ip'],
target['wild'])
if childRules != []:
Globals.TARGETLOG.write(
str(target['ip']) + "/" + str(target['wild']) + " -> " +
str(target['replica']) + ":" + str(target['traffic']) + '\n')
for j, rule in enumerate(childRules):
Globals.TARGETLOG.write('\t' + str(rule['ip']) + "/" +
str(rule['wild']) + " -> " +
str(rule['replica']) + ":" +
str(rule['traffic']) + '\n')
Globals.TARGETLOG.write('Transition:\n')
for i, target in enumerate(targetList):
childRules = IPRules.findChildRules(transitList, target['ip'],
target['wild'])
if childRules != []:
Globals.TARGETLOG.write(
str(target['ip']) + "/" + str(target['wild']) + " -> " +
str(target['replica']) + ":" + str(target['traffic']) + '\n')
for j, rule in enumerate(childRules):
Globals.TARGETLOG.write('\t' + str(rule['ip']) + "/" +
str(rule['wild']) + " -> " +
str(rule['replica']) + ":" +
str(rule['traffic']) + '\n')
def determineTargets(targetList, alphas, deeperList):
totalTraffic = long(sumTraffic(targetList) / numNonZero(targetList))
newTargetList = []
oldTargetList = []
Globals.ASSIGNLOG.write('Threshold: ' + str(totalTraffic * Globals.MINTGTTHRESH) + " < x < " + str(totalTraffic * Globals.MAXTGTTHRESH) + '\n')
for i in range(0, len(targetList)):
if targetList[i]['traffic'] > (totalTraffic * Globals.MAXTGTTHRESH) or Alphas.sumAllAlphaAssign(alphas) < Alphas.sumAllAlphaTarget(alphas):
Globals.ASSIGNLOG.write('\t' + str(targetList[i]['ip']) + " Beyond Threshold " + str(targetList[i]['traffic']) + '\n')
# Beyond Threshold: Need more rules
childList = IPRules.findChildRules(deeperList, targetList[i]['ip'], targetList[i]['wild'])
for j, child in enumerate(childList):
newTargetList.append(child)
if childList != []:
oldTargetList.append(targetList[i])
elif targetList[i]['traffic'] < (totalTraffic * Globals.MINTGTTHRESH):
Globals.ASSIGNLOG.write('\t' + str(targetList[i]['ip']) + " Under Threshold " + str(targetList[i]['traffic']) + '\n')
# Below Theshold: Reduce rules
(siblingIP, siblingWild) = IPRules.findSiblingRule(targetList[i])
siblingRule = IPRules.findMatch(targetList, siblingIP, siblingWild)
if siblingRule != []:
if siblingRule[0]['traffic'] <= (totalTraffic * Globals.MINTGTTHRESH) and \
targetList.index(siblingRule[0]) < i:
parentRule = IPRules.getParentRule(siblingRule[0])
Globals.ASSIGNLOG.write('\t' + str(parentRule['ip']) + '/' + str(parentRule['wild']) + ' combined\n')
newTargetList.append(parentRule)
oldTargetList.append(targetList[i])
oldTargetList.append(siblingRule[0])
else:
Globals.ASSIGNLOG.write('\t' + str(targetList[i]['ip']) + " Fine Threshold " + str(targetList[i]['traffic']) + '\n')
if targetList[i]['replica'] == -1:
newTargetList.append(targetList[i])
oldTargetList.append(targetList[i])
return (oldTargetList, newTargetList)
def updateInstallTable():
for i, install in enumerate(Globals.INSTALLEDRULES):
install['traffic'] = 0L
for i, transit in enumerate(Globals.TRANSITRULES):
transit['traffic'] = 0L
for i, switch in enumerate(Globals.SWITCHES):
for j, client in enumerate(switch['clients']):
for k, install in enumerate(Globals.INSTALLEDRULES):
if IPRules.isChild(install, client['ip']):
# install['traffic'] += client['newCount']
# install['traffic'] = Globals.EMA_CONSTANT * client['newCount'] + (1 - Globals.EMA_CONSTANT) * install['traffic']
# install['traffic'] = long(Globals.EMA_CONSTANT * (client['newCount'] - client['oldCount']) + ((1 - Globals.EMA_CONSTANT) * install['traffic']))
install['traffic'] += client['avg']
for k, transit in enumerate(Globals.TRANSITRULES):
if IPRules.isChild(transit, client['ip']):
# transit['traffic'] += client['newCount']
# transit['traffic'] = Globals.EMA_CONSTANT * client['newCount'] + (1 - Globals.EMA_CONSTANT) * transit['traffic']
# transit['traffic'] = long(Globals.EMA_CONSTANT * (client['newCount'] - client['oldCount']) + (1 - Globals.EMA_CONSTANT) * transit['traffic'])
transit['traffic'] += client['avg']
for i, target in enumerate(Globals.TARGETRULES):
sumTraffic = 0
for j, install in enumerate(Globals.INSTALLEDRULES):
if IPRules.isChild(target, install['ip']):
sumTraffic += install['traffic']
for j, transit in enumerate(Globals.TRANSITRULES):
if IPRules.isChild(target, transit['ip']):
sumTraffic += transit['traffic']
target['traffic'] = sumTraffic
def findReplicaAssignment(srcIP, targetList, transitList):
newTarget = 0
oldTarget = 0
for i, target in enumerate(targetList):
if IPRules.isChild(target, srcIP):
newTarget = target['replica']
for i, transit in enumerate(transitList):
if IPRules.isChild(transit, srcIP):
oldTarget = transit['replica']
return (newTarget, oldTarget)
def findReplicaAssignment(srcIP, targetList, transitList):
newTarget = 0
oldTarget = 0
for i, target in enumerate(targetList):
if IPRules.isChild(target, srcIP):
newTarget = target['replica']
for i, transit in enumerate(transitList):
if IPRules.isChild(transit, srcIP):
oldTarget = transit['replica']
return (newTarget, oldTarget)
def printTargetInstallPairs(targetList, installList, transitList):
Globals.TARGETLOG.write('Installed:\n')
for i, target in enumerate(targetList):
childRules = IPRules.findChildRules(installList, target['ip'], target['wild'])
if childRules != []:
Globals.TARGETLOG.write(str(target['ip']) + "/" + str(target['wild']) + " -> " + str(target['replica']) + ":" + str(target['traffic']) + '\n')
for j, rule in enumerate(childRules):
Globals.TARGETLOG.write('\t' + str(rule['ip']) + "/" + str(rule['wild']) + " -> " + str(rule['replica']) + ":" + str(rule['traffic']) + '\n')
Globals.TARGETLOG.write('Transition:\n')
for i, target in enumerate(targetList):
childRules = IPRules.findChildRules(transitList, target['ip'], target['wild'])
if childRules != []:
Globals.TARGETLOG.write(str(target['ip']) + "/" + str(target['wild']) + " -> " + str(target['replica']) + ":" + str(target['traffic']) + '\n')
for j, rule in enumerate(childRules):
Globals.TARGETLOG.write('\t' + str(rule['ip']) + "/" + str(rule['wild']) + " -> " + str(rule['replica']) + ":" + str(rule['traffic']) + '\n')
def redistributeTargets(oldTargetRules, oldAlphas, newAlphas, oldTargetList, newTargetList):
totalTraffic = sumTraffic(oldTargetRules)
for i, oldAlpha in enumerate(oldAlphas):
for j, newAlpha in enumerate(newAlphas):
if oldAlpha['replica'] == newAlpha['replica']:
overAssigns = 0
if oldAlpha['alphaTarget'] == 0:
overAssigns = oldAlpha['alphaAssign'] - (Globals.MAXTGTTHRESH * totalTraffic * newAlpha['alphaTarget'])
else:
overAssigns = oldAlpha['alphaAssign'] - (Globals.MAXTGTTHRESH * totalTraffic * newAlpha['alphaTarget'] / oldAlpha['alphaTarget'])
if overAssigns > 0:
for k, target in enumerate(oldTargetRules):
parentRule = IPRules.getParentRule(target)
matchParentRule = IPRules.findMatch(newTargetList, parentRule['ip'], parentRule['wild'])
matchTargetRule = IPRules.findMatch(newTargetList, target['ip'], target['wild'])
childrenRule = IPRules.genChildrenRules(target)
matchChild1Rule = IPRules.findMatch(newTargetList, childrenRule[0]['ip'], childrenRule[0]['wild'])
matchChild2Rule = IPRules.findMatch(newTargetList, childrenRule[1]['ip'], childrenRule[1]['wild'])
if target['replica'] == oldAlpha['replica'] and overAssigns > 0 and matchTargetRule == [] and matchChild1Rule == [] and matchChild2Rule == [] and matchParentRule == [] and target['traffic'] != 0:
Globals.ASSIGNLOG.write("Redistributing: " + str(target['ip']) + '/' + str(target['wild']) + ':' + str(target['replica']) + '\n')
oldTargetList.append(target)
newTargetList.append(target)
break
return (oldTargetList, newTargetList)
def handleTimeoutRule(timeoutRule):
printTimeoutRule(timeoutRule)
for i, targetRule in enumerate(Globals.TARGETRULES):
childRules = IPRules.findChildRules(Globals.TRANSITRULES,
targetRule['ip'],
targetRule['wild'])
allDone = True
for j, childRule in enumerate(childRules):
if childRule['ip'] == timeoutRule['ip'] and childRule[
'wild'] == timeoutRule['wild']:
Globals.TRANSITRULES.remove(timeoutRule)
timeoutRule['replica'] = targetRule['replica']
Globals.TRANSITRULES.append(timeoutRule)
elif childRule['replica'] != targetRule['replica']:
allDone = False
if allDone and childRules != []:
Globals.ASSIGNLOG.write("Moving to Installed: " +
str(targetRule['ip']) + "/" +
str(targetRule['wild']) + ":" +
str(targetRule['replica']) + "\n")
childRules = IPRules.findChildRules(Globals.TRANSITRULES,
targetRule['ip'],
targetRule['wild'])
for j, childRule in enumerate(childRules):
Globals.TRANSITRULES.remove(childRule)
actionDeleteRule(childRule)
newChildren = IPRules.genChildrenRules(targetRule)
newChildren[0]['replica'] = targetRule['replica']
newChildren[1]['replica'] = targetRule['replica']
newChildren[0]['traffic'] = long(targetRule['traffic'] / 2)
newChildren[1]['traffic'] = long(targetRule['traffic'] / 2)
Globals.INSTALLEDRULES.append(newChildren[0])
Globals.INSTALLEDRULES.append(newChildren[1])
actionInstallRule(newChildren[0])
actionInstallRule(newChildren[1])
def __init__(self, router=None, locale=None, interface=None, hint=None,
service_type=None, preference=100,poolid=0, spares=0, product=None, status="SPARE", blocksize=None, debug=0):
self.reduced=0 # if 1, pool reduction has been run
self.router=router
self.status=status
self.locale=locale
self.interface=interface
self.service_type=service_type
self.preference=int(preference)
self.pools=None
self.poollist=[] # Collection of IPPool objects
self.poolcount=0
self.poolid=int(poolid)
self.product=product
self.rules=None
self.hint=hint
self.debug=debug
if self.debug>2: print "In AutoIP, product:", self.product
if self.product:
self.db = dbconnect.dbConnect(dbc=DBHANDLE)
# Apply a ruleset to given product
try:
self.rules=IPRules.ProductRules(self.product)
except:
raise IPPoolError, (20, "Rules Violation, Unknown product")
# Check if have required criteria for current ruleset
# if self.rules.PR_locale and not self.locale:
# raise IPPoolError, (13, "Rules Violation, Must supply a locale")
# if self.rules.PR_router and not self.router:
# raise IPPoolError, (13, "Rules Violation, Must supply a router")
# if self.rules.PR_service_type and not self.service_type:
# raise IPPoolError, (13, "Rules Violation, Must supply a service_type")
# if self.rules.PR_interface and not self.interface:
# raise IPPoolError, (13, "Rules Violation, Must supply a interface")
self.getpools(poolid=self.poolid, spares=spares, status=status)
def updateAlphas():
oldTargetRules = copyTargets(Globals.TARGETRULES)
oldInstallRules = copyTargets(Globals.INSTALLEDRULES)
newAlphas = Alphas.getAlphas(Globals.ALPHAFILE)
Globals.log.info("NEW ALPHAS " + str(newAlphas))
Globals.log.info("OLD ALPHAS" + str(Globals.ALPHAS))
(oldTargetList, newTargetList) = Bins.determineTargets(oldTargetRules, Globals.ALPHAS, oldInstallRules)
(oldTargetList, newTargetList) = Bins.redistributeTargets(oldTargetRules, Globals.ALPHAS, newAlphas, oldTargetList, newTargetList)
# Bins.printTargetList(newTargetList)
totalAssignments = len(newTargetList)
Globals.log.info("Total Assigns: " + str(totalAssignments))
newAlphas = Alphas.distributeAlphas(newAlphas, Globals.ALPHAS, totalAssignments)
Globals.log.info("DISTRIBUTE NEW ALPHAS " + str(newAlphas))
newAlphas = Alphas.distributeEven(newAlphas, totalAssignments)
# newAlphas = Alphas.distributeOld(newAlphas, Globals.ALPHAS, totalAssignments)
Globals.log.info("DISTRIBUTE OLD ALPHAS " + str(newAlphas))
newAlphas = Alphas.distributeRemaining(newAlphas, totalAssignments)
Globals.log.info("REMAINING NEW ALPHAS " + str(newAlphas))
Alphas.updateAlphas(newAlphas, oldTargetList)
newTargetList = IPRules.assignReplicas(newAlphas, newTargetList)
Bins.printTargetList(newTargetList)
newRuleList = IPRules.getNewRules(Globals.TARGETRULES, newTargetList)
IPRules.printRules(newRuleList)
# (oldRules, newRules) = IPRules.redistributeRules(Globals.TARGETRULES, Globals.ALPHAS, newAlphas)
# newRuleList.extend(newRules)
# IPRules.updateAssigns(newRuleList, oldRules)
IPRules.updateAssigns(newRuleList)
Alphas.printAlphas(Globals.ALPHAS)
return newRuleList
def handleRules(rulesList):
for i, rule in enumerate(rulesList):
newRules = rule['NewRules']
oldRules = rule['OldRules']
# 1:1 means simply replacement! Never should be N:N
if len(newRules) == len(oldRules):
oldRule = oldRules[0]
newRule = newRules[0]
oldIP = oldRule['ip']
oldWild = oldRule['wild']
oldTraffic = oldRule['traffic']
newIP = newRule['ip']
newWild = newRule['wild']
newReplica = newRule['replica']
Globals.TARGETRULES.remove(oldRule)
oldInstalledChildren = IPRules.findChildRules(
Globals.INSTALLEDRULES, oldIP, oldWild)
for i, oldChildRule in enumerate(oldInstalledChildren):
newChildRule = copyRule(oldChildRule)
newChildRule['replica'] = newReplica
actionReplaceRule([oldChildRule], [newChildRule])
newRule['traffic'] = oldTraffic
Globals.TARGETRULES.append(newRule)
# 1:2 install more specific targets
elif len(newRules) > len(oldRules):
oldRule = oldRules[0]
newRule1 = newRules[0]
newRule2 = newRules[1]
oldIP = oldRule['ip']
oldWild = oldRule['wild']
oldReplica = oldRule['replica']
oldTraffic = oldRule['traffic']
newIP1 = newRule1['ip']
newWild1 = newRule1['wild']
newReplica1 = newRule1['replica']
newTraffic1 = long(oldTraffic / 2)
newIP2 = newRule2['ip']
newWild2 = newRule2['wild']
newReplica2 = newRule2['replica']
newTraffic2 = long(oldTraffic / 2)
Globals.TARGETRULES.remove(oldRule)
oldInstalledChildren = IPRules.findChildRules(
Globals.INSTALLEDRULES, oldIP, oldWild)
for i, rule in enumerate(oldInstalledChildren):
newChildren = IPRules.genChildrenRules(rule)
if IPRules.findMatch([IPRules.getParentRule(newChildren[0])],
newIP1, newWild1) != []:
newChildren[0]['replica'] = newReplica1
newChildren[1]['replica'] = newReplica1
else:
newChildren[0]['replica'] = newReplica2
newChildren[1]['replica'] = newReplica2
newChildren[0]['traffic'] = long(rule['traffic'] / 2)
newChildren[1]['traffic'] = long(rule['traffic'] / 2)
rule['replica'] = oldReplica
actionReplaceRule([rule], newChildren)
Globals.TARGETRULES.append(newRule1)
Globals.TARGETRULES.append(newRule2)
# 2:1 install more general targets
else:
newRule = newRules[0]
newIP = newRule['ip']
newWild = newRule['wild']
newReplica = newRule['replica']
oldRule1 = oldRules[0]
oldRule2 = oldRules[1]
oldIP1 = oldRule1['ip']
oldWild1 = oldRule1['wild']
oldTraffic1 = oldRule1['traffic']
oldIP2 = oldRule2['ip']
oldWild2 = oldRule2['wild']
oldTraffic2 = oldRule2['traffic']
Globals.TARGETRULES.remove(oldRule1)
Globals.TARGETRULES.remove(oldRule2)
oldInstalledChildren1 = IPRules.findChildRules(
Globals.INSTALLEDRULES, oldIP1, oldWild1)
oldTransitionChildren1 = IPRules.findChildRules(
Globals.TRANSITRULES, oldIP1, oldWild1)
oldInstalledChildren1.extend(oldTransitionChildren1)
oldRule1['replica'] = newReplica
actionReplaceRule(oldInstalledChildren1, [oldRule1])
oldInstalledChildren2 = IPRules.findChildRules(
Globals.INSTALLEDRULES, oldIP2, oldWild2)
oldTransitionChildren2 = IPRules.findChildRules(
Globals.TRANSITRULES, oldIP2, oldWild2)
oldInstalledChildren2.extend(oldTransitionChildren2)
oldRule2['replica'] = newReplica
actionReplaceRule(oldInstalledChildren2, [oldRule2])
newRule['traffic'] = long(oldTraffic1 + oldTraffic2)
Globals.TARGETRULES.append(newRule)
def handleRules(rulesList):
for i, rule in enumerate(rulesList):
newRules = rule['NewRules']
oldRules = rule['OldRules']
# 1:1 means simply replacement! Never should be N:N
if len(newRules) == len(oldRules):
oldRule = oldRules[0]
newRule = newRules[0]
oldIP = oldRule['ip']
oldWild = oldRule['wild']
oldTraffic = oldRule['traffic']
newIP = newRule['ip']
newWild = newRule['wild']
newReplica = newRule['replica']
Globals.TARGETRULES.remove(oldRule)
oldInstalledChildren = IPRules.findChildRules(Globals.INSTALLEDRULES, oldIP, oldWild)
for i, oldChildRule in enumerate(oldInstalledChildren):
newChildRule = copyRule(oldChildRule)
newChildRule['replica'] = newReplica
actionReplaceRule([oldChildRule], [newChildRule])
newRule['traffic'] = oldTraffic
Globals.TARGETRULES.append(newRule)
# 1:2 install more specific targets
elif len(newRules) > len(oldRules):
oldRule = oldRules[0]
newRule1 = newRules[0]
newRule2 = newRules[1]
oldIP = oldRule['ip']
oldWild = oldRule['wild']
oldReplica = oldRule['replica']
oldTraffic = oldRule['traffic']
newIP1 = newRule1['ip']
newWild1 = newRule1['wild']
newReplica1 = newRule1['replica']
newTraffic1 = long(oldTraffic / 2)
newIP2 = newRule2['ip']
newWild2 = newRule2['wild']
newReplica2 = newRule2['replica']
newTraffic2 = long(oldTraffic / 2)
Globals.TARGETRULES.remove(oldRule)
oldInstalledChildren = IPRules.findChildRules(Globals.INSTALLEDRULES, oldIP, oldWild)
for i, rule in enumerate(oldInstalledChildren):
newChildren = IPRules.genChildrenRules(rule)
if IPRules.findMatch([IPRules.getParentRule(newChildren[0])], newIP1, newWild1) != []:
newChildren[0]['replica'] = newReplica1
newChildren[1]['replica'] = newReplica1
else:
newChildren[0]['replica'] = newReplica2
newChildren[1]['replica'] = newReplica2
newChildren[0]['traffic'] = long(rule['traffic'] / 2)
newChildren[1]['traffic'] = long(rule['traffic'] / 2)
rule['replica'] = oldReplica
actionReplaceRule([rule], newChildren)
Globals.TARGETRULES.append(newRule1)
Globals.TARGETRULES.append(newRule2)
# 2:1 install more general targets
else:
newRule = newRules[0]
newIP = newRule['ip']
newWild = newRule['wild']
newReplica = newRule['replica']
oldRule1 = oldRules[0]
oldRule2 = oldRules[1]
oldIP1 = oldRule1['ip']
oldWild1 = oldRule1['wild']
oldTraffic1 = oldRule1['traffic']
oldIP2 = oldRule2['ip']
oldWild2 = oldRule2['wild']
oldTraffic2 = oldRule2['traffic']
Globals.TARGETRULES.remove(oldRule1)
Globals.TARGETRULES.remove(oldRule2)
oldInstalledChildren1 = IPRules.findChildRules(Globals.INSTALLEDRULES, oldIP1, oldWild1)
oldTransitionChildren1 = IPRules.findChildRules(Globals.TRANSITRULES, oldIP1, oldWild1)
oldInstalledChildren1.extend(oldTransitionChildren1)
oldRule1['replica'] = newReplica
actionReplaceRule(oldInstalledChildren1, [oldRule1])
oldInstalledChildren2 = IPRules.findChildRules(Globals.INSTALLEDRULES, oldIP2, oldWild2)
oldTransitionChildren2 = IPRules.findChildRules(Globals.TRANSITRULES, oldIP2, oldWild2)
oldInstalledChildren2.extend(oldTransitionChildren2)
oldRule2['replica'] = newReplica
actionReplaceRule(oldInstalledChildren2, [oldRule2])
newRule['traffic'] = long(oldTraffic1 + oldTraffic2)
Globals.TARGETRULES.append(newRule)
