def generatePrefixRuleListND(self,listDecision,dictFieldRange, fmt,dictCost=None,getCode=False, listNodeOfInterest=None): ''' Generate multi-dimensional prefix rule list from the subtree starting from this node. fmt (format): prefix_overlap - predicates can overlap (calling the DP algorithm) prefix_nonoverlap - the predicates in the rules will be disjoint; range - predicate is in range format dictCost (optional): Dictionary to store the cost at each node. Key: the node (ADNode) of the tree. Value: the weight of the node times number of rules associated with the node. getCode (optional): If True, code returned is the dot code for visualizing the tree If False, code returned is '' listNodeOfInterest (optional): Only the rule list associated with the list of interested nodes are output to lnd. If None, all nodes in the subtree are of interest. Returns: -- A weight -- A list of dictionary, with each entry including the fields and decision. -- Code for plot the resulting dot file ''' '-----------code------------' code='' '-----------code------------' if self.hasChild(): #add children to dictionary dictNode=_generateDictNode(self.children) fieldRange=dictFieldRange[self.label] #sanity check range l,h=fieldRange assert(l==0) nBit=int(math.log(h+1,2)) assert(2**nBit-1==h) #iterate dictionary listRangeRule=[] listChildren=[] dictWeight={} dictLnd={} dictNewCode={} for c,rs in sorted(dictNode.items(), key=lambda x: x[1]): #sorted(dictNode.items(), key=lambda x: x[1]) #make sure sorting in order of values #print 'rs:' #print rs w,lnd,newCode=c.generatePrefixRuleListND(listDecision, dictFieldRange,fmt,dictCost=dictCost, getCode=getCode, listNodeOfInterest=listNodeOfInterest) listChildren.append(c) dictWeight[c]=w dictLnd[c]=lnd dictNewCode[c]=newCode for r in rs: appendRuleToRangeRuleList(listRangeRule,r,c) listCurrRule=[] if fmt=='prefix_overlap':#rules can overlap: use DP tree=PMTree(listRangeRule,listIsRange=True,fieldRange=fieldRange) #print listRangeRule #saveSvgFile(tree.getDotCode(),'../output/tree.svg',True) res=generateShortestPrefixRuleListFromTree(tree,nBit, listChildren,dictWeight) listCurrRule=res[1][0] elif fmt=='prefix_nonoverlap': for c,rs in sorted(dictNode.items(), key=lambda x: x[1]): #sorted(dictNode.items(), key=lambda x: x[1]) #make sure sorting in order of values ps=listRange2ListPrefix(rs,fieldRange) for p in ps: appendRuleToPrefixRuleList(listCurrRule,p,c) elif fmt=='range': for c,rs in sorted(dictNode.items(), key=lambda x: x[1]): #sorted(dictNode.items(), key=lambda x: x[1]) #make sure sorting in order of values for r in rs: appendRuleToPrefixRuleList(listCurrRule,r,c) # #sort the list, such that more specific rules (less *) are in front #20131008 result still correct if not used #listCurrRule=sorted(listCurrRule,key=lambda x: x['prefix'].count('*')) #count num rules for each child dictNRule={}#dict of decision (i.e. child) -> number of rules for rule in listCurrRule: c=rule['decision'] dictNRule.setdefault(c,0) dictNRule[c]+=1 #generate the new n-dim rule list via 'cross product' lnd=[] for rule in listCurrRule: c=rule['decision'] currLnd=dictLnd[c] for rule2 in currLnd: newRule=rule2.copy() ###newRule[self.label]=(prio,rule['prefix']) #20131008 remove redundant prio newRule[self.label]=rule['prefix'] if listNodeOfInterest is None: lnd.append(newRule) else: for n in listNodeOfInterest: if c==n or c.label!=n.label: lnd.append(newRule) #the weight of this node is the sum of child weight x child rules w=sum([dictWeight[c]*dictNRule[c] for c in listChildren]) #NOTE: above not particularly better than belows. do more test #w=sum([dictNRule[c] for c in listChildren]) #w=sum([dictWeight[c] for c in listChildren]) #w=1 if dictCost is not None: dictCost[self]=w #also update all the subtree nodes' cost by times the #rules for c in listChildren: c.propagateCostGain(dictNRule[c],dictCost) '-----------code------------' if getCode: code+=repr(self)+' [label="'+self.label+'",shape=oval];\n' #code+=repr(self)+' [label="'+self.label+' %d' % w+'",shape=oval];\n' '-----------code------------' '-----------code------------' if getCode: #record the rules (for generate dot code only) ###for iRule,rule in zip(range(len(listCurrRule)),listCurrRule): #20131008 remove redundant prio dictPrefixs={}#dict of decision (i.e. child) -> list of (priority,prefix) for rule in listCurrRule: c=rule['decision'] ###prio=nCurrRule-iRule-1 #20131008 remove redundant prio lr=dictPrefixs.setdefault(c,[]) ###lr.append((prio,rule['prefix'])) #20131008 remove redundant prio lr.append(rule['prefix']) for c,rs in sorted(dictNode.items(), key=lambda x: x[1]): #sorted(dictNode.items(), key=lambda x: x[1]) #make sure sorting in order of values code+=repr(c)+' [label="'+c.label+'"];\n' code+=repr(self)+' -> '+repr(c)+' [label="' nP=0 for res in dictPrefixs[c]: ###prio,prefix=res #20131008 remove redundant prio #prefix=res nP+=1 if nP<=2: if fmt=='range': code+='(%d-%d)' % (res[0],res[1]) else: ###code+='(%d:%s)' % (prio,prefix) #20131008 remove redundant prio #code+='(%s)' % prefix code+='(%s)' % res else: code+=' ...' break code+='"];\n' code+=dictNewCode[c] '-----------code------------' #print 'weight (non-terminal): %d' % w return (w,lnd,code) else:#is terminal node. assert(self.label in listDecision) w=1 if dictCost is not None: dictCost[self]=w #lnd=[{'decision':self.label}]#list n-dim lnd=[({'decision':self.label})]#list n-dim '-----------code------------' if getCode: code+=repr(self)+' [label="'+self.label+'",shape=box];\n' #code+=repr(self)+' [label="'+self.label+' %d' % w+'",shape=box];\n' '-----------code------------' #print 'weight (terminal): %d' % w return (w,lnd,code)
def _cost(prefix,tree,decision,listDecision,dictDecisionWeight,dictCost=None): ''' Compute the cost of a prefix in the tree with decision as the in the last rule of the corresponding rule list. dictCost: (prefix,decision) -> (minCost,minListRule) is an optional dictionary, that if used, stores previously computed costs to avoid repeated computation. Returns: the minimum cost and the minimum rule list Algorithm: dynamic programming in backward recursive implementation. Examples: >>> from acl.pmtree import PMTree >>> listPrefixRule=[{'prefix': '101***', 'decision': 'permit'}, {'prefix': '0*****', 'decision': 'permit'}, {'prefix': '******', 'decision': 'deny'}] >>> tree=PMTree(listPrefixRule) >>> listDecision=('deny', 'permit') >>> dictDecisionWeight={'deny':1,'permit':1} >>> dictCost={} >>> _cost('000000',tree,'deny',listDecision,dictDecisionWeight,dictCost=dictCost)[0] 2 >>> _cost('100000',tree,'deny',listDecision,dictDecisionWeight,dictCost=dictCost)[0] 1 >>> _cost('101000',tree,'deny',listDecision,dictDecisionWeight,dictCost=dictCost)[0] 2 >>> _cost('111000',tree,'deny',listDecision,dictDecisionWeight,dictCost=dictCost)[0] 1 >>> _cost('0*****',tree,'deny',listDecision,dictDecisionWeight,dictCost=dictCost)[0] 2 >>> _cost('11****',tree,'deny',listDecision,dictDecisionWeight,dictCost=dictCost)[0] 1 >>> _cost('10****',tree,'deny',listDecision,dictDecisionWeight,dictCost=dictCost)[0] 2 >>> _cost('1*****',tree,'deny',listDecision,dictDecisionWeight,dictCost=dictCost)[0] 2 >>> _cost('******',tree,'deny',listDecision,dictDecisionWeight,dictCost=dictCost)[0] 3 >>> _cost('******',tree,'permit',listDecision,dictDecisionWeight,dictCost=dictCost)[0] 3 ''' debugLevel=0 #if cost result already computed and stored, use it if dictCost is not None: res=dictCost.get((prefix,decision)) if res is not None: if debugLevel>0: print (prefix,decision)+res return res #otherwise, do recursive computation, and store result isConsistent,consistencyDecision=tree.isConsistentIn(prefix) if isConsistent:#baseline case if consistencyDecision==decision: minCost=dictDecisionWeight[consistencyDecision] minListRule=[] appendRuleToPrefixRuleList(minListRule,prefix,consistencyDecision) else: minCost=dictDecisionWeight[consistencyDecision]\ +dictDecisionWeight[decision] minListRule=[] appendRuleToPrefixRuleList(minListRule,prefix,consistencyDecision) appendRuleToPrefixRuleList(minListRule,prefix,decision) if debugLevel>1: print '**********************************' print 'prefix= '+prefix print 'decision= '+decision print 'consistencyDecision= '+consistencyDecision print '**********************************' else: minCost=float('inf') minListRule=[] for thisDecision in listDecision: prefixUnderscore=prefix.replace('*','0',1) prefixOverscore=prefix.replace('*','1',1) costUnderscore,listRuleUnderscore=_cost(prefixUnderscore,tree, thisDecision,listDecision,dictDecisionWeight,dictCost=dictCost) costOverscore, listRuleOverscore =_cost(prefixOverscore ,tree, thisDecision,listDecision,dictDecisionWeight,dictCost=dictCost) if thisDecision==decision: thisCost=(costUnderscore+costOverscore -dictDecisionWeight[thisDecision]) thisListRule=listRuleUnderscore[0:-1]+listRuleOverscore[0:-1] appendRuleToPrefixRuleList(thisListRule,prefix,thisDecision) else: thisCost=(costUnderscore+costOverscore -dictDecisionWeight[thisDecision])\ +dictDecisionWeight[decision] thisListRule=listRuleUnderscore[0:-1]+listRuleOverscore[0:-1] appendRuleToPrefixRuleList(thisListRule,prefix,thisDecision) appendRuleToPrefixRuleList(thisListRule,prefix,decision) if minCost>thisCost: minCost=thisCost minListRule=thisListRule minListRuleUnderscore=listRuleUnderscore minListRuleOverscore=listRuleOverscore if debugLevel>1: print '**********************************' print 'prefix= '+prefix print 'decision= '+decision print minListRuleUnderscore print minListRuleOverscore print '**********************************' if dictCost is not None: dictCost[(prefix,decision)]=(minCost,minListRule) # print '*******curr dictCost is:********' # print dictCost if debugLevel>0: print (prefix,decision)+(minCost,minListRule) return (minCost,minListRule)