예제 #1
0
	def assemble(self, subtrees):
		'''this is nice. to be used by k-best tree generation.'''
		t = Tree(self.label, self.span, subs=subtrees, sym=False) if not self._spurious else subtrees[0]
			
		assert t is not None, (self.label, self.span, subtrees, self._spurious)
		if self._root:
			## notice that, roots are spurious! so...
			t.set_root(True)
		return t
예제 #2
0
def build_aggregate_tree(options,args,fields,dim,types,dict_dim, otfa = lambda x,y: x.get_root(),randomize=False,no_final_aggregation=False,from_gui=True):
    
    tree_src = None
    tree_dst = None
   
    list_res = []
    list_window = []
    tree = None
    f = open(options.input) 
    lines = [ l for l in f]
    f.close()
    
    
    total_count = 0.0
    current_window = 0
    lineno = 0
    for line in lines:
        print lineno
        lineno+=1
        find = re.search(options.reg_exp,line)

        if find:

            dict_fields = {}
            
            for i in range(len(fields)):
                dict_fields[fields[i]] = find.group(i+1)
            try:
                sec = time.mktime(time.strptime(dict_fields["timestamp"],"%Y-%m-%d %H:%M:%S"))
            except:
                sec = int(dict_fields["timestamp"])

            if sec >= current_window + options.window: 
                if tree != None:
                    list_res.append((tree,total_count))
                    tree = Tree(dim)
                    current_window = current_window + options.window
                    
                    while sec>= current_window + options.window:
                        list_res.append((None,0.0))
                        list_window.append(current_window)
                        current_window = current_window + options.window
                    
                else:
                    tree = Tree(dim)
                    current_window = sec
                total_count = 0.0
                
                list_window.append(current_window)
            
            update_tree(tree,dict_fields,dict_dim,options.type_aggr)
            total_count+= float(dict_fields[VALUE])
            
            #SKIP HERE
            if 1 == 0:
                try:    
                    update_tree(tree,dict_fields,dict_dim,options.type_aggr)
                    #print tree,dict_fields,dict_dim,options.type_aggr
                except Exception, e:
                    print e
                    raise
                    tree.set_root(tree.get_root().post_aggregate())
                    update_tree(tree,dict_fields,dict_dim,options.type_aggr)
                
                total_nodes = len(tree.get_root().preorder())
                if total_nodes > options.max_nodes:
                    
                    tree.set_root(otfa(tree,total_count))
                    tree.set_root(tree.get_root().post_aggregate())
                    
                    #aggregate_LRU(tree,options.max_nodes,options.aggregate,total_count)
                
                #print tree.get_root().preorder()
                #tree.increase_age_tree()
                
                total_count+= float(dict_fields[VALUE])
                
                map(lambda x: x.increase_age(),tree.get_root().preorder())
예제 #3
0
def benchmark_aggregation(options,args,fields,dim,types,dict_dim, otfa = lambda x: x.get_root()):
    
    tree_src = None
    tree_dst = None
   
    list_res = []
    list_window = []
    tree = None
    f = open(options.input) 
    
    total_count = 0.0
    current_window = 0
    tree = Tree(dim)
    k = 0
    for line in f:
        find = re.search(options.reg_exp,line)
        if find:
            dict_fields = {}
            for i in range(len(fields)):
                dict_fields[fields[i]] = find.group(i+1)
            #print dict_fields
            try:
                sec = time.mktime(time.strptime(dict_fields["timestamp"],"%Y-%m-%d %H:%M:%S"))
            except:
                sec = int(dict_fields["timestamp"])

            
            
            list_window.append(current_window)
            
            try:    
                update_tree(tree,dict_fields,dict_dim,options.type_aggr)
            except:
                tree.set_root(tree.get_root().post_aggregate())
                update_tree(tree,dict_fields,dict_dim,options.type_aggr)
            total_nodes = len(tree.get_root().preorder())
            if total_nodes > options.max_nodes:
                tree.set_root(otfa(tree,total_count))
                tree.set_root(tree.get_root().post_aggregate())
                #aggregate_LRU(tree,options.max_nodes,options.aggregate,total_count)
            
            #print tree.get_root().preorder()
            #tree.increase_age_tree()
            
            total_count+= float(dict_fields[VALUE])
            
            map(lambda x: x.increase_age(),tree.get_root().preorder())
        
            
        
        k=k+1
        
        if options.max_lines > 0 and options.max_lines < k :
            break    
    
    if tree.get_root() != None:
        list_res.append((tree,total_count))
    
            
    pretotal_nodes_before_aggregation = len(tree.get_root().preorder())
    tree.aggregate(options.aggregate,total_count)
       
            
    print "Total nodes before pre order aggregation %s"%pretotal_nodes_before_aggregation
    print "Total nodes after aggregation %s"%len(tree.get_root().preorder())
예제 #4
0
def build_stability_aggregation_trees(options,args,fields,dim,types,dict_dim, otfa = lambda x,y: x.get_root(),randomize=False,no_final_aggregation=False):
    
    #TODO: remove nof_final aggregtion parameter and replace it by the test below
    if not options.aggregate>0:
        no_final_aggregation = True
    
    tree_src = None
    tree_dst = None
   
    list_res = []
    list_window = []
    tree = None
    f = open(options.input) 
    lines = [ l for l in f]
    f.close()
    
    if randomize: random.shuffle(lines)
    
    total_count = 0.0
    current_window = 0
    
    for line in lines:
        #print line
        find = re.search(options.reg_exp,line)
        #print find
        if find:
            dict_fields = {}
            for i in range(len(fields)):
                dict_fields[fields[i]] = find.group(i+1)
            #print dict_fields
            try:
                #    print dict_fields["timestamp"]
                sec = time.mktime(time.strptime(dict_fields["timestamp"],"%Y-%m-%d %H:%M:%S"))
            except:
                sec = int(dict_fields["timestamp"])

            if sec >= current_window + options.window: 
                if tree != None:
                    list_res.append((tree,total_count))
                    tree = Tree(dim)
                    current_window = current_window + options.window
                    
                    while sec>= current_window + options.window:
                        list_res.append((None,0.0))
                        list_window.append(current_window)
                        current_window = current_window + options.window
                    
                else:
                    tree = Tree(dim)
                    current_window = sec
                total_count = 0.0
                
                list_window.append(current_window)
                
            try:    
                update_tree(tree,dict_fields,dict_dim,options.type_aggr)
            except Exception, e:
                print e
                raise
                tree.set_root(tree.get_root().post_aggregate())
                update_tree(tree,dict_fields,dict_dim,options.type_aggr)
            total_nodes = len(tree.get_root().preorder())
            if total_nodes > options.max_nodes:
                tree.set_root(otfa(tree,total_count))
                tree.set_root(tree.get_root().post_aggregate())
                #aggregate_LRU(tree,options.max_nodes,options.aggregate,total_count)
            
            #print tree.get_root().preorder()
            #tree.increase_age_tree()
            
            total_count+= float(dict_fields[VALUE])
            
            map(lambda x: x.increase_age(),tree.get_root().preorder())