def assemble(self, subtrees): '''this is nice. to be used by k-best tree generation.''' t = Tree(self.label, self.span, subs=subtrees, sym=False) if not self._spurious else subtrees[0] assert t is not None, (self.label, self.span, subtrees, self._spurious) if self._root: ## notice that, roots are spurious! so... t.set_root(True) return t
def build_aggregate_tree(options,args,fields,dim,types,dict_dim, otfa = lambda x,y: x.get_root(),randomize=False,no_final_aggregation=False,from_gui=True):
tree_src = None
tree_dst = None
list_res = []
list_window = []
tree = None
f = open(options.input)
lines = [ l for l in f]
f.close()
total_count = 0.0
current_window = 0
lineno = 0
for line in lines:
print lineno
lineno+=1
find = re.search(options.reg_exp,line)
if find:
dict_fields = {}
for i in range(len(fields)):
dict_fields[fields[i]] = find.group(i+1)
try:
sec = time.mktime(time.strptime(dict_fields["timestamp"],"%Y-%m-%d %H:%M:%S"))
except:
sec = int(dict_fields["timestamp"])
if sec >= current_window + options.window:
if tree != None:
list_res.append((tree,total_count))
tree = Tree(dim)
current_window = current_window + options.window
while sec>= current_window + options.window:
list_res.append((None,0.0))
list_window.append(current_window)
current_window = current_window + options.window
else:
tree = Tree(dim)
current_window = sec
total_count = 0.0
list_window.append(current_window)
update_tree(tree,dict_fields,dict_dim,options.type_aggr)
total_count+= float(dict_fields[VALUE])
#SKIP HERE
if 1 == 0:
try:
update_tree(tree,dict_fields,dict_dim,options.type_aggr)
#print tree,dict_fields,dict_dim,options.type_aggr
except Exception, e:
print e
raise
tree.set_root(tree.get_root().post_aggregate())
update_tree(tree,dict_fields,dict_dim,options.type_aggr)
total_nodes = len(tree.get_root().preorder())
if total_nodes > options.max_nodes:
tree.set_root(otfa(tree,total_count))
tree.set_root(tree.get_root().post_aggregate())
#aggregate_LRU(tree,options.max_nodes,options.aggregate,total_count)
#print tree.get_root().preorder()
#tree.increase_age_tree()
total_count+= float(dict_fields[VALUE])
map(lambda x: x.increase_age(),tree.get_root().preorder())
def benchmark_aggregation(options,args,fields,dim,types,dict_dim, otfa = lambda x: x.get_root()):
tree_src = None
tree_dst = None
list_res = []
list_window = []
tree = None
f = open(options.input)
total_count = 0.0
current_window = 0
tree = Tree(dim)
k = 0
for line in f:
find = re.search(options.reg_exp,line)
if find:
dict_fields = {}
for i in range(len(fields)):
dict_fields[fields[i]] = find.group(i+1)
#print dict_fields
try:
sec = time.mktime(time.strptime(dict_fields["timestamp"],"%Y-%m-%d %H:%M:%S"))
except:
sec = int(dict_fields["timestamp"])
list_window.append(current_window)
try:
update_tree(tree,dict_fields,dict_dim,options.type_aggr)
except:
tree.set_root(tree.get_root().post_aggregate())
update_tree(tree,dict_fields,dict_dim,options.type_aggr)
total_nodes = len(tree.get_root().preorder())
if total_nodes > options.max_nodes:
tree.set_root(otfa(tree,total_count))
tree.set_root(tree.get_root().post_aggregate())
#aggregate_LRU(tree,options.max_nodes,options.aggregate,total_count)
#print tree.get_root().preorder()
#tree.increase_age_tree()
total_count+= float(dict_fields[VALUE])
map(lambda x: x.increase_age(),tree.get_root().preorder())
k=k+1
if options.max_lines > 0 and options.max_lines < k :
break
if tree.get_root() != None:
list_res.append((tree,total_count))
pretotal_nodes_before_aggregation = len(tree.get_root().preorder())
tree.aggregate(options.aggregate,total_count)
print "Total nodes before pre order aggregation %s"%pretotal_nodes_before_aggregation
print "Total nodes after aggregation %s"%len(tree.get_root().preorder())
def build_stability_aggregation_trees(options,args,fields,dim,types,dict_dim, otfa = lambda x,y: x.get_root(),randomize=False,no_final_aggregation=False):
#TODO: remove nof_final aggregtion parameter and replace it by the test below
if not options.aggregate>0:
no_final_aggregation = True
tree_src = None
tree_dst = None
list_res = []
list_window = []
tree = None
f = open(options.input)
lines = [ l for l in f]
f.close()
if randomize: random.shuffle(lines)
total_count = 0.0
current_window = 0
for line in lines:
#print line
find = re.search(options.reg_exp,line)
#print find
if find:
dict_fields = {}
for i in range(len(fields)):
dict_fields[fields[i]] = find.group(i+1)
#print dict_fields
try:
# print dict_fields["timestamp"]
sec = time.mktime(time.strptime(dict_fields["timestamp"],"%Y-%m-%d %H:%M:%S"))
except:
sec = int(dict_fields["timestamp"])
if sec >= current_window + options.window:
if tree != None:
list_res.append((tree,total_count))
tree = Tree(dim)
current_window = current_window + options.window
while sec>= current_window + options.window:
list_res.append((None,0.0))
list_window.append(current_window)
current_window = current_window + options.window
else:
tree = Tree(dim)
current_window = sec
total_count = 0.0
list_window.append(current_window)
try:
update_tree(tree,dict_fields,dict_dim,options.type_aggr)
except Exception, e:
print e
raise
tree.set_root(tree.get_root().post_aggregate())
update_tree(tree,dict_fields,dict_dim,options.type_aggr)
total_nodes = len(tree.get_root().preorder())
if total_nodes > options.max_nodes:
tree.set_root(otfa(tree,total_count))
tree.set_root(tree.get_root().post_aggregate())
#aggregate_LRU(tree,options.max_nodes,options.aggregate,total_count)
#print tree.get_root().preorder()
#tree.increase_age_tree()
total_count+= float(dict_fields[VALUE])
map(lambda x: x.increase_age(),tree.get_root().preorder())