mapbox_access_token = "pk.eyJ1IjoicGxvdGx5bWFwYm94IiwiYSI6ImNrOWJqb2F4djBnMjEzbG50amg0dnJieG4ifQ.Zme1-Uzoi75IaFbieBDl3A"
tree_file, metadata_file, metadata_file_stat = create_paths_file(virus_name,
level1="",
level2="",
level3="")
# To know the minimum and the maximum values of date for slicer
df_stat_metadata = pd.read_csv(metadata_file_stat)
min_date, max_date = min_max_date(df_stat_metadata)
# create the dictionary of slider
marks_data = slicer(min_date, max_date)
min_max_date_value = [min_date, max_date]
fig = create_tree(virus_name, tree_file, metadata_file, "Country")
tree_fig[tree_file] = fig
fig_map_bubble = create_map_bubble_year(virus_name, metadata_file_stat, 2,
min_date, max_date)
fig_curve_line = create_curve_line(df_stat_metadata, virus_name, min_date,
max_date)
######################################### MAIN APP #########################################
app.layout = html.Div([
# Banner display
html.Div(
className="header-title",
children=[
html.H2(
def update_phylogeny_tree(
virus_name,
mumps,
dengue,
lassa,
avian_opt1,
avian_opt2,
flu_opt1,
flu_opt2,
flu_opt3,
):
virus_name = virus_name.lower()
ord_by_elt = "Country"
if virus_name == "ebola" or virus_name == "zika" or virus_name == "measles":
(
tree_file_filtred,
metadata_file_filtred,
metadata_file_stat_filtred,
) = create_paths_file(virus_name, level1="", level2="", level3="")
elif virus_name == "mumps":
(
tree_file_filtred,
metadata_file_filtred,
metadata_file_stat_filtred,
) = create_paths_file(virus_name, level1=mumps, level2="", level3="")
elif virus_name == "dengue":
(
tree_file_filtred,
metadata_file_filtred,
metadata_file_stat,
) = create_paths_file(virus_name, level1=dengue, level2="", level3="")
elif virus_name == "lassa":
(
tree_file_filtred,
metadata_file_filtred,
metadata_file_stat_filtred,
) = create_paths_file(virus_name, level1=lassa, level2="", level3="")
elif virus_name == "avian":
(
tree_file_filtred,
metadata_file_filtred,
metadata_file_stat_filtred,
) = create_paths_file(virus_name,
level1=avian_opt1,
level2=avian_opt2,
level3="")
elif virus_name == "flu":
(
tree_file_filtred,
metadata_file_filtred,
metadata_file_stat_filtred,
) = create_paths_file(virus_name,
level1=flu_opt1,
level2=flu_opt2,
level3=flu_opt3)
if tree_file_filtred in tree_fig:
fig = tree_fig[tree_file_filtred]
else:
if ord_by_elt == "Country" or ord_by_elt == "Division" or ord_by_elt == "Date":
fig = create_tree(virus_name, tree_file_filtred,
metadata_file_filtred, ord_by_elt)
tree_fig[tree_file_filtred] = fig
return fig
while node:
stack.append(node)
node = node.left
count = 0
while stack:
node = stack.pop()
count += 1
if count == k:
break
node = node.right
while node:
stack.append(node)
node = node.left
if count == k:
return node.val
else:
return None
a = create_tree([
31, 30, 48, 3, None, 38, 49, 0, 16, 35, 47, None, None, None, 2, 15, 27,
33, 37, 39, None, 1, None, 5, None, 22, 28, 32, 34, 36, None, None, 43,
None, None, 4, 11, 19, 23, None, 29, None, None, None, None, None, None,
40, 46, None, None, 7, 14, 17, 21, None, 26, None, None, None, 41, 44,
None, 6, 10, 13, None, None, 18, 20, None, 25, None, None, 42, None, 45,
None, None, 8, None, 12, None, None, None, None, None, 24, None, None,
None, None, None, None, 9
])
print(Solution().kthSmallest(a, 100000))
print(Solution().kthSmallest2(a, 100000))
node.left, node.right = node.right, node.left
self.reverse(node.left)
self.reverse(node.right)
return node
def same(self, l, r):
if not l and not r:
return True
elif l and r:
return l.val == r.val and self.same(l.left, r.left) and self.same(
l.right, r.right)
else:
return False
def isSymmetric2(self, root):
"""
二叉树是否对称，先反转左子树，再比较左右子树是否相同
:type root: TreeNode
:rtype: bool
"""
if not root:
return True
left2 = self.reverse(root.left)
return self.same(left2, root.right)
if __name__ == '__main__':
tree = create_tree([1, 2, 2, 3, 4, 4, 3])
result = Solution().isSymmetric(tree)
print(result)
def minibatch_(functions, clf,scaler,w, loss__,mse,hinge1,hinge2,full_image,img_nr,alphas,learning_rate,subsamples, mode):
X_p, y_p, inv = get_data_from_img_nr(class_,img_nr, subsamples)
if X_p != []:
boxes = []
ground_truth = inv[0][2]
img_nr = inv[0][0]
print img_nr
if less_features:
X_p = [fts[0:features_used] for fts in X_p]
if os.path.isfile('/var/node436/local/tstahl/Coords_prop_windows/'+ (format(img_nr, "06d")) +'.txt'):
f = open('/var/node436/local/tstahl/Coords_prop_windows/'+ (format(img_nr, "06d")) +'.txt', 'r')
else:
print 'warning'
for line, y in zip(f, inv):
tmp = line.split(',')
coord = []
for s in tmp:
coord.append(float(s))
boxes.append([coord, y[2]])
#assert(len(boxes)<500)
boxes, y_p, X_p = sort_boxes(boxes, y_p, X_p, 0,5000)
if os.path.isfile('/var/node436/local/tstahl/GroundTruth/%s/%s.txt'%(class_,format(img_nr, "06d"))):
gr = open('/var/node436/local/tstahl/GroundTruth/%s/%s.txt'%(class_,format(img_nr, "06d")), 'r')
else:
gr = []
ground_truths = []
for line in gr:
tmp = line.split(',')
ground_truth = []
for s in tmp:
ground_truth.append(int(s))
ground_truths.append(ground_truth)
#prune boxes
pruned_x = []
pruned_y = []
pruned_boxes = []
if prune:
for i, y_ in enumerate(y_p):
if y_ > 0:
pruned_x.append(X_p[i])
pruned_y.append(y_p[i])
pruned_boxes.append(boxes[i])
else:
pruned_x = X_p
pruned_y = y_p
pruned_boxes = boxes
if subsampling and pruned_boxes > subsamples:
pruned_x = pruned_x[0:subsamples]
pruned_y = pruned_y[0:subsamples]
pruned_boxes = pruned_boxes[0:subsamples]
# create_tree
G, levels = create_tree(pruned_boxes)
#prune tree to only have levels which fully cover the image, tested
if prune_fully_covered:
nr_levels_covered = 100
total_size = surface_area(pruned_boxes, levels[0])
for level in levels:
sa = surface_area(pruned_boxes, levels[level])
sa_co = sa/total_size
if sa_co != 1.0:
G.remove_nodes_from(levels[level])
else:
nr_levels_covered = level
levels = {k: levels[k] for k in range(0,nr_levels_covered + 1)}
# prune levels, speedup + performance
levels_tmp = {k:v for k,v in levels.iteritems() if k<prune_tree_levels}
levels_gone = {k:v for k,v in levels.iteritems() if k>=prune_tree_levels}
levels = levels_tmp
#prune tree as well, for patches training
for trash_level in levels_gone.values():
G.remove_nodes_from(trash_level)
coords = []
features = []
f_c = []
f = []
#either subsampling or prune_fully_covered
#assert(subsampling != prune_fully_covered)
if subsampling:
if os.path.isfile('/var/node436/local/tstahl/Features_prop_windows/upper_levels/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples)):
f_c = open('/var/node436/local/tstahl/Features_prop_windows/upper_levels/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples), 'r+')
else:
if mode == 'extract_train' or mode == 'extract_test':
print 'coords for %s with %s samples have to be extracted'%(img_nr,subsamples)
f_c = open('/var/node436/local/tstahl/Features_prop_windows/upper_levels/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples), 'w')
for level in levels:
levl_boxes = extract_coords(levels[level], pruned_boxes)
if levl_boxes != []:
for lvl_box in levl_boxes:
if lvl_box not in coords:
coords.append(lvl_box)
f_c.write('%s,%s,%s,%s'%(lvl_box[0],lvl_box[1],lvl_box[2],lvl_box[3]))
f_c.write('\n')
f_c.close()
print 'features for %s with %s samples have to be extracted'%(img_nr,subsamples)
os.system('export PATH=$PATH:/home/koelma/impala/lib/x86_64-linux-gcc')
os.system('export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/koelma/impala/third.13.03/x86_64-linux/lib')
#print "EuVisual /var/node436/local/tstahl/Images/%s.jpg /var/node436/local/tstahl/Features_prop_windows/Features_upper/sheep_%s_%s.txt --eudata /home/koelma/EuDataBig --imageroifile /var/node436/local/tstahl/Features_prop_windows/upper_levels/sheep_%s_%s.txt"%((format(img_nr, "06d")),format(img_nr, "06d"),subsamples,format(img_nr, "06d"),subsamples)
os.system("EuVisual /var/node436/local/tstahl/Images/%s.jpg /var/node436/local/tstahl/Features_prop_windows/Features_upper/%s_%s_%s.txt --eudata /home/koelma/EuDataBig --imageroifile /var/node436/local/tstahl/Features_prop_windows/upper_levels/%s_%s_%s.txt"%(class_,(format(img_nr, "06d")),format(img_nr, "06d"),subsamples,class_,format(img_nr, "06d"),subsamples))
if os.path.isfile('/var/node436/local/tstahl/Features_prop_windows/upper_levels/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples)):
f_c = open('/var/node436/local/tstahl/Features_prop_windows/upper_levels/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples), 'r')
else:
f_c = []
coords = []
if os.path.isfile('/var/node436/local/tstahl/Features_prop_windows/Features_upper/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples)):
f = open('/var/node436/local/tstahl/Features_prop_windows/Features_upper/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples), 'r')
elif prune_fully_covered:
if os.path.isfile('/var/node436/local/tstahl/Features_prop_windows/upper_levels/sheep_%s_fully_cover_tree.txt'%(format(img_nr, "06d"))):
f_c = open('/var/node436/local/tstahl/Features_prop_windows/upper_levels/sheep_%s_fully_cover_tree.txt'%(format(img_nr, "06d")), 'r+')
else:
if mode == 'extract_train' or mode == 'extract_test':
print 'coords for %s with fully_cover_tree samples have to be extracted'%(img_nr)
f_c = open('/var/node436/local/tstahl/Features_prop_windows/upper_levels/sheep_%s_fully_cover_tree.txt'%(format(img_nr, "06d")), 'w')
for level in levels:
levl_boxes = extract_coords(levels[level], pruned_boxes)
if levl_boxes != []:
for lvl_box in levl_boxes:
if lvl_box not in coords:
coords.append(lvl_box)
f_c.write('%s,%s,%s,%s'%(lvl_box[0],lvl_box[1],lvl_box[2],lvl_box[3]))
f_c.write('\n')
f_c.close()
print 'features for %s with fully_cover_tree samples have to be extracted'%(img_nr)
os.system('export PATH=$PATH:/home/koelma/impala/lib/x86_64-linux-gcc')
os.system('export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/koelma/impala/third.13.03/x86_64-linux/lib')
#print "EuVisual /var/node436/local/tstahl/Images/%s.jpg /var/node436/local/tstahl/Features_prop_windows/Features_upper/sheep_%s_%s.txt --eudata /home/koelma/EuDataBig --imageroifile /var/node436/local/tstahl/Features_prop_windows/upper_levels/sheep_%s_%s.txt"%((format(img_nr, "06d")),format(img_nr, "06d"),subsamples,format(img_nr, "06d"),subsamples)
print "EuVisual /var/node436/local/tstahl/Images/%s.jpg /var/node436/local/tstahl/Features_prop_windows/Features_upper/sheep_%s_fully_cover_tree.txt --eudata /home/koelma/EuDataBig --imageroifile /var/node436/local/tstahl/Features_prop_windows/upper_levels/sheep_%s_fully_cover_tree.txt"%((format(img_nr, "06d")),format(img_nr, "06d"),format(img_nr, "06d"))
os.system("EuVisual /var/node436/local/tstahl/Images/%s.jpg /var/node436/local/tstahl/Features_prop_windows/Features_upper/sheep_%s_fully_cover_tree.txt --eudata /home/koelma/EuDataBig --imageroifile /var/node436/local/tstahl/Features_prop_windows/upper_levels/sheep_%s_fully_cover_tree.txt"%((format(img_nr, "06d")),format(img_nr, "06d"),format(img_nr, "06d")))
if os.path.isfile('/var/node436/local/tstahl/Features_prop_windows/upper_levels/sheep_%s_fully_cover_tree.txt'%(format(img_nr, "06d"))):
f_c = open('/var/node436/local/tstahl/Features_prop_windows/upper_levels/sheep_%s_fully_cover_tree.txt'%(format(img_nr, "06d")), 'r')
else:
f_c = []
coords = []
if os.path.isfile('/var/node436/local/tstahl/Features_prop_windows/Features_upper/sheep_%s_fully_cover_tree.txt'%(format(img_nr, "06d"))):
f = open('/var/node436/local/tstahl/Features_prop_windows/Features_upper/sheep_%s_fully_cover_tree.txt'%(format(img_nr, "06d")), 'r')
else:
if os.path.isfile('/var/node436/local/tstahl/Features_prop_windows/Features_upper/sheep%s.txt'%(format(img_nr, "06d"))):
f = open('/var/node436/local/tstahl/Features_prop_windows/Features_upper/sheep%s.txt'%(format(img_nr, "06d")), 'r')
if os.path.isfile('/var/node436/local/tstahl/Features_prop_windows/upper_levels/sheep%s.txt'%(format(img_nr, "06d"))):
f_c = open('/var/node436/local/tstahl/Features_prop_windows/upper_levels/sheep%s.txt'%(format(img_nr, "06d")), 'r+')
if f_c != []:
for i,line in enumerate(f_c):
str_ = line.rstrip('\n').split(',')
cc = []
for s in str_:
cc.append(float(s))
coords.append(cc)
if f != []:
for i,line in enumerate(f):
str_ = line.rstrip('\n').split(',')
ff = []
for s in str_:
ff.append(float(s))
features.append(ff)
#assert len(coords) == len(features)
# append x,y of intersections
if learn_intersections:
for inters,coord in zip(features,coords):
# if inters not in pruned_x:
pruned_x.append(inters)
ol = 0.0
ol = get_intersection_count(coord, ground_truths)
pruned_y.append(ol)
if mode == 'mean_variance':
print 'normalizing'
sum_x += np.array(pruned_x).sum(axis=0)
n_samples += len(pruned_x)
sum_sq_x += (np.array(pruned_x)**2).sum(axis=0)
scaler.partial_fit(pruned_x) # Don't cheat - fit only on training data
return sum_x,n_samples,sum_sq_x, scaler
if less_features:
features = [fts[0:features_used] for fts in features]
#normalize
norm_x = []
if normalize and (mode != 'extract_train' and mode != 'extract_test'):
# for p_x in pruned_x:
# norm_x.append((p_x-mean)/variance)
norm_x = scaler.transform(pruned_x)
if features != []:
features = scaler.transform(features)
else:
norm_x = pruned_x
data = (G, levels, pruned_y, norm_x, pruned_boxes, ground_truths, alphas)
sucs = nx.dfs_successors(G)
predecs = nx.dfs_predecessors(G)
#preprocess: node - children
children = {}
last = -1
for node,children_ in zip(sucs.keys(),sucs.values()):
if node != last+1:
for i in range(last+1,node):
children[i] = []
children[node] = children_
elif node == last +1:
children[node] = children_
last = node
if mode == 'training':
if alphas[0] == 0: #if we don't learn the proposals, we learn just the levels: better, because every level has same importance and faster
print 'training levels', img_nr
for level in levels:
print 'level' , level
if img_nr in functions:
if level in functions[img_nr]:
function = functions[img_nr][level]
else:
function = []
else:
functions[img_nr] = {}
function = []
w, function = tree_level_regression(class_,function,levels,level,features,coords,scaler,w,norm_x,pruned_y,None,predecs,children,pruned_boxes,learning_rate,alphas,img_nr,jans_idea)
if level not in functions[img_nr]:
functions[img_nr][level] = function
return w, len(pruned_y), len(levels)
else: #if we learn proposals, levels with more proposals have more significance...., slow - need to change
print 'training patches', img_nr
print predecs
nodes = list(G.nodes())
for node in nodes:
print node
if node == 0:
w = learn_root(w,norm_x[0],pruned_y[0],learning_rate,alphas)
else:
for num,n in enumerate(levels.values()):
if node in n:
level = num
break
if img_nr in functions:
if level in functions[img_nr]:
function = functions[img_nr][level]
else:
function = []
else:
functions[img_nr] = {}
function = []
#w, function = tree_level_regression(class_,function,levels,level,features,coords,scaler,w,norm_x,pruned_y,node,predecs,children,pruned_boxes,learning_rate,alphas,img_nr)
w, function = constrained_regression(class_,function,features,coords,scaler,w,norm_x,pruned_y,node,predecs,children,pruned_boxes,learning_rate,alphas,img_nr,squared_hinge_loss)
#TODO: train regressor/classifier that predicts/chooses level. Features: level, number of proposals, number of intersections, avg size of proposal, predictions(for regressor), etc.
if level not in functions[img_nr]:
functions[img_nr][level] = function
return w, len(pruned_y), len(G.nodes())
elif mode == 'scikit_train':
clf.partial_fit(norm_x,pruned_y)
return clf
elif mode == 'loss_train':
if alphas[0] == 0: #levels
loss__.append(tree_level_loss(class_,features,coords,scaler, w, data, predecs, children,img_nr,-1,functions))
return loss__
else:
loss__.append(loss(class_,squared_hinge_loss,features,coords,scaler,w, data, predecs, children,img_nr, -1))
elif mode == 'loss_test' or mode == 'loss_eval':
print mode, loss__
if alphas[0] == 0: #levels
loss__.append(tree_level_loss(class_,features,coords,scaler, w, data, predecs, children,img_nr,-1,functions))
cpl = max(0, np.dot(w,np.array(norm_x[0]).T))
full_image.append([pruned_y[0],cpl])
return loss__,full_image
else:
loss__.append(loss(class_,squared_hinge_loss,features,coords,scaler,w, data, predecs, children,img_nr, -1))
cpl = max(0, np.dot(w,np.array(norm_x[0]).T))
full_image.append([pruned_y[0],cpl])
return loss__,full_image
elif mode == 'loss_scikit_test' or mode == 'loss_scikit_train':
loss__.append(((clf.predict(norm_x) - pruned_y)**2).sum())
return loss__
elif mode == 'levels_train' or mode == 'levels_test':
preds = []
for i,x_ in enumerate(norm_x):
preds.append(np.dot(w, x_))
cpls = []
truelvls = []
used_boxes_ = []
total_size = surface_area(pruned_boxes, levels[0])
fully_covered_score = 0.0
fully_covered_score_lvls = 0.0
covered_levels = []
print mode, len(levels)
for level in levels:
function = functions[img_nr][level]
cpl,used_boxes,_ = count_per_level([],class_,features,coords,scaler,w, preds, img_nr, pruned_boxes,levels[level], '',function)
# clipp negative predictions
cpl = max(0,cpl)
if used_boxes != []:
used_boxes_.append(used_boxes[0][1])
tru = y_p[0]
cpls.append(cpl)
sa = surface_area(pruned_boxes, levels[level])
sa_co = sa/total_size
if sa_co == 1.0:
fully_covered_score += cpl
fully_covered_score_lvls += 1
covered_levels.append(cpl)
truelvls.append(tru)
return cpls,truelvls
class Solution:
def rightSideView(self, root: TreeNode) -> List[int]:
if not root:
return []
q = deque()
q.append(root)
ret = []
while q:
tmp = []
path = []
l = len(q)
for _ in range(l):
node = q.popleft()
tmp.append(node)
path.append(node.val)
ret.append(path[-1])
for node in tmp:
if node.left:
q.append(node.left)
if node.right:
q.append(node.right)
return ret
from utils import create_tree
root = create_tree([1, 2, 3, 4, 5, 6])
ret = Solution().rightSideView(root)
print(ret)
# testResult = utils.get_shannon_entropy(labels)
# print(testResult)
# #测试条件熵
# print('测试条件熵')
# testResult = utils.get_conditional_entropy(datas[:, 2], labels)
# print(testResult)
# # 测试信息增益
# print('测试信息增益')
# testResult = utils.get_best_gain(datas, labels)
# print(testResult)
# # 测试建树字典
# print(测试建树字典)
# testResult = utils.create_tree(datas_header, datas, labels)
# print(testResult)
# 测试预测函数
# print(测试预测函数)
tree_model = utils.create_tree(datas_header, datas, labels)
input_data = ['帅', '好', '矮', '上进']
testResult = utils.predict_result(tree_model, input_data, datas_header)
print(testResult)
# 测试保存和读取函数
utils.store_tree(tree_model, '嫁不嫁.pkl') # .pkl和.txt什么的都可以，但建议.pkl
testResult = utils.restore_tree('嫁不嫁.pkl')
print(testResult)
import time
import gates as gate
from utils import create_tree, save_tree
basis = [gate.su2(gate.H), gate.su2(gate.T)]
depth = 15
t = time.time()
tree = create_tree(basis, max_depth=depth)
print('Elapsed time:', time.time() - t)
print('Number of gates:', len(tree['names']))
save_tree(tree, 'trees/HT_{}.pkl'.format(depth))
# 把二叉树打印成多行
# https://www.nowcoder.com/practice/445c44d982d04483b04a54f298796288?tpId=13&tqId=11213&tPage=2&rp=2&ru=/ta/coding-interviews&qru=/ta/coding-interviews/question-ranking
from utils import create_tree
class Solution:
# 返回二维列表[[1,2],[4,5]]
def Print(self, pRoot):
if not pRoot:
return []
level = 0
ret = [[]]
t = []
t.append([pRoot, 0])
while t:
node = t.pop(0)
if not node[0]:
continue
if node[1] != level:
ret.append([])
level = node[1]
ret[level].append(node[0].val)
t.append([node[0].left, node[1] + 1])
t.append([node[0].right, node[1] + 1])
return ret
r = create_tree([1, 2, 3, 4, 5, None, 6, 7])
print(Solution().Print(r))
ret = []
ret.append((0, root))
old_v = None
count = 0
res = 0
while ret:
color, node = ret.pop()
if not node:
continue
if color == 0:
ret.append((0, node.right))
ret.append((1, node))
ret.append((0, node.left))
else:
print(node.val)
if count == 0:
res = node.val
if not old_v or old_v == node.val:
count += 1
old_v = node.val
else:
count -= 1
old_v = node.val
return res
from utils import create_tree
root = create_tree([3, 2, 3, 1, 3, 3, 4])
ret = Solution().findMode(root)
print(ret)
root.left = dps(TreeNode(None), left1, left2)
root.right = dps(TreeNode(None), right1, right2)
return root
root = TreeNode(None)
root = dps(root, t1, t2)
return root
class Solution2:
def mergeTrees(self, t1: TreeNode, t2: TreeNode) -> TreeNode:
if not t1 or not t2:
return t1 or t2
t1.val = t1.val + t2.val
t1.left = self.mergeTrees(t1.left, t2.left)
t1.right = self.mergeTrees(t1.right, t2.right)
return t1
from utils import create_tree, print_tree
a = create_tree([1, 3, 2, 5])
b = create_tree([2, 1, 3, 4])
root = Solution().mergeTrees(a, b)
print_tree(root)
class Solution:
def inverseTree(self, root):
if root is None: return []
ret = []
de = deque()
level = 1
de.append((root, level))
while de:
tmp = deque()
for _ in range(len(de)):
(node, level) = de.popleft()
if level % 2 == 0:
tmp.appendleft(node.val)
else:
tmp.append(node.val)
if node.left:
de.append((node.left, level + 1))
if node.right:
de.append((node.right, level + 1))
ret.append(list(tmp))
return ret
from utils import create_tree
cli = Solution()
print(cli.inverseTree(create_tree([1, 2, 3, 4, 5, 6, 7, 8, 9])))
###思路：
# 使用队列将节点按照正常层次遍历顺序存储
# 使用双端队列将节点的值进行存储。奇数层的节点从右边插入，偶数层的节点从左边插入。
def minibatch_(all_train_imgs,all_test_imgs,clf,scaler,w, loss__,mse,hinge1,hinge2,full_image,alphas,learning_rate,test_imgs, train_imgs,minibatch,subsamples,sum_x,n_samples,sum_sq_x,mean,variance, mode,mous):
if mode == 'loss_test' or mode == 'loss_scikit_test' or mode == 'levels_test':
X_p, y_p, inv = get_data(class_, test_imgs, train_imgs, minibatch, minibatch + 1, 'test', c, subsamples)
else:
X_p, y_p, inv = get_data(class_, test_imgs, train_imgs, minibatch, minibatch + 1, 'training', c, subsamples)
if X_p != []:
boxes = []
ground_truth = inv[0][2]
img_nr = inv[0][0]
if less_features:
X_p = [fts[0:features_used] for fts in X_p]
if os.path.isfile('/home/stahl/Coords_prop_windows/'+ (format(img_nr, "06d")) +'.txt'):
f = open('/home/stahl/Coords_prop_windows/'+ (format(img_nr, "06d")) +'.txt', 'r')
else:
print 'warning'
for line, y in zip(f, inv):
tmp = line.split(',')
coord = []
for s in tmp:
coord.append(float(s))
boxes.append([coord,y[2]])
assert(len(boxes)<1500)
boxes, y_p, X_p = sort_boxes(boxes, y_p, X_p, 0,1500)
gr = []
if os.path.isfile('/home/stahl/GroundTruth/%s/%s.txt'%(class_,format(img_nr, "06d"))):
gr = open('/home/stahl/GroundTruth/%s/%s.txt'%(class_,format(img_nr, "06d")), 'r')
ground_truths = []
if gr != []: #if no class image -> no ground truth. (I think this is only needed for learn _ntersection)
for line in gr:
tmp = line.split(',')
ground_truth = []
for s in tmp:
ground_truth.append(int(s))
ground_truths.append(ground_truth)
if mode == 'mean_variance':
scaler.partial_fit(X_p) # Don't cheat - fit only on training data
return scaler
# create_tree
G, levels = create_tree(boxes)
#prune tree to only have levels which fully cover the image
# tested
if prune_fully_covered:
nr_levels_covered = 100
total_size = surface_area(boxes, levels[0])
for level in levels:
sa = surface_area(boxes, levels[level])
sa_co = sa/total_size
if sa_co != 1.0:
G.remove_nodes_from(levels[level])
else:
nr_levels_covered = level
levels = {k: levels[k] for k in range(0,nr_levels_covered + 1)}
#either subsampling or prune_fully_covered
#assert(subsampling != prune_fully_covered)
# prune levels, speedup + performance
levels = {k:v for k,v in levels.iteritems() if k<prune_tree_levels}
#prune G in order to remove nodes of the lower levels
remaining_nodes = []
for lev in levels.values():
remaining_nodes.extend(lev)
for g_node in G.nodes():
if g_node not in remaining_nodes:
G.remove_node(g_node)
coords = []
features = []
f_c = []
f = []
if learn_intersections and not prune_fully_covered:
if os.path.isfile('/home/stahl/Features_prop_windows/upper_levels/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples)):
f_c = open('/home/stahl/Features_prop_windows/upper_levels/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples), 'r+')
else:
if mode == 'extract_train' or mode == 'extract_test':
print 'coords for %s with %s samples have to be extracted'%(img_nr,subsamples)
f_c = open('/home/stahl/Features_prop_windows/upper_levels/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples), 'w')
for level in levels:
levl_boxes = extract_coords(levels[level], boxes)
if levl_boxes != []:
for lvl_box in levl_boxes:
if lvl_box not in coords:
coords.append(lvl_box)
f_c.write('%s,%s,%s,%s'%(lvl_box[0],lvl_box[1],lvl_box[2],lvl_box[3]))
f_c.write('\n')
f_c.close()
print 'features for %s with %s samples have to be extracted'%(img_nr,subsamples)
os.system('export PATH=$PATH:/home/koelma/impala/lib/x86_64-linux-gcc')
os.system('export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/koelma/impala/third.13.03/x86_64-linux/lib')
#print "EuVisual /home/stahl/Images/%s.jpg /home/stahl/Features_prop_windows/Features_upper/sheep_%s_%s.txt --eudata /home/koelma/EuDataBig --imageroifile /home/stahl/Features_prop_windows/upper_levels/sheep_%s_%s.txt"%((format(img_nr, "06d")),format(img_nr, "06d"),subsamples,format(img_nr, "06d"),subsamples)
os.system("EuVisual /home/stahl/Images/%s.jpg /home/stahl/Features_prop_windows/Features_upper/%s_%s_%s.txt --eudata /home/koelma/EuDataBig --imageroifile /home/stahl/Features_prop_windows/upper_levels/%s_%s_%s.txt"%(class_,(format(img_nr, "06d")),format(img_nr, "06d"),subsamples,class_,format(img_nr, "06d"),subsamples))
if os.path.isfile('/home/stahl/Features_prop_windows/upper_levels/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples)):
f_c = open('/home/stahl/Features_prop_windows/upper_levels/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples), 'r')
else:
f_c = []
coords = []
if os.path.isfile('/home/stahl/Features_prop_windows/Features_upper/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples)):
f = open('/home/stahl/Features_prop_windows/Features_upper/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples), 'r')
elif prune_fully_covered:
if os.path.isfile('/home/stahl/Features_prop_windows/upper_levels/sheep_%s_fully_cover_tree.txt'%(format(img_nr, "06d"))):
f_c = open('/home/stahl/Features_prop_windows/upper_levels/sheep_%s_fully_cover_tree.txt'%(format(img_nr, "06d")), 'r+')
else:
if mode == 'extract_train' or mode == 'extract_test':
print 'coords for %s with fully_cover_tree samples have to be extracted'%(img_nr)
f_c = open('/home/stahl/Features_prop_windows/upper_levels/%s_%s_fully_cover_tree.txt'%(class_,format(img_nr, "06d")), 'w')
for level in levels:
levl_boxes = extract_coords(levels[level], boxes)
if levl_boxes != []:
for lvl_box in levl_boxes:
if lvl_box not in coords:
coords.append(lvl_box)
f_c.write('%s,%s,%s,%s'%(lvl_box[0],lvl_box[1],lvl_box[2],lvl_box[3]))
f_c.write('\n')
f_c.close()
print 'features for %s with fully_cover_tree samples have to be extracted'%(img_nr)
os.system('export PATH=$PATH:/home/koelma/impala/lib/x86_64-linux-gcc')
os.system('export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/koelma/impala/third.13.03/x86_64-linux/lib')
#print "EuVisual /home/stahl/Images/%s.jpg /home/stahl/Features_prop_windows/Features_upper/sheep_%s_%s.txt --eudata /home/koelma/EuDataBig --imageroifile /home/stahl/Features_prop_windows/upper_levels/sheep_%s_%s.txt"%((format(img_nr, "06d")),format(img_nr, "06d"),subsamples,format(img_nr, "06d"),subsamples)
os.system("EuVisual /home/stahl/Images/%s.jpg /home/stahl/Features_prop_windows/Features_upper/%s_%s_fully_cover_tree.txt --eudata /home/koelma/EuDataBig --imageroifile /home/stahl/Features_prop_windows/upper_levels/%s_%s_fully_cover_tree.txt"%(class_,(format(img_nr, "06d")),format(img_nr, "06d"),class_,format(img_nr, "06d")))
if os.path.isfile('/home/stahl/Features_prop_windows/upper_levels/%s_%s_fully_cover_tree.txt'%(class_,format(img_nr, "06d"))):
f_c = open('/home/stahl/Features_prop_windows/upper_levels/%s_%s_fully_cover_tree.txt'%(class_,format(img_nr, "06d")), 'r')
else:
f_c = []
coords = []
if os.path.isfile('/home/stahl/Features_prop_windows/Features_upper/sheep_%s_fully_cover_tree.txt'%(format(img_nr, "06d"))):
f = open('/home/stahl/Features_prop_windows/Features_upper/sheep_%s_fully_cover_tree.txt'%(format(img_nr, "06d")), 'r')
# else: #we don't need to load intersections
# if os.path.isfile('/home/stahl/Features_prop_windows/Features_upper/%s%s.txt'%(class_,format(img_nr, "06d"))):
# f = open('/home/stahl/Features_prop_windows/Features_upper/%s%s.txt'%(class_,format(img_nr, "06d")), 'r')
# if os.path.isfile('/home/stahl/Features_prop_windows/upper_levels/%s%s.txt'%(class_,format(img_nr, "06d"))):
# f_c = open('/home/stahl/Features_prop_windows/upper_levels/%s%s.txt'%(class_,format(img_nr, "06d")), 'r+')
# else:
# print '/home/stahl/Features_prop_windows/upper_levels/%s%s.txt does not exist'%(class_,format(img_nr, "06d"))
for i,line in enumerate(f_c):
str_ = line.rstrip('\n').split(',')
cc = []
for s in str_:
cc.append(float(s))
coords.append(cc)
for i,line in enumerate(f):
str_ = line.rstrip('\n').split(',')
ff = []
for s in str_:
ff.append(float(s))
features.append(ff)
if less_features:
features = [fts[0:features_used] for fts in features]
if normalize and features != []:
features = scaler.transform(features)
print len(y_p), len(X_p)
print len(features), len(coords)
assert len(coords) == len(features)
# append x,y of intersections
#if learn_intersections:
# for inters,coord in zip(features,coords):
# if inters not in pruned_x:
# X_p.append(inters)
# ol = 0.0
# ol = get_intersection_count(coord, ground_truths)
# y_p.append(ol)
print len(y_p), len(X_p)
#normalize
norm_x = []
if normalize:
# for p_x in pruned_x:
# norm_x.append((p_x-mean)/variance)
norm_x = scaler.transform(X_p)
else:
norm_x = X_p
data = (G, levels, y_p, norm_x, boxes, ground_truths, alphas)
sucs = nx.dfs_successors(G)
predecs = nx.dfs_predecessors(G)
#preprocess: node - children
children = {}
last = -1
for node,children_ in zip(sucs.keys(),sucs.values()):
if node != last+1:
for i in range(last+1,node):
children[i] = []
children[node] = children_
elif node == last +1:
children[node] = children_
last = node
if mode == 'train':
if alphas[2] == 0 and alphas[3] == 0: #just learn proposals and intersections
# only use proposals and intersections used in pruned tree
used_ind = get_used_proposals(G, boxes, coords, levels)
used_x = []
used_y = []
for ind in used_ind['prop']:
used_x.append(norm_x[ind])
used_y.append(y_p[ind])
for ind in used_ind['inters']:
used_x.append(features[ind])
used_y.append(get_intersection_count(coords[ind], ground_truths))
print len(used_x),len(used_y)
for x_i,y_i in zip(used_x,used_y):
w = learn_root(w,x_i,y_i,learning_rate,alphas)
else:
nodes = list(G.nodes())
for node in nodes:
if node == 0:
if alphas[0] != 0:
w = learn_root(w,norm_x[0],y_p[0],learning_rate,alphas)
else:
print 'learn nothing'
else:
w = constrained_regression(class_,features,coords,scaler,w,norm_x,y_p,node,predecs,children,boxes,learning_rate,alphas,img_nr, squared_hinge_loss)
return w, len(y_p), len(G.nodes())
elif mode == 'scikit_train':
print norm_x,y_p
clf.partial_fit(norm_x,y_p)
return clf
elif mode == 'loss_train' or mode == 'loss_test':
loss__.append(loss(class_,squared_hinge_loss,features,coords,scaler, w, data, predecs, children,img_nr,-1))
# mse.append(((data[2] - np.dot(w,np.array(data[3]).T)) ** 2).sum())
# a2 = alphas[2]
# data = (G, levels, y_p, norm_x, boxes, ground_truths, [0,0,a2,0])
# hinge1.append(loss(class_,squared_hinge_loss,features,coords,scaler, w, data, predecs, children,img_nr,-1))
# a3 = alphas[3]
# data = (G, levels, y_p, norm_x, boxes, ground_truths, [0,0,0,a3])
# hinge2.append(loss(class_,squared_hinge_loss,features,coords,scaler, w, data, predecs, children,img_nr,-1))
full_image.append([y_p[0],max(0,np.dot(w,np.array(norm_x[0]).T))])
return loss__, full_image
elif mode == 'loss_scikit_test' or mode == 'loss_scikit_train':
loss__.append(((clf.predict(norm_x) - y_p)**2).sum())
return loss__
elif mode == 'levels_train' or mode == 'levels_test':
#im = mpimg.imread('/home/stahl/Images/'+ (format(img_nr, "06d")) +'.jpg')
preds = []
for i,x_ in enumerate(norm_x):
preds.append(np.dot(w, x_))
cpls = []
truelvls = []
used_boxes_ = []
total_size = surface_area(boxes, levels[0])
fully_covered_score = 0.0
fully_covered_score_lvls = 0.0
covered_levels = []
for level in levels:
#tru and truelvls was in order to check if count_per_level method is correct
cpl,used_boxes,_ = count_per_level(class_,features,coords,scaler,w, preds, img_nr, boxes,levels[level], '',[])
cpl = max(0,cpl)
if used_boxes != []:
used_boxes_.append(used_boxes[0][1])
tru,_,_ = count_per_level(class_,features,coords,scaler,w, preds, img_nr, boxes,levels[level], 'gt',[])
cpls.append(cpl)
sa = surface_area(boxes, levels[level])
sa_co = sa/total_size
if sa_co == 1.0:
fully_covered_score += cpl
fully_covered_score_lvls += 1
covered_levels.append(cpl)
truelvls.append(tru)
return cpls, truelvls, used_boxes_,boxes,preds,fully_covered_score/fully_covered_score_lvls,covered_levels
