mapbox_access_token = "pk.eyJ1IjoicGxvdGx5bWFwYm94IiwiYSI6ImNrOWJqb2F4djBnMjEzbG50amg0dnJieG4ifQ.Zme1-Uzoi75IaFbieBDl3A" tree_file, metadata_file, metadata_file_stat = create_paths_file(virus_name, level1="", level2="", level3="") # To know the minimum and the maximum values of date for slicer df_stat_metadata = pd.read_csv(metadata_file_stat) min_date, max_date = min_max_date(df_stat_metadata) # create the dictionary of slider marks_data = slicer(min_date, max_date) min_max_date_value = [min_date, max_date] fig = create_tree(virus_name, tree_file, metadata_file, "Country") tree_fig[tree_file] = fig fig_map_bubble = create_map_bubble_year(virus_name, metadata_file_stat, 2, min_date, max_date) fig_curve_line = create_curve_line(df_stat_metadata, virus_name, min_date, max_date) ######################################### MAIN APP ######################################### app.layout = html.Div([ # Banner display html.Div( className="header-title", children=[ html.H2(
def update_phylogeny_tree( virus_name, mumps, dengue, lassa, avian_opt1, avian_opt2, flu_opt1, flu_opt2, flu_opt3, ): virus_name = virus_name.lower() ord_by_elt = "Country" if virus_name == "ebola" or virus_name == "zika" or virus_name == "measles": ( tree_file_filtred, metadata_file_filtred, metadata_file_stat_filtred, ) = create_paths_file(virus_name, level1="", level2="", level3="") elif virus_name == "mumps": ( tree_file_filtred, metadata_file_filtred, metadata_file_stat_filtred, ) = create_paths_file(virus_name, level1=mumps, level2="", level3="") elif virus_name == "dengue": ( tree_file_filtred, metadata_file_filtred, metadata_file_stat, ) = create_paths_file(virus_name, level1=dengue, level2="", level3="") elif virus_name == "lassa": ( tree_file_filtred, metadata_file_filtred, metadata_file_stat_filtred, ) = create_paths_file(virus_name, level1=lassa, level2="", level3="") elif virus_name == "avian": ( tree_file_filtred, metadata_file_filtred, metadata_file_stat_filtred, ) = create_paths_file(virus_name, level1=avian_opt1, level2=avian_opt2, level3="") elif virus_name == "flu": ( tree_file_filtred, metadata_file_filtred, metadata_file_stat_filtred, ) = create_paths_file(virus_name, level1=flu_opt1, level2=flu_opt2, level3=flu_opt3) if tree_file_filtred in tree_fig: fig = tree_fig[tree_file_filtred] else: if ord_by_elt == "Country" or ord_by_elt == "Division" or ord_by_elt == "Date": fig = create_tree(virus_name, tree_file_filtred, metadata_file_filtred, ord_by_elt) tree_fig[tree_file_filtred] = fig return fig
while node: stack.append(node) node = node.left count = 0 while stack: node = stack.pop() count += 1 if count == k: break node = node.right while node: stack.append(node) node = node.left if count == k: return node.val else: return None a = create_tree([ 31, 30, 48, 3, None, 38, 49, 0, 16, 35, 47, None, None, None, 2, 15, 27, 33, 37, 39, None, 1, None, 5, None, 22, 28, 32, 34, 36, None, None, 43, None, None, 4, 11, 19, 23, None, 29, None, None, None, None, None, None, 40, 46, None, None, 7, 14, 17, 21, None, 26, None, None, None, 41, 44, None, 6, 10, 13, None, None, 18, 20, None, 25, None, None, 42, None, 45, None, None, 8, None, 12, None, None, None, None, None, 24, None, None, None, None, None, None, 9 ]) print(Solution().kthSmallest(a, 100000)) print(Solution().kthSmallest2(a, 100000))
node.left, node.right = node.right, node.left self.reverse(node.left) self.reverse(node.right) return node def same(self, l, r): if not l and not r: return True elif l and r: return l.val == r.val and self.same(l.left, r.left) and self.same( l.right, r.right) else: return False def isSymmetric2(self, root): """ 二叉树是否对称,先反转左子树,再比较左右子树是否相同 :type root: TreeNode :rtype: bool """ if not root: return True left2 = self.reverse(root.left) return self.same(left2, root.right) if __name__ == '__main__': tree = create_tree([1, 2, 2, 3, 4, 4, 3]) result = Solution().isSymmetric(tree) print(result)
def minibatch_(functions, clf,scaler,w, loss__,mse,hinge1,hinge2,full_image,img_nr,alphas,learning_rate,subsamples, mode): X_p, y_p, inv = get_data_from_img_nr(class_,img_nr, subsamples) if X_p != []: boxes = [] ground_truth = inv[0][2] img_nr = inv[0][0] print img_nr if less_features: X_p = [fts[0:features_used] for fts in X_p] if os.path.isfile('/var/node436/local/tstahl/Coords_prop_windows/'+ (format(img_nr, "06d")) +'.txt'): f = open('/var/node436/local/tstahl/Coords_prop_windows/'+ (format(img_nr, "06d")) +'.txt', 'r') else: print 'warning' for line, y in zip(f, inv): tmp = line.split(',') coord = [] for s in tmp: coord.append(float(s)) boxes.append([coord, y[2]]) #assert(len(boxes)<500) boxes, y_p, X_p = sort_boxes(boxes, y_p, X_p, 0,5000) if os.path.isfile('/var/node436/local/tstahl/GroundTruth/%s/%s.txt'%(class_,format(img_nr, "06d"))): gr = open('/var/node436/local/tstahl/GroundTruth/%s/%s.txt'%(class_,format(img_nr, "06d")), 'r') else: gr = [] ground_truths = [] for line in gr: tmp = line.split(',') ground_truth = [] for s in tmp: ground_truth.append(int(s)) ground_truths.append(ground_truth) #prune boxes pruned_x = [] pruned_y = [] pruned_boxes = [] if prune: for i, y_ in enumerate(y_p): if y_ > 0: pruned_x.append(X_p[i]) pruned_y.append(y_p[i]) pruned_boxes.append(boxes[i]) else: pruned_x = X_p pruned_y = y_p pruned_boxes = boxes if subsampling and pruned_boxes > subsamples: pruned_x = pruned_x[0:subsamples] pruned_y = pruned_y[0:subsamples] pruned_boxes = pruned_boxes[0:subsamples] # create_tree G, levels = create_tree(pruned_boxes) #prune tree to only have levels which fully cover the image, tested if prune_fully_covered: nr_levels_covered = 100 total_size = surface_area(pruned_boxes, levels[0]) for level in levels: sa = surface_area(pruned_boxes, levels[level]) sa_co = sa/total_size if sa_co != 1.0: G.remove_nodes_from(levels[level]) else: nr_levels_covered = level levels = {k: levels[k] for k in range(0,nr_levels_covered + 1)} # prune levels, speedup + performance levels_tmp = {k:v for k,v in levels.iteritems() if k<prune_tree_levels} levels_gone = {k:v for k,v in levels.iteritems() if k>=prune_tree_levels} levels = levels_tmp #prune tree as well, for patches training for trash_level in levels_gone.values(): G.remove_nodes_from(trash_level) coords = [] features = [] f_c = [] f = [] #either subsampling or prune_fully_covered #assert(subsampling != prune_fully_covered) if subsampling: if os.path.isfile('/var/node436/local/tstahl/Features_prop_windows/upper_levels/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples)): f_c = open('/var/node436/local/tstahl/Features_prop_windows/upper_levels/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples), 'r+') else: if mode == 'extract_train' or mode == 'extract_test': print 'coords for %s with %s samples have to be extracted'%(img_nr,subsamples) f_c = open('/var/node436/local/tstahl/Features_prop_windows/upper_levels/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples), 'w') for level in levels: levl_boxes = extract_coords(levels[level], pruned_boxes) if levl_boxes != []: for lvl_box in levl_boxes: if lvl_box not in coords: coords.append(lvl_box) f_c.write('%s,%s,%s,%s'%(lvl_box[0],lvl_box[1],lvl_box[2],lvl_box[3])) f_c.write('\n') f_c.close() print 'features for %s with %s samples have to be extracted'%(img_nr,subsamples) os.system('export PATH=$PATH:/home/koelma/impala/lib/x86_64-linux-gcc') os.system('export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/koelma/impala/third.13.03/x86_64-linux/lib') #print "EuVisual /var/node436/local/tstahl/Images/%s.jpg /var/node436/local/tstahl/Features_prop_windows/Features_upper/sheep_%s_%s.txt --eudata /home/koelma/EuDataBig --imageroifile /var/node436/local/tstahl/Features_prop_windows/upper_levels/sheep_%s_%s.txt"%((format(img_nr, "06d")),format(img_nr, "06d"),subsamples,format(img_nr, "06d"),subsamples) os.system("EuVisual /var/node436/local/tstahl/Images/%s.jpg /var/node436/local/tstahl/Features_prop_windows/Features_upper/%s_%s_%s.txt --eudata /home/koelma/EuDataBig --imageroifile /var/node436/local/tstahl/Features_prop_windows/upper_levels/%s_%s_%s.txt"%(class_,(format(img_nr, "06d")),format(img_nr, "06d"),subsamples,class_,format(img_nr, "06d"),subsamples)) if os.path.isfile('/var/node436/local/tstahl/Features_prop_windows/upper_levels/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples)): f_c = open('/var/node436/local/tstahl/Features_prop_windows/upper_levels/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples), 'r') else: f_c = [] coords = [] if os.path.isfile('/var/node436/local/tstahl/Features_prop_windows/Features_upper/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples)): f = open('/var/node436/local/tstahl/Features_prop_windows/Features_upper/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples), 'r') elif prune_fully_covered: if os.path.isfile('/var/node436/local/tstahl/Features_prop_windows/upper_levels/sheep_%s_fully_cover_tree.txt'%(format(img_nr, "06d"))): f_c = open('/var/node436/local/tstahl/Features_prop_windows/upper_levels/sheep_%s_fully_cover_tree.txt'%(format(img_nr, "06d")), 'r+') else: if mode == 'extract_train' or mode == 'extract_test': print 'coords for %s with fully_cover_tree samples have to be extracted'%(img_nr) f_c = open('/var/node436/local/tstahl/Features_prop_windows/upper_levels/sheep_%s_fully_cover_tree.txt'%(format(img_nr, "06d")), 'w') for level in levels: levl_boxes = extract_coords(levels[level], pruned_boxes) if levl_boxes != []: for lvl_box in levl_boxes: if lvl_box not in coords: coords.append(lvl_box) f_c.write('%s,%s,%s,%s'%(lvl_box[0],lvl_box[1],lvl_box[2],lvl_box[3])) f_c.write('\n') f_c.close() print 'features for %s with fully_cover_tree samples have to be extracted'%(img_nr) os.system('export PATH=$PATH:/home/koelma/impala/lib/x86_64-linux-gcc') os.system('export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/koelma/impala/third.13.03/x86_64-linux/lib') #print "EuVisual /var/node436/local/tstahl/Images/%s.jpg /var/node436/local/tstahl/Features_prop_windows/Features_upper/sheep_%s_%s.txt --eudata /home/koelma/EuDataBig --imageroifile /var/node436/local/tstahl/Features_prop_windows/upper_levels/sheep_%s_%s.txt"%((format(img_nr, "06d")),format(img_nr, "06d"),subsamples,format(img_nr, "06d"),subsamples) print "EuVisual /var/node436/local/tstahl/Images/%s.jpg /var/node436/local/tstahl/Features_prop_windows/Features_upper/sheep_%s_fully_cover_tree.txt --eudata /home/koelma/EuDataBig --imageroifile /var/node436/local/tstahl/Features_prop_windows/upper_levels/sheep_%s_fully_cover_tree.txt"%((format(img_nr, "06d")),format(img_nr, "06d"),format(img_nr, "06d")) os.system("EuVisual /var/node436/local/tstahl/Images/%s.jpg /var/node436/local/tstahl/Features_prop_windows/Features_upper/sheep_%s_fully_cover_tree.txt --eudata /home/koelma/EuDataBig --imageroifile /var/node436/local/tstahl/Features_prop_windows/upper_levels/sheep_%s_fully_cover_tree.txt"%((format(img_nr, "06d")),format(img_nr, "06d"),format(img_nr, "06d"))) if os.path.isfile('/var/node436/local/tstahl/Features_prop_windows/upper_levels/sheep_%s_fully_cover_tree.txt'%(format(img_nr, "06d"))): f_c = open('/var/node436/local/tstahl/Features_prop_windows/upper_levels/sheep_%s_fully_cover_tree.txt'%(format(img_nr, "06d")), 'r') else: f_c = [] coords = [] if os.path.isfile('/var/node436/local/tstahl/Features_prop_windows/Features_upper/sheep_%s_fully_cover_tree.txt'%(format(img_nr, "06d"))): f = open('/var/node436/local/tstahl/Features_prop_windows/Features_upper/sheep_%s_fully_cover_tree.txt'%(format(img_nr, "06d")), 'r') else: if os.path.isfile('/var/node436/local/tstahl/Features_prop_windows/Features_upper/sheep%s.txt'%(format(img_nr, "06d"))): f = open('/var/node436/local/tstahl/Features_prop_windows/Features_upper/sheep%s.txt'%(format(img_nr, "06d")), 'r') if os.path.isfile('/var/node436/local/tstahl/Features_prop_windows/upper_levels/sheep%s.txt'%(format(img_nr, "06d"))): f_c = open('/var/node436/local/tstahl/Features_prop_windows/upper_levels/sheep%s.txt'%(format(img_nr, "06d")), 'r+') if f_c != []: for i,line in enumerate(f_c): str_ = line.rstrip('\n').split(',') cc = [] for s in str_: cc.append(float(s)) coords.append(cc) if f != []: for i,line in enumerate(f): str_ = line.rstrip('\n').split(',') ff = [] for s in str_: ff.append(float(s)) features.append(ff) #assert len(coords) == len(features) # append x,y of intersections if learn_intersections: for inters,coord in zip(features,coords): # if inters not in pruned_x: pruned_x.append(inters) ol = 0.0 ol = get_intersection_count(coord, ground_truths) pruned_y.append(ol) if mode == 'mean_variance': print 'normalizing' sum_x += np.array(pruned_x).sum(axis=0) n_samples += len(pruned_x) sum_sq_x += (np.array(pruned_x)**2).sum(axis=0) scaler.partial_fit(pruned_x) # Don't cheat - fit only on training data return sum_x,n_samples,sum_sq_x, scaler if less_features: features = [fts[0:features_used] for fts in features] #normalize norm_x = [] if normalize and (mode != 'extract_train' and mode != 'extract_test'): # for p_x in pruned_x: # norm_x.append((p_x-mean)/variance) norm_x = scaler.transform(pruned_x) if features != []: features = scaler.transform(features) else: norm_x = pruned_x data = (G, levels, pruned_y, norm_x, pruned_boxes, ground_truths, alphas) sucs = nx.dfs_successors(G) predecs = nx.dfs_predecessors(G) #preprocess: node - children children = {} last = -1 for node,children_ in zip(sucs.keys(),sucs.values()): if node != last+1: for i in range(last+1,node): children[i] = [] children[node] = children_ elif node == last +1: children[node] = children_ last = node if mode == 'training': if alphas[0] == 0: #if we don't learn the proposals, we learn just the levels: better, because every level has same importance and faster print 'training levels', img_nr for level in levels: print 'level' , level if img_nr in functions: if level in functions[img_nr]: function = functions[img_nr][level] else: function = [] else: functions[img_nr] = {} function = [] w, function = tree_level_regression(class_,function,levels,level,features,coords,scaler,w,norm_x,pruned_y,None,predecs,children,pruned_boxes,learning_rate,alphas,img_nr,jans_idea) if level not in functions[img_nr]: functions[img_nr][level] = function return w, len(pruned_y), len(levels) else: #if we learn proposals, levels with more proposals have more significance...., slow - need to change print 'training patches', img_nr print predecs nodes = list(G.nodes()) for node in nodes: print node if node == 0: w = learn_root(w,norm_x[0],pruned_y[0],learning_rate,alphas) else: for num,n in enumerate(levels.values()): if node in n: level = num break if img_nr in functions: if level in functions[img_nr]: function = functions[img_nr][level] else: function = [] else: functions[img_nr] = {} function = [] #w, function = tree_level_regression(class_,function,levels,level,features,coords,scaler,w,norm_x,pruned_y,node,predecs,children,pruned_boxes,learning_rate,alphas,img_nr) w, function = constrained_regression(class_,function,features,coords,scaler,w,norm_x,pruned_y,node,predecs,children,pruned_boxes,learning_rate,alphas,img_nr,squared_hinge_loss) #TODO: train regressor/classifier that predicts/chooses level. Features: level, number of proposals, number of intersections, avg size of proposal, predictions(for regressor), etc. if level not in functions[img_nr]: functions[img_nr][level] = function return w, len(pruned_y), len(G.nodes()) elif mode == 'scikit_train': clf.partial_fit(norm_x,pruned_y) return clf elif mode == 'loss_train': if alphas[0] == 0: #levels loss__.append(tree_level_loss(class_,features,coords,scaler, w, data, predecs, children,img_nr,-1,functions)) return loss__ else: loss__.append(loss(class_,squared_hinge_loss,features,coords,scaler,w, data, predecs, children,img_nr, -1)) elif mode == 'loss_test' or mode == 'loss_eval': print mode, loss__ if alphas[0] == 0: #levels loss__.append(tree_level_loss(class_,features,coords,scaler, w, data, predecs, children,img_nr,-1,functions)) cpl = max(0, np.dot(w,np.array(norm_x[0]).T)) full_image.append([pruned_y[0],cpl]) return loss__,full_image else: loss__.append(loss(class_,squared_hinge_loss,features,coords,scaler,w, data, predecs, children,img_nr, -1)) cpl = max(0, np.dot(w,np.array(norm_x[0]).T)) full_image.append([pruned_y[0],cpl]) return loss__,full_image elif mode == 'loss_scikit_test' or mode == 'loss_scikit_train': loss__.append(((clf.predict(norm_x) - pruned_y)**2).sum()) return loss__ elif mode == 'levels_train' or mode == 'levels_test': preds = [] for i,x_ in enumerate(norm_x): preds.append(np.dot(w, x_)) cpls = [] truelvls = [] used_boxes_ = [] total_size = surface_area(pruned_boxes, levels[0]) fully_covered_score = 0.0 fully_covered_score_lvls = 0.0 covered_levels = [] print mode, len(levels) for level in levels: function = functions[img_nr][level] cpl,used_boxes,_ = count_per_level([],class_,features,coords,scaler,w, preds, img_nr, pruned_boxes,levels[level], '',function) # clipp negative predictions cpl = max(0,cpl) if used_boxes != []: used_boxes_.append(used_boxes[0][1]) tru = y_p[0] cpls.append(cpl) sa = surface_area(pruned_boxes, levels[level]) sa_co = sa/total_size if sa_co == 1.0: fully_covered_score += cpl fully_covered_score_lvls += 1 covered_levels.append(cpl) truelvls.append(tru) return cpls,truelvls
class Solution: def rightSideView(self, root: TreeNode) -> List[int]: if not root: return [] q = deque() q.append(root) ret = [] while q: tmp = [] path = [] l = len(q) for _ in range(l): node = q.popleft() tmp.append(node) path.append(node.val) ret.append(path[-1]) for node in tmp: if node.left: q.append(node.left) if node.right: q.append(node.right) return ret from utils import create_tree root = create_tree([1, 2, 3, 4, 5, 6]) ret = Solution().rightSideView(root) print(ret)
# testResult = utils.get_shannon_entropy(labels) # print(testResult) # #测试条件熵 # print('测试条件熵') # testResult = utils.get_conditional_entropy(datas[:, 2], labels) # print(testResult) # # 测试信息增益 # print('测试信息增益') # testResult = utils.get_best_gain(datas, labels) # print(testResult) # # 测试建树字典 # print(测试建树字典) # testResult = utils.create_tree(datas_header, datas, labels) # print(testResult) # 测试预测函数 # print(测试预测函数) tree_model = utils.create_tree(datas_header, datas, labels) input_data = ['帅', '好', '矮', '上进'] testResult = utils.predict_result(tree_model, input_data, datas_header) print(testResult) # 测试保存和读取函数 utils.store_tree(tree_model, '嫁不嫁.pkl') # .pkl和.txt什么的都可以,但建议.pkl testResult = utils.restore_tree('嫁不嫁.pkl') print(testResult)
import time import gates as gate from utils import create_tree, save_tree basis = [gate.su2(gate.H), gate.su2(gate.T)] depth = 15 t = time.time() tree = create_tree(basis, max_depth=depth) print('Elapsed time:', time.time() - t) print('Number of gates:', len(tree['names'])) save_tree(tree, 'trees/HT_{}.pkl'.format(depth))
# 把二叉树打印成多行 # https://www.nowcoder.com/practice/445c44d982d04483b04a54f298796288?tpId=13&tqId=11213&tPage=2&rp=2&ru=/ta/coding-interviews&qru=/ta/coding-interviews/question-ranking from utils import create_tree class Solution: # 返回二维列表[[1,2],[4,5]] def Print(self, pRoot): if not pRoot: return [] level = 0 ret = [[]] t = [] t.append([pRoot, 0]) while t: node = t.pop(0) if not node[0]: continue if node[1] != level: ret.append([]) level = node[1] ret[level].append(node[0].val) t.append([node[0].left, node[1] + 1]) t.append([node[0].right, node[1] + 1]) return ret r = create_tree([1, 2, 3, 4, 5, None, 6, 7]) print(Solution().Print(r))
ret = [] ret.append((0, root)) old_v = None count = 0 res = 0 while ret: color, node = ret.pop() if not node: continue if color == 0: ret.append((0, node.right)) ret.append((1, node)) ret.append((0, node.left)) else: print(node.val) if count == 0: res = node.val if not old_v or old_v == node.val: count += 1 old_v = node.val else: count -= 1 old_v = node.val return res from utils import create_tree root = create_tree([3, 2, 3, 1, 3, 3, 4]) ret = Solution().findMode(root) print(ret)
root.left = dps(TreeNode(None), left1, left2) root.right = dps(TreeNode(None), right1, right2) return root root = TreeNode(None) root = dps(root, t1, t2) return root class Solution2: def mergeTrees(self, t1: TreeNode, t2: TreeNode) -> TreeNode: if not t1 or not t2: return t1 or t2 t1.val = t1.val + t2.val t1.left = self.mergeTrees(t1.left, t2.left) t1.right = self.mergeTrees(t1.right, t2.right) return t1 from utils import create_tree, print_tree a = create_tree([1, 3, 2, 5]) b = create_tree([2, 1, 3, 4]) root = Solution().mergeTrees(a, b) print_tree(root)
class Solution: def inverseTree(self, root): if root is None: return [] ret = [] de = deque() level = 1 de.append((root, level)) while de: tmp = deque() for _ in range(len(de)): (node, level) = de.popleft() if level % 2 == 0: tmp.appendleft(node.val) else: tmp.append(node.val) if node.left: de.append((node.left, level + 1)) if node.right: de.append((node.right, level + 1)) ret.append(list(tmp)) return ret from utils import create_tree cli = Solution() print(cli.inverseTree(create_tree([1, 2, 3, 4, 5, 6, 7, 8, 9]))) ###思路: # 使用队列将节点按照正常层次遍历顺序存储 # 使用双端队列将节点的值进行存储。奇数层的节点从右边插入,偶数层的节点从左边插入。
def minibatch_(all_train_imgs,all_test_imgs,clf,scaler,w, loss__,mse,hinge1,hinge2,full_image,alphas,learning_rate,test_imgs, train_imgs,minibatch,subsamples,sum_x,n_samples,sum_sq_x,mean,variance, mode,mous): if mode == 'loss_test' or mode == 'loss_scikit_test' or mode == 'levels_test': X_p, y_p, inv = get_data(class_, test_imgs, train_imgs, minibatch, minibatch + 1, 'test', c, subsamples) else: X_p, y_p, inv = get_data(class_, test_imgs, train_imgs, minibatch, minibatch + 1, 'training', c, subsamples) if X_p != []: boxes = [] ground_truth = inv[0][2] img_nr = inv[0][0] if less_features: X_p = [fts[0:features_used] for fts in X_p] if os.path.isfile('/home/stahl/Coords_prop_windows/'+ (format(img_nr, "06d")) +'.txt'): f = open('/home/stahl/Coords_prop_windows/'+ (format(img_nr, "06d")) +'.txt', 'r') else: print 'warning' for line, y in zip(f, inv): tmp = line.split(',') coord = [] for s in tmp: coord.append(float(s)) boxes.append([coord,y[2]]) assert(len(boxes)<1500) boxes, y_p, X_p = sort_boxes(boxes, y_p, X_p, 0,1500) gr = [] if os.path.isfile('/home/stahl/GroundTruth/%s/%s.txt'%(class_,format(img_nr, "06d"))): gr = open('/home/stahl/GroundTruth/%s/%s.txt'%(class_,format(img_nr, "06d")), 'r') ground_truths = [] if gr != []: #if no class image -> no ground truth. (I think this is only needed for learn _ntersection) for line in gr: tmp = line.split(',') ground_truth = [] for s in tmp: ground_truth.append(int(s)) ground_truths.append(ground_truth) if mode == 'mean_variance': scaler.partial_fit(X_p) # Don't cheat - fit only on training data return scaler # create_tree G, levels = create_tree(boxes) #prune tree to only have levels which fully cover the image # tested if prune_fully_covered: nr_levels_covered = 100 total_size = surface_area(boxes, levels[0]) for level in levels: sa = surface_area(boxes, levels[level]) sa_co = sa/total_size if sa_co != 1.0: G.remove_nodes_from(levels[level]) else: nr_levels_covered = level levels = {k: levels[k] for k in range(0,nr_levels_covered + 1)} #either subsampling or prune_fully_covered #assert(subsampling != prune_fully_covered) # prune levels, speedup + performance levels = {k:v for k,v in levels.iteritems() if k<prune_tree_levels} #prune G in order to remove nodes of the lower levels remaining_nodes = [] for lev in levels.values(): remaining_nodes.extend(lev) for g_node in G.nodes(): if g_node not in remaining_nodes: G.remove_node(g_node) coords = [] features = [] f_c = [] f = [] if learn_intersections and not prune_fully_covered: if os.path.isfile('/home/stahl/Features_prop_windows/upper_levels/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples)): f_c = open('/home/stahl/Features_prop_windows/upper_levels/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples), 'r+') else: if mode == 'extract_train' or mode == 'extract_test': print 'coords for %s with %s samples have to be extracted'%(img_nr,subsamples) f_c = open('/home/stahl/Features_prop_windows/upper_levels/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples), 'w') for level in levels: levl_boxes = extract_coords(levels[level], boxes) if levl_boxes != []: for lvl_box in levl_boxes: if lvl_box not in coords: coords.append(lvl_box) f_c.write('%s,%s,%s,%s'%(lvl_box[0],lvl_box[1],lvl_box[2],lvl_box[3])) f_c.write('\n') f_c.close() print 'features for %s with %s samples have to be extracted'%(img_nr,subsamples) os.system('export PATH=$PATH:/home/koelma/impala/lib/x86_64-linux-gcc') os.system('export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/koelma/impala/third.13.03/x86_64-linux/lib') #print "EuVisual /home/stahl/Images/%s.jpg /home/stahl/Features_prop_windows/Features_upper/sheep_%s_%s.txt --eudata /home/koelma/EuDataBig --imageroifile /home/stahl/Features_prop_windows/upper_levels/sheep_%s_%s.txt"%((format(img_nr, "06d")),format(img_nr, "06d"),subsamples,format(img_nr, "06d"),subsamples) os.system("EuVisual /home/stahl/Images/%s.jpg /home/stahl/Features_prop_windows/Features_upper/%s_%s_%s.txt --eudata /home/koelma/EuDataBig --imageroifile /home/stahl/Features_prop_windows/upper_levels/%s_%s_%s.txt"%(class_,(format(img_nr, "06d")),format(img_nr, "06d"),subsamples,class_,format(img_nr, "06d"),subsamples)) if os.path.isfile('/home/stahl/Features_prop_windows/upper_levels/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples)): f_c = open('/home/stahl/Features_prop_windows/upper_levels/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples), 'r') else: f_c = [] coords = [] if os.path.isfile('/home/stahl/Features_prop_windows/Features_upper/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples)): f = open('/home/stahl/Features_prop_windows/Features_upper/%s_%s_%s.txt'%(class_,format(img_nr, "06d"),subsamples), 'r') elif prune_fully_covered: if os.path.isfile('/home/stahl/Features_prop_windows/upper_levels/sheep_%s_fully_cover_tree.txt'%(format(img_nr, "06d"))): f_c = open('/home/stahl/Features_prop_windows/upper_levels/sheep_%s_fully_cover_tree.txt'%(format(img_nr, "06d")), 'r+') else: if mode == 'extract_train' or mode == 'extract_test': print 'coords for %s with fully_cover_tree samples have to be extracted'%(img_nr) f_c = open('/home/stahl/Features_prop_windows/upper_levels/%s_%s_fully_cover_tree.txt'%(class_,format(img_nr, "06d")), 'w') for level in levels: levl_boxes = extract_coords(levels[level], boxes) if levl_boxes != []: for lvl_box in levl_boxes: if lvl_box not in coords: coords.append(lvl_box) f_c.write('%s,%s,%s,%s'%(lvl_box[0],lvl_box[1],lvl_box[2],lvl_box[3])) f_c.write('\n') f_c.close() print 'features for %s with fully_cover_tree samples have to be extracted'%(img_nr) os.system('export PATH=$PATH:/home/koelma/impala/lib/x86_64-linux-gcc') os.system('export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/koelma/impala/third.13.03/x86_64-linux/lib') #print "EuVisual /home/stahl/Images/%s.jpg /home/stahl/Features_prop_windows/Features_upper/sheep_%s_%s.txt --eudata /home/koelma/EuDataBig --imageroifile /home/stahl/Features_prop_windows/upper_levels/sheep_%s_%s.txt"%((format(img_nr, "06d")),format(img_nr, "06d"),subsamples,format(img_nr, "06d"),subsamples) os.system("EuVisual /home/stahl/Images/%s.jpg /home/stahl/Features_prop_windows/Features_upper/%s_%s_fully_cover_tree.txt --eudata /home/koelma/EuDataBig --imageroifile /home/stahl/Features_prop_windows/upper_levels/%s_%s_fully_cover_tree.txt"%(class_,(format(img_nr, "06d")),format(img_nr, "06d"),class_,format(img_nr, "06d"))) if os.path.isfile('/home/stahl/Features_prop_windows/upper_levels/%s_%s_fully_cover_tree.txt'%(class_,format(img_nr, "06d"))): f_c = open('/home/stahl/Features_prop_windows/upper_levels/%s_%s_fully_cover_tree.txt'%(class_,format(img_nr, "06d")), 'r') else: f_c = [] coords = [] if os.path.isfile('/home/stahl/Features_prop_windows/Features_upper/sheep_%s_fully_cover_tree.txt'%(format(img_nr, "06d"))): f = open('/home/stahl/Features_prop_windows/Features_upper/sheep_%s_fully_cover_tree.txt'%(format(img_nr, "06d")), 'r') # else: #we don't need to load intersections # if os.path.isfile('/home/stahl/Features_prop_windows/Features_upper/%s%s.txt'%(class_,format(img_nr, "06d"))): # f = open('/home/stahl/Features_prop_windows/Features_upper/%s%s.txt'%(class_,format(img_nr, "06d")), 'r') # if os.path.isfile('/home/stahl/Features_prop_windows/upper_levels/%s%s.txt'%(class_,format(img_nr, "06d"))): # f_c = open('/home/stahl/Features_prop_windows/upper_levels/%s%s.txt'%(class_,format(img_nr, "06d")), 'r+') # else: # print '/home/stahl/Features_prop_windows/upper_levels/%s%s.txt does not exist'%(class_,format(img_nr, "06d")) for i,line in enumerate(f_c): str_ = line.rstrip('\n').split(',') cc = [] for s in str_: cc.append(float(s)) coords.append(cc) for i,line in enumerate(f): str_ = line.rstrip('\n').split(',') ff = [] for s in str_: ff.append(float(s)) features.append(ff) if less_features: features = [fts[0:features_used] for fts in features] if normalize and features != []: features = scaler.transform(features) print len(y_p), len(X_p) print len(features), len(coords) assert len(coords) == len(features) # append x,y of intersections #if learn_intersections: # for inters,coord in zip(features,coords): # if inters not in pruned_x: # X_p.append(inters) # ol = 0.0 # ol = get_intersection_count(coord, ground_truths) # y_p.append(ol) print len(y_p), len(X_p) #normalize norm_x = [] if normalize: # for p_x in pruned_x: # norm_x.append((p_x-mean)/variance) norm_x = scaler.transform(X_p) else: norm_x = X_p data = (G, levels, y_p, norm_x, boxes, ground_truths, alphas) sucs = nx.dfs_successors(G) predecs = nx.dfs_predecessors(G) #preprocess: node - children children = {} last = -1 for node,children_ in zip(sucs.keys(),sucs.values()): if node != last+1: for i in range(last+1,node): children[i] = [] children[node] = children_ elif node == last +1: children[node] = children_ last = node if mode == 'train': if alphas[2] == 0 and alphas[3] == 0: #just learn proposals and intersections # only use proposals and intersections used in pruned tree used_ind = get_used_proposals(G, boxes, coords, levels) used_x = [] used_y = [] for ind in used_ind['prop']: used_x.append(norm_x[ind]) used_y.append(y_p[ind]) for ind in used_ind['inters']: used_x.append(features[ind]) used_y.append(get_intersection_count(coords[ind], ground_truths)) print len(used_x),len(used_y) for x_i,y_i in zip(used_x,used_y): w = learn_root(w,x_i,y_i,learning_rate,alphas) else: nodes = list(G.nodes()) for node in nodes: if node == 0: if alphas[0] != 0: w = learn_root(w,norm_x[0],y_p[0],learning_rate,alphas) else: print 'learn nothing' else: w = constrained_regression(class_,features,coords,scaler,w,norm_x,y_p,node,predecs,children,boxes,learning_rate,alphas,img_nr, squared_hinge_loss) return w, len(y_p), len(G.nodes()) elif mode == 'scikit_train': print norm_x,y_p clf.partial_fit(norm_x,y_p) return clf elif mode == 'loss_train' or mode == 'loss_test': loss__.append(loss(class_,squared_hinge_loss,features,coords,scaler, w, data, predecs, children,img_nr,-1)) # mse.append(((data[2] - np.dot(w,np.array(data[3]).T)) ** 2).sum()) # a2 = alphas[2] # data = (G, levels, y_p, norm_x, boxes, ground_truths, [0,0,a2,0]) # hinge1.append(loss(class_,squared_hinge_loss,features,coords,scaler, w, data, predecs, children,img_nr,-1)) # a3 = alphas[3] # data = (G, levels, y_p, norm_x, boxes, ground_truths, [0,0,0,a3]) # hinge2.append(loss(class_,squared_hinge_loss,features,coords,scaler, w, data, predecs, children,img_nr,-1)) full_image.append([y_p[0],max(0,np.dot(w,np.array(norm_x[0]).T))]) return loss__, full_image elif mode == 'loss_scikit_test' or mode == 'loss_scikit_train': loss__.append(((clf.predict(norm_x) - y_p)**2).sum()) return loss__ elif mode == 'levels_train' or mode == 'levels_test': #im = mpimg.imread('/home/stahl/Images/'+ (format(img_nr, "06d")) +'.jpg') preds = [] for i,x_ in enumerate(norm_x): preds.append(np.dot(w, x_)) cpls = [] truelvls = [] used_boxes_ = [] total_size = surface_area(boxes, levels[0]) fully_covered_score = 0.0 fully_covered_score_lvls = 0.0 covered_levels = [] for level in levels: #tru and truelvls was in order to check if count_per_level method is correct cpl,used_boxes,_ = count_per_level(class_,features,coords,scaler,w, preds, img_nr, boxes,levels[level], '',[]) cpl = max(0,cpl) if used_boxes != []: used_boxes_.append(used_boxes[0][1]) tru,_,_ = count_per_level(class_,features,coords,scaler,w, preds, img_nr, boxes,levels[level], 'gt',[]) cpls.append(cpl) sa = surface_area(boxes, levels[level]) sa_co = sa/total_size if sa_co == 1.0: fully_covered_score += cpl fully_covered_score_lvls += 1 covered_levels.append(cpl) truelvls.append(tru) return cpls, truelvls, used_boxes_,boxes,preds,fully_covered_score/fully_covered_score_lvls,covered_levels