def remove_hidden(items, descs): mark_order(items, sorted(items)[0], 0) removed = [] for item in sorted(items): me = items[item] if 'x' not in me: continue if me['parent'] in removed: removed.append(item) print('removing child %s' % util.describe(items[item])) if item in descs: print('WARNING: removing labeled item!') continue for other in sorted(items): him = items[other] if him['order'] > me['order']: if (him['x'] <= me['x'] and him['x'] + him['width'] >= me['x'] + me['width'] and him['y'] <= me['y'] and him['y'] + him['height'] >= me['y'] + me['height'] and not (him['x'] == me['x'] and him['y'] == me['y'] and him['width'] == me['width'] and him['height'] == me['height'])): removed.append(item) logger.info('removing %s because %s', util.describe(me), util.describe(him)) if item in descs: print('WARNING: removing labeled item!') break for item in sorted(removed): parent = items[item]['parent'] del items[item] if parent in items: items[parent]['children'].remove(item)
def draw_ulh(self, minuit=None, bins=100, ax=None, bound=None, parmloc=(0.05, 0.95), nfbins=200, print_par=True, grid=True, args=None, errors=None, parts=False, show_errbars='normal'): data_ret = None error_ret = None total_ret = None part_ret = [] ax = plt.gca() if ax is None else ax arg, error = _get_args_and_errors(self, minuit, args, errors) n,e= np.histogram(self.data, bins=bins, range=bound, weights=self.weights) dataint= (n*np.diff(e)).sum() data_ret = (e, n) if not show_errbars: pp= ax.hist(mid(e), bins=e, weights=n, histtype='step') error_ret = (np.sqrt(n), np.sqrt(n)) else: w2= None if show_errbars=='normal': w2=n error_ret = (np.sqrt(n), np.sqrt(n)) elif show_errbars=='sumw2': weights= None if self.weights!= None: weights= self.weights**2 w2,e= np.histogram(self.data, bins=e, weights=weights) error_ret = (np.sqrt(w2), np.sqrt(w2)) else: raise ValueError('show_errbars must be \'normal\' or \'sumw2\'') pp= ax.errorbar(mid(e), n, np.sqrt(w2) , fmt='b+', capsize=0) #bound = (e[0], e[-1]) draw_arg = [('lw', 2)] if not parts: draw_arg.append(('color', 'r')) # Draw pdf with finer bins ef= np.linspace(e[0],e[-1], nfbins+1) scale= dataint if not self.extended else nfbins/float(bins) total_ret = draw_pdf_with_edges(self.f, arg, ef, ax=ax, density=not self.extended, scale=scale, **dict(draw_arg)) if parts: f_parts = getattr(self.f, 'parts', None) if f_parts is not None: for p in f_parts(): ret = draw_pdf_with_edges(p, arg, ef, ax=ax, scale=scale, density=not self.extended) part_ret.append(ret) ax.grid(grid) txt = _param_text(describe(self), arg, error) if print_par: ax.text(parmloc[0], parmloc[1], txt, ha='left', va='top', transform=ax.transAxes) return (data_ret, error_ret, total_ret, part_ret)
def draw_residual_ulh( self, minuit=None, bins=100, ax=None, bound=None, parmloc=(0.05, 0.95), print_par=False, grid=True, args=None, errors=None, show_errbars=True, errbar_algo="normal", norm=False, ): ax = plt.gca() if ax is None else ax arg, error = _get_args_and_errors(self, minuit, args, errors) n, e = np.histogram(self.data, bins=bins, range=bound, weights=self.weights) dataint = (n * np.diff(e)).sum() scale = dataint if not self.extended else 1.0 w2 = None if errbar_algo == "normal": w2 = n elif errbar_algo == "sumw2": weights = None if self.weights != None: weights = self.weights ** 2 w2, e = np.histogram(self.data, bins=e, weights=weights) else: raise ValueError("errbar_algo must be 'normal' or 'sumw2'") yerr = np.sqrt(w2) arg = parse_arg(self.f, arg, 1) if isinstance(arg, dict) else arg yf = vector_apply(self.f, mid(e), *arg) yf *= scale * np.diff(e) if self.extended else scale n = n - yf if norm: sel = yerr > 0 n[sel] /= yerr[sel] yerr = np.ones(len(yerr)) if show_errbars: pp = ax.errorbar(mid(e), n, yerr, fmt="b.", capsize=0) else: # No errorbars pp = ax.bar(e[:-1], n, width=np.diff(e)) # bound = (e[0], e[-1]) # draw_arg = [('lw', 2), ('color', 'r')] ax.plot([e[0], e[-1]], [0.0, 0.0], "r-") ax.grid(grid) txt = _param_text(describe(self), arg, error) if print_par: ax.text(parmloc[0], parmloc[1], txt, ha="left", va="top", transform=ax.transAxes)
def draw_blh(self, minuit=None, parmloc=(0.05, 0.95), nfbins=1000, ax=None, print_par=True, grid=True, args=None, errors=None, parts=False): data_ret = None error_ret = None total_ret = None part_ret = [] ax = plt.gca() if ax is None else ax arg, error = _get_args_and_errors(self, minuit, args, errors) m = mid(self.edges) if self.use_w2: err = np.sqrt(self.w2) else: err = np.sqrt(self.h) n= np.copy(self.h) dataint= (n*np.diff(self.edges)).sum() scale= dataint if not self.extended else 1.0 ax.errorbar(m, n, err, fmt='+', capsize=0) data_ret = (self.edges, n) error_ret = (err, err) draw_arg = [('lw', 2)] if not parts: draw_arg.append(('color', 'r')) bound = (self.edges[0], self.edges[-1]) #scale back to bins if self.extended: scale= nfbins/float(self.bins) total_ret = draw_pdf(self.f, arg, bins=nfbins, bound=bound, ax=ax, density=not self.extended, scale=scale, **dict(draw_arg)) if parts: f_parts = getattr(self.f, 'parts', None) if f_parts is not None: for p in f_parts(): tmp = draw_pdf(p, arg, bins=nfbins, bound=bound, ax=ax, density=not self.extended, scale=scale) part_ret.append(tmp) ax.grid(grid) txt = _param_text(describe(self), arg, error) if print_par: ax.text(parmloc[0], parmloc[1], txt, ha='left', va='top', transform=ax.transAxes) return (data_ret, error_ret, total_ret, part_ret)
def draw_x2(self, minuit=None, ax=None, parmloc=(0.05, 0.95), print_par=True, args=None, errors=None, grid=True, parts=False, nbins=None): data_ret = None error_ret = None total_ret = None part_ret = [] ax = plt.gca() if ax is None else ax arg, error = _get_args_and_errors(self, minuit, args, errors) x=self.x y=self.y data_err = self.error # Draw data points data_ret = x,y if data_err is None: ax.plot(x, y, '+') err_ret = (np.ones(len(self.x)), np.ones(len(self.x))) else: ax.errorbar(x, y, data_err, fmt='+', capsize=0) err_ret = (data_err, data_err) draw_arg = [('lw', 2)] draw_arg.append(('color', 'r')) # Draw PDF curve(s) if nbins is not None: x = np.linspace(x[0],x[-1], nbins) total_ret = draw_pdf_with_midpoints(self.f, arg, x, ax=ax, **dict(draw_arg)) if parts: f_parts = getattr(self.f, 'parts', None) if f_parts is not None: for p in f_parts(): tmp = draw_pdf_with_midpoints(p, arg, x, ax=ax, **dict(draw_arg)) part_ret.append(tmp) # Print text txt = _param_text(describe(self), arg, error) chi2 = self(*arg) if self.ndof > 0: txt+=u'chi2/ndof = %5.4g(%5.4g/%d)'%(chi2/self.ndof, chi2, self.ndof) else: txt+=u'chi2/ndof = (%5.4g/%d)'%(chi2, self.ndof) if print_par: ax.text(parmloc[0], parmloc[1], txt, ha='left', va='top', transform=ax.transAxes) ax.grid(grid) return (data_ret, error_ret, total_ret , part_ret)
def draw_bx2(self, minuit=None, parmloc=(0.05, 0.95), nfbins=500, ax=None, print_par=True, args=None, errors=None, parts=False, grid=True): data_ret = None error_ret = None total_ret = None part_ret = [] ax = plt.gca() if ax is None else ax arg, error = _get_args_and_errors(self, minuit, args, errors) m = mid(self.edges) ax.errorbar(m, self.h, self.err, fmt='+', capsize=0) data_ret = (self.edges, self.h) error_ret = (self.err, self.err) bound = (self.edges[0], self.edges[-1]) scale = nfbins/float(self.bins) #scale back to bins draw_arg = [('lw', 2)] if not parts: draw_arg.append(('color', 'r')) total_ret = draw_pdf(self.f, arg, bins=nfbins, bound=bound, ax=ax, density=False, scale=scale, **dict(draw_arg)) if parts: f_parts = getattr(self.f, 'parts', None) if f_parts is not None: for p in f_parts(): tmp = draw_pdf(p, arg, bound=bound, bins=nfbins, ax=ax, density=False, scale=scale) part_ret.append(tmp) ax.grid(grid) txt = _param_text(describe(self), arg, error) chi2 = self(*arg) if self.ndof > 0: txt+=u'chi2/ndof = %5.4g(%5.4g/%d)'%(chi2/self.ndof, chi2, self.ndof) else: txt+=u'chi2/ndof = (%5.4g/%d)'%(chi2, self.ndof) if print_par: ax.text(parmloc[0], parmloc[1], txt, ha='left', va='top', transform=ax.transAxes) return (data_ret, error_ret, total_ret, part_ret)
def gen_toy(f, nsample, bound, accuracy=10000, quiet=True, **kwd): """ generate ntoy :param f: :param nsample: :param ntoy: :param bound: :param accuracy: :param quiet: :param kwd: the rest of keyword argument will be passed to f :return: numpy.ndarray """ #based on inverting cdf this is fast but you will need to give it a reasonable range #unlike roofit which is based on accept reject vnames = describe(f) if not quiet: print vnames my_arg = [kwd[v] for v in vnames[1:]] #random number #if accuracy is None: accuracy=10*numtoys r = npr.random_sample(nsample) x = np.linspace(bound[0], bound[1], accuracy) pdf = _vector_apply(f, x, tuple(my_arg)) cdf = compute_cdf(pdf, x) if cdf[-1] < 0.01: warn(SmallIntegralWarning('Integral for given funcition is' ' really low. Did you give it a reasonable range?')) cdfnorm = cdf[-1] cdf /= cdfnorm #now convert that to toy ret = invert_cdf(r, cdf, x) if not quiet: #move this to plotting plt.figure() plt.title('comparison') numbin = 100 h, e = np.histogram(ret, bins=numbin) mp = (e[1:]+e[:-1])/2. err = np.sqrt(h) plt.errorbar(mp, h, err, fmt='.b') bw = e[1] - e[0] y = pdf * len(ret) / cdfnorm * bw ylow = y + np.sqrt(y) yhigh = y - np.sqrt(y) plt.plot(x, y, label='pdf', color='r') plt.fill_between(x, yhigh, ylow, color='g', alpha=0.2) plt.grid(True) plt.xlim(bound) plt.ylim(ymin=0) return ret
def extract(datalist, like, bbox, dst_crs, out_path): """Extract/reproject data within lat/lon bbox """ # check that enough options are provided if not like and not bbox: util.error("Provide bounds as either a 'like' dataset or a bbox") # read input csv listing layers layers = [s for s in csv.DictReader(open(datalist, 'rb'))] # parse bbox if bbox: bbox = tuple(map(float, bbox.split(','))) # derive bbox/crs from aoi/like layer (ignoring any provided bbox) if like: bbox = util.get_bbox(like) dst_crs = util.get_crs(like) # parse provided epsg code elif dst_crs: dst_crs = from_epsg(dst_crs.split(':')[1]) # name is derived from config file name if not out_path: b = os.path.basename(datalist) out_path = os.path.join(os.getcwd(), os.path.splitext(b)[0]) util.make_sure_path_exists(out_path) # process each layer for layer in layers: click.echo('Extracting %s' % layer['name']) if util.describe(layer['path'])['type'] == 'VECTOR': util.bbox_copy(layer['path'], os.path.join(out_path, layer['name'] + ".shp"), bbox, in_layer=layer['layer'], dst_crs=dst_crs) elif util.describe(layer['path'])['type'] == 'RASTER': util.bbox_copyraster(layer['path'], os.path.join(out_path, layer['name'] + '.tif'), bbox, dst_crs=dst_crs)
def draw_residual_blh(self, minuit=None, parmloc=(0.05, 0.95), ax=None, print_par=False, args=None, errors=None, norm=False, grid=True): ax = plt.gca() if ax is None else ax arg, error = _get_args_and_errors(self, minuit, args, errors) m = mid(self.edges) if self.use_w2: err = np.sqrt(self.w2) else: err = np.sqrt(self.h) n = np.copy(self.h) dataint = (n * np.diff(self.edges)).sum() scale = dataint if not self.extended else 1.0 arg = parse_arg(self.f, arg, 1) if isinstance(arg, dict) else arg yf = vector_apply(self.f, m, *arg) yf *= (scale * np.diff(self.edges) if self.extended else scale) n = n - yf if norm: sel = err > 0 n[sel] /= err[sel] err = np.ones(len(err)) ax.errorbar(m, n, err, fmt='.') ax.plot([self.edges[0], self.edges[-1]], [0., 0.], 'r-') ax.grid(grid) txt = _param_text(describe(self), arg, error) if print_par: ax.text(parmloc[0], parmloc[1], txt, ha='left', va='top', transform=ax.transAxes)
def postprocess(self, tree, items, history): for nodeid in sorted(tree): if nodeid in tree: node = tree[nodeid] children = [] for childid in tree: if tree[childid]['parent'] == nodeid: children.append(childid) if len(children) > 0: child0 = tree[children[0]] if len(children) > 1: child1 = tree[children[1]] if config.REMOVE_OVERLAP_OLD: if (len(children) > 1 and node['class'] == 'FrameLayout' and child0['width'] == child1['width'] == config.width and child0['height'] == child1['height'] == config.real_height_nostatus): st1 = subtree_collect(tree, children[-1]) if len(st1) > 1: for (olditems, oldtree) in reversed(history): oldme = find_in_old(tree, items, nodeid, oldtree, olditems, True) if oldme is not None: logger.debug("old me: %s", util.describe(olditems[oldme])) appear_in_old = [] for childid in children: oldchild = find_in_old( tree, items, childid, oldtree, olditems, False) if oldchild is not None: appear_in_old.append(childid) else: break if (appear_in_old != [] and len(appear_in_old) < len(children)): for childid in appear_in_old: self.del_subtree(tree, childid) self.incstat("overlap old") logger.debug("overlap old %s children %s", node['id'], appear_in_old) return True return False
def fix_size(items, filename): imgfile = os.path.splitext(filename)[0] + '.png' if not os.path.exists(imgfile): return img = Image.open(imgfile) for itemid in sorted(items): if items[itemid]['width'] + items[itemid]['x'] > 0: orig_width = items[itemid]['width'] + items[itemid]['x'] orig_height = items[itemid]['height'] + items[itemid]['y'] minid = itemid break #orig_width = items[1]['width'] + items[1]['x'] #orig_height = items[1]['height'] + items[1]['y'] if orig_width == config.width or orig_height == config.real_height: return if items[minid]['x'] != 0: # not at top-left, usually dialog if orig_width <= config.width and orig_height <= config.real_height: # seems fine return real_height = img.height * orig_width / img.width for itemid in items: item = items[itemid] if (item['x'] + item['width'] < 0 or item['x'] > orig_width or item['y'] + item['height'] < 0 or item['y'] > real_height): logger.debug("removing OOS %s", util.describe(item)) item['x'] = item['y'] = item['width'] = item['height'] = 0 ratio = 1.0 * orig_width / img.width img_ratio = find_closest(ratio, SCALE_RATIOS) fix_ratio = img_ratio * img.width / config.width logger.debug("fixing %s: %d != %d %d, ratio %.3f -> %.3f -> %.3f" % (filename, orig_width, config.width, img.width, ratio, img_ratio, fix_ratio)) for itemid in items: items[itemid]['x'] = int(items[itemid]['x'] / fix_ratio) items[itemid]['y'] = int(items[itemid]['y'] / fix_ratio) items[itemid]['width'] = int(items[itemid]['width'] / fix_ratio) items[itemid]['height'] = int(items[itemid]['height'] / fix_ratio)
def draw_residual_blh(self, minuit=None, parmloc=(0.05, 0.95), ax=None, print_par=False, args=None, errors=None, norm=False, grid=True): ax = plt.gca() if ax is None else ax arg, error = _get_args_and_errors(self, minuit, args, errors) m = mid(self.edges) if self.use_w2: err = np.sqrt(self.w2) else: err = np.sqrt(self.h) n= np.copy(self.h) dataint= (n*np.diff(self.edges)).sum() scale= dataint if not self.extended else 1.0 arg = parse_arg(self.f, arg, 1) if isinstance(arg, dict) else arg yf = vector_apply(self.f, m, *arg) yf*= (scale*np.diff(self.edges) if self.extended else scale) n = n- yf if norm: sel= err>0 n[sel]/= err[sel] err= np.ones(len(err)) ax.errorbar(m, n, err, fmt='+', capsize=0) ax.plot([self.edges[0],self.edges[-1]],[0.,0.], 'r-') ax.grid(grid) txt = _param_text(describe(self), arg, error) if print_par: ax.text(parmloc[0], parmloc[1], txt, ha='left', va='top', transform=ax.transAxes) return m, n, err
def draw_residual_ulh(self, minuit=None, bins=100, ax=None, bound=None, parmloc=(0.05, 0.95), print_par=False, grid=True, args=None, errors=None, show_errbars=True, errbar_algo='normal', norm=False): ax = plt.gca() if ax is None else ax arg, error = _get_args_and_errors(self, minuit, args, errors) n, e = np.histogram(self.data, bins=bins, range=bound, weights=self.weights) dataint = (n * np.diff(e)).sum() scale = dataint if not self.extended else 1.0 w2 = None if errbar_algo == 'normal': w2 = n elif errbar_algo == 'sumw2': weights = None if self.weights != None: weights = self.weights**2 w2, e = np.histogram(self.data, bins=e, weights=weights) else: raise ValueError('errbar_algo must be \'normal\' or \'sumw2\'') yerr = np.sqrt(w2) arg = parse_arg(self.f, arg, 1) if isinstance(arg, dict) else arg yf = vector_apply(self.f, mid(e), *arg) yf *= (scale * np.diff(e) if self.extended else scale) n = n - yf if norm: sel = yerr > 0 n[sel] /= yerr[sel] yerr = np.ones(len(yerr)) if show_errbars: pp = ax.errorbar(mid(e), n, yerr, fmt='b.', capsize=0) else: # No errorbars pp = ax.bar(e[:-1], n, width=np.diff(e)) #bound = (e[0], e[-1]) #draw_arg = [('lw', 2), ('color', 'r')] ax.plot([e[0], e[-1]], [0., 0.], 'r-') ax.grid(grid) txt = _param_text(describe(self), arg, error) if print_par: ax.text(parmloc[0], parmloc[1], txt, ha='left', va='top', transform=ax.transAxes)
def draw_ulh(self, minuit=None, bins=100, ax=None, bound=None, parmloc=(0.05, 0.95), nfbins=200, print_par=True, grid=True, args=None, errors=None, parts=False, show_errbars='normal', no_plot=False): data_ret = None error_ret = None total_ret = None part_ret = [] ax = plt.gca() if ax is None and not no_plot else ax arg, error = _get_args_and_errors(self, minuit, args, errors) n, e = np.histogram(self.data, bins=bins, range=bound, weights=self.weights) dataint = (n * np.diff(e)).sum() data_ret = (e, n) if not show_errbars: if not no_plot: pp = ax.hist(mid(e), bins=e, weights=n, histtype='step') error_ret = (np.sqrt(n), np.sqrt(n)) else: w2 = None if show_errbars == 'normal': w2 = n error_ret = (np.sqrt(n), np.sqrt(n)) elif show_errbars == 'sumw2': weights = None if self.weights != None: weights = self.weights**2 w2, e = np.histogram(self.data, bins=e, weights=weights) error_ret = (np.sqrt(w2), np.sqrt(w2)) else: raise ValueError('show_errbars must be \'normal\' or \'sumw2\'') if not no_plot: pp = ax.errorbar(mid(e), n, np.sqrt(w2), fmt='b.', capsize=0) #bound = (e[0], e[-1]) draw_arg = [('lw', 2)] if not parts: draw_arg.append(('color', 'r')) # Draw pdf with finer bins ef = np.linspace(e[0], e[-1], nfbins + 1) scale = dataint if not self.extended else nfbins / float(bins) total_ret = draw_pdf_with_edges(self.f, arg, ef, ax=ax, density=not self.extended, scale=scale, **dict(draw_arg)) if parts: f_parts = getattr(self.f, 'parts', None) if f_parts is not None: for p in f_parts(): ret = draw_pdf_with_edges(p, arg, ef, ax=ax, scale=scale, density=not self.extended, no_plot=no_plot) part_ret.append(ret) if not no_plot: ax.grid(grid) txt = _param_text(describe(self), arg, error) if not no_plot and print_par: ax.text(parmloc[0], parmloc[1], txt, ha='left', va='top', transform=ax.transAxes) return (data_ret, error_ret, total_ret, part_ret)
def draw_blh(self, minuit=None, parmloc=(0.05, 0.95), nfbins=1000, ax=None, print_par=True, grid=True, args=None, errors=None, parts=False, no_plot=False): data_ret = None error_ret = None total_ret = None part_ret = [] ax = plt.gca() if ax is None and not no_plot else ax arg, error = _get_args_and_errors(self, minuit, args, errors) m = mid(self.edges) if self.use_w2: err = np.sqrt(self.w2) else: err = np.sqrt(self.h) n = np.copy(self.h) dataint = (n * np.diff(self.edges)).sum() scale = dataint if not self.extended else 1.0 if not no_plot: ax.errorbar(m, n, err, fmt='.') data_ret = (self.edges, n) error_ret = (err, err) draw_arg = [('lw', 2)] if not parts: draw_arg.append(('color', 'r')) bound = (self.edges[0], self.edges[-1]) #scale back to bins if self.extended: scale = nfbins / float(self.bins) total_ret = draw_pdf(self.f, arg, bins=nfbins, bound=bound, ax=ax, density=not self.extended, scale=scale, no_plot=no_plot, **dict(draw_arg)) if parts: f_parts = getattr(self.f, 'parts', None) if f_parts is not None: for p in f_parts(): tmp = draw_pdf(p, arg, bins=nfbins, bound=bound, ax=ax, density=not self.extended, scale=scale, no_plot=no_plot) part_ret.append(tmp) if not no_plot: ax.grid(grid) txt = _param_text(describe(self), arg, error) if print_par and not no_plot: ax.text(parmloc[0], parmloc[1], txt, ha='left', va='top', transform=ax.transAxes) return (data_ret, error_ret, total_ret, part_ret)
def draw_x2( self, minuit=None, ax=None, parmloc=(0.05, 0.95), print_par=True, args=None, errors=None, grid=True, parts=False, no_plot=False, ): data_ret = None error_ret = None total_ret = None part_ret = [] ax = plt.gca() if ax is None and not no_plot else ax arg, error = _get_args_and_errors(self, minuit, args, errors) x = self.x y = self.y data_err = self.error data_ret = x, y if data_err is None: if not no_plot: ax.plot(x, y, "+") err_ret = (np.ones(len(self.x)), np.ones(len(self.x))) else: if not no_plot: ax.errorbar(x, y, data_err, fmt=".") err_ret = (data_err, data_err) draw_arg = [("lw", 2)] draw_arg.append(("color", "r")) total_ret = draw_pdf_with_midpoints(self.f, arg, x, ax=ax, no_plot=no_plot, **dict(draw_arg)) if not no_plot: ax.grid(grid) txt = _param_text(describe(self), arg, error) chi2 = self(*arg) if self.ndof > 0: txt += u"chi2/ndof = %5.4g(%5.4g/%d)" % (chi2 / self.ndof, chi2, self.ndof) else: txt += u"chi2/ndof = (%5.4g/%d)" % (chi2, self.ndof) if parts: f_parts = getattr(self.f, "parts", None) if f_parts is not None: for p in f_parts(): tmp = draw_pdf_with_midpoints(p, arg, x, ax=ax, no_plot=no_plot, **dict(draw_arg)) part_ret.append(tmp) if print_par and not no_plot: ax.text(parmloc[0], parmloc[1], txt, ha="left", va="top", transform=ax.transAxes) return (data_ret, error_ret, total_ret, part_ret)
def analyze_items(self, items, descs, regs, print_rets, print_error, history): self.fix_size_to_scr(items) self.mark_important(items) parents = [0] newtree = {} for itemid in preorder(items, 0): item = items[itemid] if item['important']: nodeinfo = {} par = -1 while len(parents) > 0 and ( item['depth'] <= items[parents[-1]]['depth'] or items[item['parent']]['depth'] < items[parents[-1]]['depth']): parents.pop() if len(parents) > 0: par = parents[-1] if print_rets: print('%3d' % itemid, self.get_prefix(items, itemid), util.describe(item), item['parent'], par) # orig = item nodeinfo['parent'] = par nodeinfo['class'] = item['class'] nodeinfo['text'] = item['text'] nodeinfo['desc'] = item['desc'] nodeinfo['id'] = item['id'] nodeinfo['raw'] = [itemid] nodeinfo['width'] = item['width'] nodeinfo['height'] = item['height'] nodeinfo['x'] = item['x'] nodeinfo['y'] = item['y'] nodeinfo['origw'] = nodeinfo['width'] nodeinfo['origh'] = nodeinfo['height'] nodeinfo['origx'] = nodeinfo['x'] nodeinfo['origy'] = nodeinfo['y'] nodeinfo['click'] = item['click'] nodeinfo['scroll'] = item['scroll'] nodeinfo['password'] = item['password'] nodeinfo['focused'] = item['focused'] nodeinfo['checkable'] = item['checkable'] nodeinfo['childid'] = 0 # placemarker nodeinfo['webview'] = item['webview'] nodeinfo['origitem'] = item if itemid in descs: nodeinfo['tags'] = [descs[itemid]] else: nodeinfo['tags'] = [] if itemid in regs: nodeinfo['regs'] = [regs[itemid]] else: nodeinfo['regs'] = [] newtree[itemid] = nodeinfo parents.append(itemid) self.incstat("before", len(newtree)) if print_rets: util.print_tree(newtree) while self.process(newtree, items, history): if print_rets: util.print_tree(newtree) if print_error: for itemid in descs: found = False for nodeid in newtree: if itemid in newtree[nodeid]['raw']: found = True break if not found: logger.error("REMOVED: %d %s %s", itemid, descs[itemid], util.describe(items[itemid])) pass while self.postprocess(newtree, items, history): pass self.incstat("after", len(newtree)) collect_children(newtree) return newtree
def analyze(files, print_rets=False, show_progress=False, print_items=False, print_error=False, show_ocr=False, show_stat=False, use_ocr=False): ret = [] if show_progress: progress = progressbar.ProgressBar() items = progress(files) else: items = files analyzer = Analyzer() for filename in items: filebase = os.path.splitext(filename)[0] logger.debug("analyzing %s" % filename) (items, descs, regs) = load_case(filename) if print_items: util.printitems(items) start_time = time.time() newtree = analyzer.analyze_items(items, descs, regs, print_items, print_error, []) ret.append(newtree) if print_rets: util.print_tree(newtree) logger.info("Time used: %.3fs", time.time() - start_time) if use_ocr: hidden.add_ocrinfo(newtree, filebase + '.png') hidden.find_hidden_ocr(newtree) util.print_tree(newtree) if print_rets: dlg = dialog.detect_dialog(newtree) if dlg[0]: logger.info("I think this is dialog") for btnid in dlg[1]: logger.info("btn: %s", util.describe_node(newtree[btnid])) logger.info( "is: %s", dialog.detect_dialog_button(newtree, btnid, dlg[1])) logger.info("decide to click: %s", dialog.decide_dialog_action(newtree)) if print_error: for itemid in descs: found = False for nodeid in newtree: if itemid in newtree[nodeid]['raw']: found = True break if not found: logger.error("REMOVED: %s %d %s %s", os.path.basename(filename), itemid, descs[itemid], util.describe(items[itemid])) if show_stat: analyzer.show_stat() return ret
def draw_bx2( self, minuit=None, parmloc=(0.05, 0.95), nfbins=500, ax=None, print_par=True, args=None, errors=None, parts=False, grid=True, no_plot=False, ): data_ret = None error_ret = None total_ret = None part_ret = [] ax = plt.gca() if ax is None and not no_plot else ax arg, error = _get_args_and_errors(self, minuit, args, errors) m = mid(self.edges) if not no_plot: ax.errorbar(m, self.h, self.err, fmt=".") data_ret = (self.edges, self.h) error_ret = (self.err, self.err) bound = (self.edges[0], self.edges[-1]) scale = nfbins / float(self.bins) # scale back to bins draw_arg = [("lw", 2)] if not parts: draw_arg.append(("color", "r")) total_ret = draw_pdf( self.f, arg, bins=nfbins, bound=bound, ax=ax, density=False, scale=scale, no_plot=no_plot, **dict(draw_arg) ) if parts: f_parts = getattr(self.f, "parts", None) if f_parts is not None: for p in f_parts(): tmp = draw_pdf(p, arg, bound=bound, bins=nfbins, ax=ax, density=False, scale=scale, no_plot=no_plot) part_ret.append(tmp) if not no_plot: ax.grid(grid) txt = _param_text(describe(self), arg, error) chi2 = self(*arg) if self.ndof > 0: txt += u"chi2/ndof = %5.4g(%5.4g/%d)" % (chi2 / self.ndof, chi2, self.ndof) else: txt += u"chi2/ndof = (%5.4g/%d)" % (chi2, self.ndof) if print_par and not no_plot: ax.text(parmloc[0], parmloc[1], txt, ha="left", va="top", transform=ax.transAxes) return (data_ret, error_ret, total_ret, part_ret)
def draw_ulh( self, minuit=None, bins=100, ax=None, bound=None, parmloc=(0.05, 0.95), nfbins=200, print_par=True, grid=True, args=None, errors=None, parts=False, show_errbars="normal", no_plot=False, ): data_ret = None error_ret = None total_ret = None part_ret = [] ax = plt.gca() if ax is None and not no_plot else ax arg, error = _get_args_and_errors(self, minuit, args, errors) n, e = np.histogram(self.data, bins=bins, range=bound, weights=self.weights) dataint = (n * np.diff(e)).sum() data_ret = (e, n) if not show_errbars: if not no_plot: pp = ax.hist(mid(e), bins=e, weights=n, histtype="step") error_ret = (np.sqrt(n), np.sqrt(n)) else: w2 = None if show_errbars == "normal": w2 = n error_ret = (np.sqrt(n), np.sqrt(n)) elif show_errbars == "sumw2": weights = None if self.weights != None: weights = self.weights ** 2 w2, e = np.histogram(self.data, bins=e, weights=weights) error_ret = (np.sqrt(w2), np.sqrt(w2)) else: raise ValueError("show_errbars must be 'normal' or 'sumw2'") if not no_plot: pp = ax.errorbar(mid(e), n, np.sqrt(w2), fmt="b.", capsize=0) # bound = (e[0], e[-1]) draw_arg = [("lw", 2)] if not parts: draw_arg.append(("color", "r")) # Draw pdf with finer bins ef = np.linspace(e[0], e[-1], nfbins + 1) scale = dataint if not self.extended else nfbins / float(bins) total_ret = draw_pdf_with_edges(self.f, arg, ef, ax=ax, density=not self.extended, scale=scale, **dict(draw_arg)) if parts: f_parts = getattr(self.f, "parts", None) if f_parts is not None: for p in f_parts(): ret = draw_pdf_with_edges(p, arg, ef, ax=ax, scale=scale, density=not self.extended, no_plot=no_plot) part_ret.append(ret) if not no_plot: ax.grid(grid) txt = _param_text(describe(self), arg, error) if not no_plot and print_par: ax.text(parmloc[0], parmloc[1], txt, ha="left", va="top", transform=ax.transAxes) return (data_ret, error_ret, total_ret, part_ret)
def draw_x2(self, minuit=None, ax=None, parmloc=(0.05, 0.95), print_par=True, args=None, errors=None, grid=True, parts=False, no_plot=False): data_ret = None error_ret = None total_ret = None part_ret = [] ax = plt.gca() if ax is None and not no_plot else ax arg, error = _get_args_and_errors(self, minuit, args, errors) x = self.x y = self.y data_err = self.error data_ret = x, y if data_err is None: if not no_plot: ax.plot(x, y, '+') err_ret = (np.ones(len(self.x)), np.ones(len(self.x))) else: if not no_plot: ax.errorbar(x, y, data_err, fmt='.') err_ret = (data_err, data_err) draw_arg = [('lw', 2)] draw_arg.append(('color', 'r')) total_ret = draw_pdf_with_midpoints(self.f, arg, x, ax=ax, no_plot=no_plot, **dict(draw_arg)) if not no_plot: ax.grid(grid) txt = _param_text(describe(self), arg, error) chi2 = self(*arg) if self.ndof > 0: txt += u'chi2/ndof = %5.4g(%5.4g/%d)' % (chi2 / self.ndof, chi2, self.ndof) else: txt += u'chi2/ndof = (%5.4g/%d)' % (chi2, self.ndof) if parts: f_parts = getattr(self.f, 'parts', None) if f_parts is not None: for p in f_parts(): tmp = draw_pdf_with_midpoints(p, arg, x, ax=ax, no_plot=no_plot, **dict(draw_arg)) part_ret.append(tmp) if print_par and not no_plot: ax.text(parmloc[0], parmloc[1], txt, ha='left', va='top', transform=ax.transAxes) return (data_ret, error_ret, total_ret, part_ret)
def draw_bx2(self, minuit=None, parmloc=(0.05, 0.95), nfbins=500, ax=None, print_par=True, args=None, errors=None, parts=False, grid=True, no_plot=False): data_ret = None error_ret = None total_ret = None part_ret = [] ax = plt.gca() if ax is None and not no_plot else ax arg, error = _get_args_and_errors(self, minuit, args, errors) m = mid(self.edges) if not no_plot: ax.errorbar(m, self.h, self.err, fmt='.') data_ret = (self.edges, self.h) error_ret = (self.err, self.err) bound = (self.edges[0], self.edges[-1]) scale = nfbins / float(self.bins) #scale back to bins draw_arg = [('lw', 2)] if not parts: draw_arg.append(('color', 'r')) total_ret = draw_pdf(self.f, arg, bins=nfbins, bound=bound, ax=ax, density=False, scale=scale, no_plot=no_plot, **dict(draw_arg)) if parts: f_parts = getattr(self.f, 'parts', None) if f_parts is not None: for p in f_parts(): tmp = draw_pdf(p, arg, bound=bound, bins=nfbins, ax=ax, density=False, scale=scale, no_plot=no_plot) part_ret.append(tmp) if not no_plot: ax.grid(grid) txt = _param_text(describe(self), arg, error) chi2 = self(*arg) if self.ndof > 0: txt += u'chi2/ndof = %5.4g(%5.4g/%d)' % (chi2 / self.ndof, chi2, self.ndof) else: txt += u'chi2/ndof = (%5.4g/%d)' % (chi2, self.ndof) if print_par and not no_plot: ax.text(parmloc[0], parmloc[1], txt, ha='left', va='top', transform=ax.transAxes) return (data_ret, error_ret, total_ret, part_ret)
def handle_console_cmd(cons, cmd, envs): threading.current_thread().name = 'MainThread' dev = envs['dev'] ob = envs['ob'] st = envs['state'] tlib = envs['tlib'] env = envs['env'] if 'app' in envs: app = envs['app'] else: app = 'cui' def save_stat(): tlib.save_stat(os.path.join("../stat/", "%s.txt" % app)) if cmd == 'q': save_stat() dev.finish() sys.exit(0) elif cmd == 'sense': scr = ob.grab_state(dev, no_verify=True, no_print=True) if scr.get('guess_descs') is None: util.print_tree(scr.get('tree')) else: util.print_tree(scr.get('tree'), scr.get('guess_descs'), scr.get('guess_score')) st.merge(scr) return elif cmd == 'tags': config.show_guess_tags = True scr = ob.grab_state(dev, no_verify=True, no_print=True) util.print_tree(scr.get('tree'), scr.get('guess_descs'), scr.get('guess_score')) st.merge(scr) config.show_guess_tags = False return elif cmd == 'tree': scr = ob.grab_state(dev, no_img=True) util.print_tree(scr.get('tree')) st.merge(scr) return elif cmd.startswith('app '): app = cmd.split(' ')[1] load_app(app, ob, tlib, envs) return elif cmd == 'load': tlib = testlib.collect_pieces("../tlib/") envs['tlib'] = tlib load_app(app, ob, tlib, envs) return elif cmd == 'reload': value.init_params("../etc/", app) try: if app: os.remove("../model/screen_%s" % app) os.remove("../model/element_%s" % app) else: os.remove("../model/screen") os.remove("../model/element") except: pass ob.load("../model/", "../guis/", "../guis-extra/", config.extra_screens, config.extra_element_scrs) return elif cmd == 'tlib': tlib = testlib.collect_pieces("../tlib/") envs['tlib'] = tlib return elif cmd == 'test': tlib = testlib.collect_pieces("../tlib/") tlib.set_app(app) tlib.add_test(microtest.init_test(appdb.get_app(app))) envs['tlib'] = tlib config.show_guess_tags = True scr = ob.grab_state(dev) config.show_guess_tags = False st.merge(scr) begin_state = st.to_essential(tlib.essential_props()) tests = tlib.usable_tests(st) tests.sort(key=lambda test: test.prio) for i in range(len(tests)): testinfo = tlib.get_testinfo(tests[i]) print("%2d. %s: %s [%d %d]" % (i, tests[i].feature_name, tests[i].name, testinfo.succs, testinfo.fails)) if len(tests) == 0: print("no test available!") return if len(tests) == 1: tid = 0 else: tid = input("which one? ") try: tid = int(tid) except: return test = tests[tid] ret = test.attempt(dev, ob, st, tlib, environ.empty) util.print_tree(st.get('tree'), st.get('guess_descs')) end_state = st.to_essential(tlib.essential_props()) if ret: print("test SUCC") tlib.clear_stat(test) tlib.mark_succ(test, begin_state, end_state) else: print("test FAIL") tlib.mark_fail(test, begin_state) save_stat() return elif cmd == 'save': save_stat() return elif cmd == 'stat': tlib.print_stat() return elif cmd == 'items': scr = ob.grab_state(dev, no_img=True) util.printitems(scr.get('items')) return elif cmd.startswith('item'): itemid = int(cmd.split(' ')[1]) scr = ob.grab_state(dev, no_img=True) items = scr.get('items') if itemid in items: print(util.describe(items[itemid])) else: print("no such item: %d" % itemid) return elif cmd == 'debug': logging.basicConfig(level=logging.DEBUG) return elif cmd == 'idle': dev.wait_idle() return elif cmd == 'dialog': scr = ob.grab_state(dev) ret = tlib.handle_dialog(scr, dev, ob) if ret: print("dialog handled") else: print("dialog not handled") return elif cmd.startswith('find '): tag = cmd.split(' ', 1)[1] con = concept.parse(tag) if con is None: print("invalid locator") return scr = ob.grab_state(dev) widgets = con.locate(scr, ob, environ.Environment()) if widgets == []: print("can't find") else: for widget in widgets: print("FOUND: %s" % widget) print(" content:", widget.content()) return elif cmd == 'perf': perfmon.print_stat() return elif cmd == 'clear': init = tlib.find_test('meta', 'start app') st.reset() init.attempt(dev, ob, st, tlib, None) ob.update_state(dev, st) tlib.mark_succ(init, state.init_state, st) return elif cmd.startswith('set'): parts = cmd.split(' ', 2) if len(parts) == 2: key = parts[1] val = "1" else: (key, val) = parts[1:] st.set(key, val) return elif cmd.startswith('del'): parts = cmd.split(' ', 1) key = parts[1] st.remove(key) return elif cmd == 'dump': filename = util.choose_filename("../cap/", "page%d") logger.info("dumping to page %s", filename) dev.dump(filename) #sense.dump_page(dev, "../cap/") return elif cmd == 'list': scr = ob.grab_state(dev, no_img=True) tree = scr.get('tree') treeinfo = analyze.collect_treeinfo(tree) for root in sorted(treeinfo['listlike']): print("ROOT:", util.describe_node(tree[root])) for item in sorted(treeinfo['itemlike']): if listinfo.get_lca(tree, [root, item]) == root: print(" NODE:", util.describe_node(tree[item])) for field in sorted(treeinfo['dupid']): if listinfo.get_lca(tree, [item, field]) == item: print(" FIELD:", util.describe_node(tree[field])) return elif cmd == 'webon': config.GRAB_WEBVIEW = True return elif cmd == 'weboff': config.GRAB_WEBVIEW = False return elif cmd.startswith('ob '): prop = cmd.split(' ', 1)[1] wd = watchdog.Watchdog(100000) smgr = statemgr.StateMgr(tlib, None, dev, ob, wd) ret = smgr.observe_prop(prop, st) if ret: print("prop %s = %s" % (prop, st.get(prop, ''))) else: print("observe error") return elif cmd.startswith('clean '): parts = cmd.split(' ', 2) if len(parts) == 2: prop = parts[1] val = "1" else: (prop, val) = parts[1:] wd = watchdog.Watchdog(100000) smgr = statemgr.StateMgr(tlib, None, dev, ob, wd) ret = smgr.cleanup_prop(prop, val, st) if ret: print("prop %s cleaned" % prop) else: print("clean error") return elif cmd.startswith('oc '): prop = cmd.split(' ', 1)[1] wd = watchdog.Watchdog(100000) smgr = statemgr.StateMgr(tlib, None, dev, ob, wd) ret = smgr.observe_and_clean(prop, st) if ret: print("prop %s is clean now" % prop) else: print("observe/clean error") return elif cmd == 'dbg': print(tlib) return elif cmd.startswith('skip '): name = cmd.split(' ', 1)[1] add_skip(app, name) print("skipping %s" % name) return elif cmd == 'skip': skips = load_skip(app) for skip in skips: print("now skipping %s" % skip) return elif cmd.startswith('noskip '): name = cmd.split(' ', 1)[1] del_skip(app, name) print("not skipping %s" % name) return elif cmd == 'src': if st.get('xml') is not None: print(st.get('xml')) if st.get('src') is not None: print(st.get('src')) return elif cmd == 'install': apputils.install(dev, app) return elif cmd == 'uninstall': apputils.uninstall(dev, app) return op = operation.parse_line(cmd) if op is None: print("unknown op") return ret = op.do(dev, ob, st, env, tlib) if not ret: print("op failed")
def guess(filename, lasttag=None): basename = filename.replace('.xml', '') (items, descs) = analyze.load_case(filename) # if descs: # return appname = basename.split('/')[-1].split('.')[0].split('_')[0] scrname = basename.split('/')[-1].split('.')[0].split('_')[-1] imgfile = filename.replace('.xml', '.png') if not appname in tag_db: tag_db[appname] = {} if appname in tag_db: for tag in tag_db[appname]: xid = tag_db[appname][tag] for itemid in items: if items[itemid]['id'] == xid: descs[itemid] = tag print("%s Current:" % basename) util.printdesc(items, descs) # analyze.printitems(items) tree = analyze.analyze([filename])[0] util.print_tree(tree, descs) viewproc = subprocess.Popen([config.picviewer_path, basename + ".png"]) prompt = '' while True: print(tags.tag['universal']) if scrname in tags.tag: print(tags.tag[scrname]) line = input(prompt + "> ") parts = line.split(' ') cmd = parts[0] if cmd == '': util.save_desc(basename, descs) break elif cmd[0] == 'q': viewproc.kill() sys.exit(0) elif cmd[0] == 'l': listid = int(parts[1]) listtype = parts[2] guess_list(listid, listtype, items, descs) util.printdesc(items, descs) elif cmd[0] == 'a': auto_guess(filename, items, descs) util.print_tree(tree, descs) elif cmd[0] == 'y': util.save_desc(basename, descs) break elif cmd[0] >= '0' and cmd[0] <= '9': if len(parts) > 1: lasttag = parts[1] util.mark_item(parts, items, descs, scrname, tag_db[appname]) util.print_tree(tree, descs) else: if lasttag is None: print(util.describe(items[int(cmd)])) else: util.mark_item([cmd, lasttag], items, descs, scrname, tag_db[appname]) util.print_tree(tree, descs) elif cmd[0] == 'd': if cmd == 'dr': recursive = True else: recursive = False util.remove_mark(parts[1:], items, descs, recursive) util.print_tree(tree, descs) elif cmd[0] == ';': util.print_tree(tree, descs) elif cmd[0] == '!': util.printitems(items) elif cmd[0] == 'c': element_clas = elements.getmodel("../model/", "../guis/", None) imgdata = skimage.io.imread(imgfile, as_grey=True) imgdata = skimage.transform.resize(imgdata, (config.height, config.width)) treeinfo = analyze.collect_treeinfo(tree) for itemid in tree: (guess_element, score) = element_clas.classify(scrname, tree, itemid, imgdata, treeinfo) if guess_element != 'NONE': descs[itemid] = guess_element util.print_tree(tree, descs) elif cmd[0] == '?': print(util.describe(items[int(parts[1])])) else: print("Unknown command") viewproc.kill()