def remove_hidden(items, descs):
mark_order(items, sorted(items)[0], 0)
removed = []
for item in sorted(items):
me = items[item]
if 'x' not in me:
continue
if me['parent'] in removed:
removed.append(item)
print('removing child %s' % util.describe(items[item]))
if item in descs:
print('WARNING: removing labeled item!')
continue
for other in sorted(items):
him = items[other]
if him['order'] > me['order']:
if (him['x'] <= me['x']
and him['x'] + him['width'] >= me['x'] + me['width']
and him['y'] <= me['y']
and him['y'] + him['height'] >= me['y'] + me['height']
and not (him['x'] == me['x'] and him['y'] == me['y']
and him['width'] == me['width']
and him['height'] == me['height'])):
removed.append(item)
logger.info('removing %s because %s', util.describe(me),
util.describe(him))
if item in descs:
print('WARNING: removing labeled item!')
break
for item in sorted(removed):
parent = items[item]['parent']
del items[item]
if parent in items:
items[parent]['children'].remove(item)
def draw_ulh(self, minuit=None, bins=100, ax=None, bound=None,
parmloc=(0.05, 0.95), nfbins=200, print_par=True, grid=True,
args=None, errors=None, parts=False, show_errbars='normal'):
data_ret = None
error_ret = None
total_ret = None
part_ret = []
ax = plt.gca() if ax is None else ax
arg, error = _get_args_and_errors(self, minuit, args, errors)
n,e= np.histogram(self.data, bins=bins, range=bound, weights=self.weights)
dataint= (n*np.diff(e)).sum()
data_ret = (e, n)
if not show_errbars:
pp= ax.hist(mid(e), bins=e, weights=n, histtype='step')
error_ret = (np.sqrt(n), np.sqrt(n))
else:
w2= None
if show_errbars=='normal':
w2=n
error_ret = (np.sqrt(n), np.sqrt(n))
elif show_errbars=='sumw2':
weights= None
if self.weights!= None:
weights= self.weights**2
w2,e= np.histogram(self.data, bins=e, weights=weights)
error_ret = (np.sqrt(w2), np.sqrt(w2))
else:
raise ValueError('show_errbars must be \'normal\' or \'sumw2\'')
pp= ax.errorbar(mid(e), n, np.sqrt(w2) , fmt='b+', capsize=0)
#bound = (e[0], e[-1])
draw_arg = [('lw', 2)]
if not parts:
draw_arg.append(('color', 'r'))
# Draw pdf with finer bins
ef= np.linspace(e[0],e[-1], nfbins+1)
scale= dataint if not self.extended else nfbins/float(bins)
total_ret = draw_pdf_with_edges(self.f, arg, ef, ax=ax, density=not self.extended, scale=scale,
**dict(draw_arg))
if parts:
f_parts = getattr(self.f, 'parts', None)
if f_parts is not None:
for p in f_parts():
ret = draw_pdf_with_edges(p, arg, ef, ax=ax, scale=scale, density=not self.extended)
part_ret.append(ret)
ax.grid(grid)
txt = _param_text(describe(self), arg, error)
if print_par:
ax.text(parmloc[0], parmloc[1], txt, ha='left', va='top',
transform=ax.transAxes)
return (data_ret, error_ret, total_ret, part_ret)
def draw_residual_ulh(
self,
minuit=None,
bins=100,
ax=None,
bound=None,
parmloc=(0.05, 0.95),
print_par=False,
grid=True,
args=None,
errors=None,
show_errbars=True,
errbar_algo="normal",
norm=False,
):
ax = plt.gca() if ax is None else ax
arg, error = _get_args_and_errors(self, minuit, args, errors)
n, e = np.histogram(self.data, bins=bins, range=bound, weights=self.weights)
dataint = (n * np.diff(e)).sum()
scale = dataint if not self.extended else 1.0
w2 = None
if errbar_algo == "normal":
w2 = n
elif errbar_algo == "sumw2":
weights = None
if self.weights != None:
weights = self.weights ** 2
w2, e = np.histogram(self.data, bins=e, weights=weights)
else:
raise ValueError("errbar_algo must be 'normal' or 'sumw2'")
yerr = np.sqrt(w2)
arg = parse_arg(self.f, arg, 1) if isinstance(arg, dict) else arg
yf = vector_apply(self.f, mid(e), *arg)
yf *= scale * np.diff(e) if self.extended else scale
n = n - yf
if norm:
sel = yerr > 0
n[sel] /= yerr[sel]
yerr = np.ones(len(yerr))
if show_errbars:
pp = ax.errorbar(mid(e), n, yerr, fmt="b.", capsize=0)
else: # No errorbars
pp = ax.bar(e[:-1], n, width=np.diff(e))
# bound = (e[0], e[-1])
# draw_arg = [('lw', 2), ('color', 'r')]
ax.plot([e[0], e[-1]], [0.0, 0.0], "r-")
ax.grid(grid)
txt = _param_text(describe(self), arg, error)
if print_par:
ax.text(parmloc[0], parmloc[1], txt, ha="left", va="top", transform=ax.transAxes)
def draw_blh(self, minuit=None, parmloc=(0.05, 0.95),
nfbins=1000, ax=None, print_par=True, grid=True,
args=None, errors=None, parts=False):
data_ret = None
error_ret = None
total_ret = None
part_ret = []
ax = plt.gca() if ax is None else ax
arg, error = _get_args_and_errors(self, minuit, args, errors)
m = mid(self.edges)
if self.use_w2:
err = np.sqrt(self.w2)
else:
err = np.sqrt(self.h)
n= np.copy(self.h)
dataint= (n*np.diff(self.edges)).sum()
scale= dataint if not self.extended else 1.0
ax.errorbar(m, n, err, fmt='+', capsize=0)
data_ret = (self.edges, n)
error_ret = (err, err)
draw_arg = [('lw', 2)]
if not parts:
draw_arg.append(('color', 'r'))
bound = (self.edges[0], self.edges[-1])
#scale back to bins
if self.extended:
scale= nfbins/float(self.bins)
total_ret = draw_pdf(self.f, arg, bins=nfbins, bound=bound, ax=ax, density=not self.extended,
scale=scale, **dict(draw_arg))
if parts:
f_parts = getattr(self.f, 'parts', None)
if f_parts is not None:
for p in f_parts():
tmp = draw_pdf(p, arg, bins=nfbins, bound=bound, ax=ax,
density=not self.extended, scale=scale)
part_ret.append(tmp)
ax.grid(grid)
txt = _param_text(describe(self), arg, error)
if print_par:
ax.text(parmloc[0], parmloc[1], txt, ha='left', va='top',
transform=ax.transAxes)
return (data_ret, error_ret, total_ret, part_ret)
def draw_x2(self, minuit=None, ax=None, parmloc=(0.05, 0.95), print_par=True,
args=None, errors=None, grid=True, parts=False, nbins=None):
data_ret = None
error_ret = None
total_ret = None
part_ret = []
ax = plt.gca() if ax is None else ax
arg, error = _get_args_and_errors(self, minuit, args, errors)
x=self.x
y=self.y
data_err = self.error
# Draw data points
data_ret = x,y
if data_err is None:
ax.plot(x, y, '+')
err_ret = (np.ones(len(self.x)), np.ones(len(self.x)))
else:
ax.errorbar(x, y, data_err, fmt='+', capsize=0)
err_ret = (data_err, data_err)
draw_arg = [('lw', 2)]
draw_arg.append(('color', 'r'))
# Draw PDF curve(s)
if nbins is not None:
x = np.linspace(x[0],x[-1], nbins)
total_ret = draw_pdf_with_midpoints(self.f, arg, x, ax=ax, **dict(draw_arg))
if parts:
f_parts = getattr(self.f, 'parts', None)
if f_parts is not None:
for p in f_parts():
tmp = draw_pdf_with_midpoints(p, arg, x, ax=ax, **dict(draw_arg))
part_ret.append(tmp)
# Print text
txt = _param_text(describe(self), arg, error)
chi2 = self(*arg)
if self.ndof > 0:
txt+=u'chi2/ndof = %5.4g(%5.4g/%d)'%(chi2/self.ndof, chi2, self.ndof)
else:
txt+=u'chi2/ndof = (%5.4g/%d)'%(chi2, self.ndof)
if print_par:
ax.text(parmloc[0], parmloc[1], txt, ha='left', va='top',
transform=ax.transAxes)
ax.grid(grid)
return (data_ret, error_ret, total_ret , part_ret)
def draw_bx2(self, minuit=None, parmloc=(0.05, 0.95), nfbins=500, ax=None,
print_par=True, args=None, errors=None, parts=False, grid=True):
data_ret = None
error_ret = None
total_ret = None
part_ret = []
ax = plt.gca() if ax is None else ax
arg, error = _get_args_and_errors(self, minuit, args, errors)
m = mid(self.edges)
ax.errorbar(m, self.h, self.err, fmt='+', capsize=0)
data_ret = (self.edges, self.h)
error_ret = (self.err, self.err)
bound = (self.edges[0], self.edges[-1])
scale = nfbins/float(self.bins) #scale back to bins
draw_arg = [('lw', 2)]
if not parts:
draw_arg.append(('color', 'r'))
total_ret = draw_pdf(self.f, arg, bins=nfbins, bound=bound, ax=ax, density=False,
scale=scale, **dict(draw_arg))
if parts:
f_parts = getattr(self.f, 'parts', None)
if f_parts is not None:
for p in f_parts():
tmp = draw_pdf(p, arg, bound=bound, bins=nfbins, ax=ax, density=False,
scale=scale)
part_ret.append(tmp)
ax.grid(grid)
txt = _param_text(describe(self), arg, error)
chi2 = self(*arg)
if self.ndof > 0:
txt+=u'chi2/ndof = %5.4g(%5.4g/%d)'%(chi2/self.ndof, chi2, self.ndof)
else:
txt+=u'chi2/ndof = (%5.4g/%d)'%(chi2, self.ndof)
if print_par:
ax.text(parmloc[0], parmloc[1], txt, ha='left', va='top',
transform=ax.transAxes)
return (data_ret, error_ret, total_ret, part_ret)
def gen_toy(f, nsample, bound, accuracy=10000, quiet=True, **kwd):
"""
generate ntoy
:param f:
:param nsample:
:param ntoy:
:param bound:
:param accuracy:
:param quiet:
:param kwd: the rest of keyword argument will be passed to f
:return: numpy.ndarray
"""
#based on inverting cdf this is fast but you will need to give it a reasonable range
#unlike roofit which is based on accept reject
vnames = describe(f)
if not quiet:
print vnames
my_arg = [kwd[v] for v in vnames[1:]]
#random number
#if accuracy is None: accuracy=10*numtoys
r = npr.random_sample(nsample)
x = np.linspace(bound[0], bound[1], accuracy)
pdf = _vector_apply(f, x, tuple(my_arg))
cdf = compute_cdf(pdf, x)
if cdf[-1] < 0.01:
warn(SmallIntegralWarning('Integral for given funcition is'
' really low. Did you give it a reasonable range?'))
cdfnorm = cdf[-1]
cdf /= cdfnorm
#now convert that to toy
ret = invert_cdf(r, cdf, x)
if not quiet:
#move this to plotting
plt.figure()
plt.title('comparison')
numbin = 100
h, e = np.histogram(ret, bins=numbin)
mp = (e[1:]+e[:-1])/2.
err = np.sqrt(h)
plt.errorbar(mp, h, err, fmt='.b')
bw = e[1] - e[0]
y = pdf * len(ret) / cdfnorm * bw
ylow = y + np.sqrt(y)
yhigh = y - np.sqrt(y)
plt.plot(x, y, label='pdf', color='r')
plt.fill_between(x, yhigh, ylow, color='g', alpha=0.2)
plt.grid(True)
plt.xlim(bound)
plt.ylim(ymin=0)
return ret
def extract(datalist, like, bbox, dst_crs, out_path):
"""Extract/reproject data within lat/lon bbox
"""
# check that enough options are provided
if not like and not bbox:
util.error("Provide bounds as either a 'like' dataset or a bbox")
# read input csv listing layers
layers = [s for s in csv.DictReader(open(datalist, 'rb'))]
# parse bbox
if bbox:
bbox = tuple(map(float, bbox.split(',')))
# derive bbox/crs from aoi/like layer (ignoring any provided bbox)
if like:
bbox = util.get_bbox(like)
dst_crs = util.get_crs(like)
# parse provided epsg code
elif dst_crs:
dst_crs = from_epsg(dst_crs.split(':')[1])
# name is derived from config file name
if not out_path:
b = os.path.basename(datalist)
out_path = os.path.join(os.getcwd(), os.path.splitext(b)[0])
util.make_sure_path_exists(out_path)
# process each layer
for layer in layers:
click.echo('Extracting %s' % layer['name'])
if util.describe(layer['path'])['type'] == 'VECTOR':
util.bbox_copy(layer['path'],
os.path.join(out_path, layer['name'] + ".shp"),
bbox,
in_layer=layer['layer'],
dst_crs=dst_crs)
elif util.describe(layer['path'])['type'] == 'RASTER':
util.bbox_copyraster(layer['path'],
os.path.join(out_path,
layer['name'] + '.tif'),
bbox,
dst_crs=dst_crs)
def draw_residual_blh(self,
minuit=None,
parmloc=(0.05, 0.95),
ax=None,
print_par=False,
args=None,
errors=None,
norm=False,
grid=True):
ax = plt.gca() if ax is None else ax
arg, error = _get_args_and_errors(self, minuit, args, errors)
m = mid(self.edges)
if self.use_w2:
err = np.sqrt(self.w2)
else:
err = np.sqrt(self.h)
n = np.copy(self.h)
dataint = (n * np.diff(self.edges)).sum()
scale = dataint if not self.extended else 1.0
arg = parse_arg(self.f, arg, 1) if isinstance(arg, dict) else arg
yf = vector_apply(self.f, m, *arg)
yf *= (scale * np.diff(self.edges) if self.extended else scale)
n = n - yf
if norm:
sel = err > 0
n[sel] /= err[sel]
err = np.ones(len(err))
ax.errorbar(m, n, err, fmt='.')
ax.plot([self.edges[0], self.edges[-1]], [0., 0.], 'r-')
ax.grid(grid)
txt = _param_text(describe(self), arg, error)
if print_par:
ax.text(parmloc[0],
parmloc[1],
txt,
ha='left',
va='top',
transform=ax.transAxes)
def postprocess(self, tree, items, history):
for nodeid in sorted(tree):
if nodeid in tree:
node = tree[nodeid]
children = []
for childid in tree:
if tree[childid]['parent'] == nodeid:
children.append(childid)
if len(children) > 0:
child0 = tree[children[0]]
if len(children) > 1:
child1 = tree[children[1]]
if config.REMOVE_OVERLAP_OLD:
if (len(children) > 1 and node['class'] == 'FrameLayout'
and child0['width'] == child1['width'] == config.width
and child0['height'] == child1['height'] ==
config.real_height_nostatus):
st1 = subtree_collect(tree, children[-1])
if len(st1) > 1:
for (olditems, oldtree) in reversed(history):
oldme = find_in_old(tree, items, nodeid, oldtree,
olditems, True)
if oldme is not None:
logger.debug("old me: %s",
util.describe(olditems[oldme]))
appear_in_old = []
for childid in children:
oldchild = find_in_old(
tree, items, childid, oldtree,
olditems, False)
if oldchild is not None:
appear_in_old.append(childid)
else:
break
if (appear_in_old != [] and
len(appear_in_old) < len(children)):
for childid in appear_in_old:
self.del_subtree(tree, childid)
self.incstat("overlap old")
logger.debug("overlap old %s children %s",
node['id'], appear_in_old)
return True
return False
def fix_size(items, filename):
imgfile = os.path.splitext(filename)[0] + '.png'
if not os.path.exists(imgfile):
return
img = Image.open(imgfile)
for itemid in sorted(items):
if items[itemid]['width'] + items[itemid]['x'] > 0:
orig_width = items[itemid]['width'] + items[itemid]['x']
orig_height = items[itemid]['height'] + items[itemid]['y']
minid = itemid
break
#orig_width = items[1]['width'] + items[1]['x']
#orig_height = items[1]['height'] + items[1]['y']
if orig_width == config.width or orig_height == config.real_height:
return
if items[minid]['x'] != 0:
# not at top-left, usually dialog
if orig_width <= config.width and orig_height <= config.real_height:
# seems fine
return
real_height = img.height * orig_width / img.width
for itemid in items:
item = items[itemid]
if (item['x'] + item['width'] < 0 or item['x'] > orig_width
or item['y'] + item['height'] < 0 or item['y'] > real_height):
logger.debug("removing OOS %s", util.describe(item))
item['x'] = item['y'] = item['width'] = item['height'] = 0
ratio = 1.0 * orig_width / img.width
img_ratio = find_closest(ratio, SCALE_RATIOS)
fix_ratio = img_ratio * img.width / config.width
logger.debug("fixing %s: %d != %d %d, ratio %.3f -> %.3f -> %.3f" %
(filename, orig_width, config.width, img.width, ratio,
img_ratio, fix_ratio))
for itemid in items:
items[itemid]['x'] = int(items[itemid]['x'] / fix_ratio)
items[itemid]['y'] = int(items[itemid]['y'] / fix_ratio)
items[itemid]['width'] = int(items[itemid]['width'] / fix_ratio)
items[itemid]['height'] = int(items[itemid]['height'] / fix_ratio)
def draw_residual_blh(self, minuit=None, parmloc=(0.05, 0.95),
ax=None, print_par=False, args=None, errors=None,
norm=False, grid=True):
ax = plt.gca() if ax is None else ax
arg, error = _get_args_and_errors(self, minuit, args, errors)
m = mid(self.edges)
if self.use_w2:
err = np.sqrt(self.w2)
else:
err = np.sqrt(self.h)
n= np.copy(self.h)
dataint= (n*np.diff(self.edges)).sum()
scale= dataint if not self.extended else 1.0
arg = parse_arg(self.f, arg, 1) if isinstance(arg, dict) else arg
yf = vector_apply(self.f, m, *arg)
yf*= (scale*np.diff(self.edges) if self.extended else scale)
n = n- yf
if norm:
sel= err>0
n[sel]/= err[sel]
err= np.ones(len(err))
ax.errorbar(m, n, err, fmt='+', capsize=0)
ax.plot([self.edges[0],self.edges[-1]],[0.,0.], 'r-')
ax.grid(grid)
txt = _param_text(describe(self), arg, error)
if print_par:
ax.text(parmloc[0], parmloc[1], txt, ha='left', va='top',
transform=ax.transAxes)
return m, n, err
def draw_residual_ulh(self,
minuit=None,
bins=100,
ax=None,
bound=None,
parmloc=(0.05, 0.95),
print_par=False,
grid=True,
args=None,
errors=None,
show_errbars=True,
errbar_algo='normal',
norm=False):
ax = plt.gca() if ax is None else ax
arg, error = _get_args_and_errors(self, minuit, args, errors)
n, e = np.histogram(self.data,
bins=bins,
range=bound,
weights=self.weights)
dataint = (n * np.diff(e)).sum()
scale = dataint if not self.extended else 1.0
w2 = None
if errbar_algo == 'normal':
w2 = n
elif errbar_algo == 'sumw2':
weights = None
if self.weights != None:
weights = self.weights**2
w2, e = np.histogram(self.data, bins=e, weights=weights)
else:
raise ValueError('errbar_algo must be \'normal\' or \'sumw2\'')
yerr = np.sqrt(w2)
arg = parse_arg(self.f, arg, 1) if isinstance(arg, dict) else arg
yf = vector_apply(self.f, mid(e), *arg)
yf *= (scale * np.diff(e) if self.extended else scale)
n = n - yf
if norm:
sel = yerr > 0
n[sel] /= yerr[sel]
yerr = np.ones(len(yerr))
if show_errbars:
pp = ax.errorbar(mid(e), n, yerr, fmt='b.', capsize=0)
else: # No errorbars
pp = ax.bar(e[:-1], n, width=np.diff(e))
#bound = (e[0], e[-1])
#draw_arg = [('lw', 2), ('color', 'r')]
ax.plot([e[0], e[-1]], [0., 0.], 'r-')
ax.grid(grid)
txt = _param_text(describe(self), arg, error)
if print_par:
ax.text(parmloc[0],
parmloc[1],
txt,
ha='left',
va='top',
transform=ax.transAxes)
def draw_ulh(self,
minuit=None,
bins=100,
ax=None,
bound=None,
parmloc=(0.05, 0.95),
nfbins=200,
print_par=True,
grid=True,
args=None,
errors=None,
parts=False,
show_errbars='normal',
no_plot=False):
data_ret = None
error_ret = None
total_ret = None
part_ret = []
ax = plt.gca() if ax is None and not no_plot else ax
arg, error = _get_args_and_errors(self, minuit, args, errors)
n, e = np.histogram(self.data,
bins=bins,
range=bound,
weights=self.weights)
dataint = (n * np.diff(e)).sum()
data_ret = (e, n)
if not show_errbars:
if not no_plot:
pp = ax.hist(mid(e), bins=e, weights=n, histtype='step')
error_ret = (np.sqrt(n), np.sqrt(n))
else:
w2 = None
if show_errbars == 'normal':
w2 = n
error_ret = (np.sqrt(n), np.sqrt(n))
elif show_errbars == 'sumw2':
weights = None
if self.weights != None:
weights = self.weights**2
w2, e = np.histogram(self.data, bins=e, weights=weights)
error_ret = (np.sqrt(w2), np.sqrt(w2))
else:
raise ValueError('show_errbars must be \'normal\' or \'sumw2\'')
if not no_plot:
pp = ax.errorbar(mid(e), n, np.sqrt(w2), fmt='b.', capsize=0)
#bound = (e[0], e[-1])
draw_arg = [('lw', 2)]
if not parts:
draw_arg.append(('color', 'r'))
# Draw pdf with finer bins
ef = np.linspace(e[0], e[-1], nfbins + 1)
scale = dataint if not self.extended else nfbins / float(bins)
total_ret = draw_pdf_with_edges(self.f,
arg,
ef,
ax=ax,
density=not self.extended,
scale=scale,
**dict(draw_arg))
if parts:
f_parts = getattr(self.f, 'parts', None)
if f_parts is not None:
for p in f_parts():
ret = draw_pdf_with_edges(p,
arg,
ef,
ax=ax,
scale=scale,
density=not self.extended,
no_plot=no_plot)
part_ret.append(ret)
if not no_plot:
ax.grid(grid)
txt = _param_text(describe(self), arg, error)
if not no_plot and print_par:
ax.text(parmloc[0],
parmloc[1],
txt,
ha='left',
va='top',
transform=ax.transAxes)
return (data_ret, error_ret, total_ret, part_ret)
def draw_blh(self,
minuit=None,
parmloc=(0.05, 0.95),
nfbins=1000,
ax=None,
print_par=True,
grid=True,
args=None,
errors=None,
parts=False,
no_plot=False):
data_ret = None
error_ret = None
total_ret = None
part_ret = []
ax = plt.gca() if ax is None and not no_plot else ax
arg, error = _get_args_and_errors(self, minuit, args, errors)
m = mid(self.edges)
if self.use_w2:
err = np.sqrt(self.w2)
else:
err = np.sqrt(self.h)
n = np.copy(self.h)
dataint = (n * np.diff(self.edges)).sum()
scale = dataint if not self.extended else 1.0
if not no_plot:
ax.errorbar(m, n, err, fmt='.')
data_ret = (self.edges, n)
error_ret = (err, err)
draw_arg = [('lw', 2)]
if not parts:
draw_arg.append(('color', 'r'))
bound = (self.edges[0], self.edges[-1])
#scale back to bins
if self.extended:
scale = nfbins / float(self.bins)
total_ret = draw_pdf(self.f,
arg,
bins=nfbins,
bound=bound,
ax=ax,
density=not self.extended,
scale=scale,
no_plot=no_plot,
**dict(draw_arg))
if parts:
f_parts = getattr(self.f, 'parts', None)
if f_parts is not None:
for p in f_parts():
tmp = draw_pdf(p,
arg,
bins=nfbins,
bound=bound,
ax=ax,
density=not self.extended,
scale=scale,
no_plot=no_plot)
part_ret.append(tmp)
if not no_plot:
ax.grid(grid)
txt = _param_text(describe(self), arg, error)
if print_par and not no_plot:
ax.text(parmloc[0],
parmloc[1],
txt,
ha='left',
va='top',
transform=ax.transAxes)
return (data_ret, error_ret, total_ret, part_ret)
def draw_x2(
self,
minuit=None,
ax=None,
parmloc=(0.05, 0.95),
print_par=True,
args=None,
errors=None,
grid=True,
parts=False,
no_plot=False,
):
data_ret = None
error_ret = None
total_ret = None
part_ret = []
ax = plt.gca() if ax is None and not no_plot else ax
arg, error = _get_args_and_errors(self, minuit, args, errors)
x = self.x
y = self.y
data_err = self.error
data_ret = x, y
if data_err is None:
if not no_plot:
ax.plot(x, y, "+")
err_ret = (np.ones(len(self.x)), np.ones(len(self.x)))
else:
if not no_plot:
ax.errorbar(x, y, data_err, fmt=".")
err_ret = (data_err, data_err)
draw_arg = [("lw", 2)]
draw_arg.append(("color", "r"))
total_ret = draw_pdf_with_midpoints(self.f, arg, x, ax=ax, no_plot=no_plot, **dict(draw_arg))
if not no_plot:
ax.grid(grid)
txt = _param_text(describe(self), arg, error)
chi2 = self(*arg)
if self.ndof > 0:
txt += u"chi2/ndof = %5.4g(%5.4g/%d)" % (chi2 / self.ndof, chi2, self.ndof)
else:
txt += u"chi2/ndof = (%5.4g/%d)" % (chi2, self.ndof)
if parts:
f_parts = getattr(self.f, "parts", None)
if f_parts is not None:
for p in f_parts():
tmp = draw_pdf_with_midpoints(p, arg, x, ax=ax, no_plot=no_plot, **dict(draw_arg))
part_ret.append(tmp)
if print_par and not no_plot:
ax.text(parmloc[0], parmloc[1], txt, ha="left", va="top", transform=ax.transAxes)
return (data_ret, error_ret, total_ret, part_ret)
def analyze_items(self, items, descs, regs, print_rets, print_error,
history):
self.fix_size_to_scr(items)
self.mark_important(items)
parents = [0]
newtree = {}
for itemid in preorder(items, 0):
item = items[itemid]
if item['important']:
nodeinfo = {}
par = -1
while len(parents) > 0 and (
item['depth'] <= items[parents[-1]]['depth']
or items[item['parent']]['depth'] <
items[parents[-1]]['depth']):
parents.pop()
if len(parents) > 0:
par = parents[-1]
if print_rets:
print('%3d' % itemid, self.get_prefix(items, itemid),
util.describe(item), item['parent'], par)
# orig = item
nodeinfo['parent'] = par
nodeinfo['class'] = item['class']
nodeinfo['text'] = item['text']
nodeinfo['desc'] = item['desc']
nodeinfo['id'] = item['id']
nodeinfo['raw'] = [itemid]
nodeinfo['width'] = item['width']
nodeinfo['height'] = item['height']
nodeinfo['x'] = item['x']
nodeinfo['y'] = item['y']
nodeinfo['origw'] = nodeinfo['width']
nodeinfo['origh'] = nodeinfo['height']
nodeinfo['origx'] = nodeinfo['x']
nodeinfo['origy'] = nodeinfo['y']
nodeinfo['click'] = item['click']
nodeinfo['scroll'] = item['scroll']
nodeinfo['password'] = item['password']
nodeinfo['focused'] = item['focused']
nodeinfo['checkable'] = item['checkable']
nodeinfo['childid'] = 0 # placemarker
nodeinfo['webview'] = item['webview']
nodeinfo['origitem'] = item
if itemid in descs:
nodeinfo['tags'] = [descs[itemid]]
else:
nodeinfo['tags'] = []
if itemid in regs:
nodeinfo['regs'] = [regs[itemid]]
else:
nodeinfo['regs'] = []
newtree[itemid] = nodeinfo
parents.append(itemid)
self.incstat("before", len(newtree))
if print_rets:
util.print_tree(newtree)
while self.process(newtree, items, history):
if print_rets:
util.print_tree(newtree)
if print_error:
for itemid in descs:
found = False
for nodeid in newtree:
if itemid in newtree[nodeid]['raw']:
found = True
break
if not found:
logger.error("REMOVED: %d %s %s", itemid,
descs[itemid],
util.describe(items[itemid]))
pass
while self.postprocess(newtree, items, history):
pass
self.incstat("after", len(newtree))
collect_children(newtree)
return newtree
def analyze(files,
print_rets=False,
show_progress=False,
print_items=False,
print_error=False,
show_ocr=False,
show_stat=False,
use_ocr=False):
ret = []
if show_progress:
progress = progressbar.ProgressBar()
items = progress(files)
else:
items = files
analyzer = Analyzer()
for filename in items:
filebase = os.path.splitext(filename)[0]
logger.debug("analyzing %s" % filename)
(items, descs, regs) = load_case(filename)
if print_items:
util.printitems(items)
start_time = time.time()
newtree = analyzer.analyze_items(items, descs, regs, print_items,
print_error, [])
ret.append(newtree)
if print_rets:
util.print_tree(newtree)
logger.info("Time used: %.3fs", time.time() - start_time)
if use_ocr:
hidden.add_ocrinfo(newtree, filebase + '.png')
hidden.find_hidden_ocr(newtree)
util.print_tree(newtree)
if print_rets:
dlg = dialog.detect_dialog(newtree)
if dlg[0]:
logger.info("I think this is dialog")
for btnid in dlg[1]:
logger.info("btn: %s", util.describe_node(newtree[btnid]))
logger.info(
"is: %s",
dialog.detect_dialog_button(newtree, btnid, dlg[1]))
logger.info("decide to click: %s",
dialog.decide_dialog_action(newtree))
if print_error:
for itemid in descs:
found = False
for nodeid in newtree:
if itemid in newtree[nodeid]['raw']:
found = True
break
if not found:
logger.error("REMOVED: %s %d %s %s",
os.path.basename(filename), itemid,
descs[itemid], util.describe(items[itemid]))
if show_stat:
analyzer.show_stat()
return ret
def draw_bx2(
self,
minuit=None,
parmloc=(0.05, 0.95),
nfbins=500,
ax=None,
print_par=True,
args=None,
errors=None,
parts=False,
grid=True,
no_plot=False,
):
data_ret = None
error_ret = None
total_ret = None
part_ret = []
ax = plt.gca() if ax is None and not no_plot else ax
arg, error = _get_args_and_errors(self, minuit, args, errors)
m = mid(self.edges)
if not no_plot:
ax.errorbar(m, self.h, self.err, fmt=".")
data_ret = (self.edges, self.h)
error_ret = (self.err, self.err)
bound = (self.edges[0], self.edges[-1])
scale = nfbins / float(self.bins) # scale back to bins
draw_arg = [("lw", 2)]
if not parts:
draw_arg.append(("color", "r"))
total_ret = draw_pdf(
self.f, arg, bins=nfbins, bound=bound, ax=ax, density=False, scale=scale, no_plot=no_plot, **dict(draw_arg)
)
if parts:
f_parts = getattr(self.f, "parts", None)
if f_parts is not None:
for p in f_parts():
tmp = draw_pdf(p, arg, bound=bound, bins=nfbins, ax=ax, density=False, scale=scale, no_plot=no_plot)
part_ret.append(tmp)
if not no_plot:
ax.grid(grid)
txt = _param_text(describe(self), arg, error)
chi2 = self(*arg)
if self.ndof > 0:
txt += u"chi2/ndof = %5.4g(%5.4g/%d)" % (chi2 / self.ndof, chi2, self.ndof)
else:
txt += u"chi2/ndof = (%5.4g/%d)" % (chi2, self.ndof)
if print_par and not no_plot:
ax.text(parmloc[0], parmloc[1], txt, ha="left", va="top", transform=ax.transAxes)
return (data_ret, error_ret, total_ret, part_ret)
def draw_ulh(
self,
minuit=None,
bins=100,
ax=None,
bound=None,
parmloc=(0.05, 0.95),
nfbins=200,
print_par=True,
grid=True,
args=None,
errors=None,
parts=False,
show_errbars="normal",
no_plot=False,
):
data_ret = None
error_ret = None
total_ret = None
part_ret = []
ax = plt.gca() if ax is None and not no_plot else ax
arg, error = _get_args_and_errors(self, minuit, args, errors)
n, e = np.histogram(self.data, bins=bins, range=bound, weights=self.weights)
dataint = (n * np.diff(e)).sum()
data_ret = (e, n)
if not show_errbars:
if not no_plot:
pp = ax.hist(mid(e), bins=e, weights=n, histtype="step")
error_ret = (np.sqrt(n), np.sqrt(n))
else:
w2 = None
if show_errbars == "normal":
w2 = n
error_ret = (np.sqrt(n), np.sqrt(n))
elif show_errbars == "sumw2":
weights = None
if self.weights != None:
weights = self.weights ** 2
w2, e = np.histogram(self.data, bins=e, weights=weights)
error_ret = (np.sqrt(w2), np.sqrt(w2))
else:
raise ValueError("show_errbars must be 'normal' or 'sumw2'")
if not no_plot:
pp = ax.errorbar(mid(e), n, np.sqrt(w2), fmt="b.", capsize=0)
# bound = (e[0], e[-1])
draw_arg = [("lw", 2)]
if not parts:
draw_arg.append(("color", "r"))
# Draw pdf with finer bins
ef = np.linspace(e[0], e[-1], nfbins + 1)
scale = dataint if not self.extended else nfbins / float(bins)
total_ret = draw_pdf_with_edges(self.f, arg, ef, ax=ax, density=not self.extended, scale=scale, **dict(draw_arg))
if parts:
f_parts = getattr(self.f, "parts", None)
if f_parts is not None:
for p in f_parts():
ret = draw_pdf_with_edges(p, arg, ef, ax=ax, scale=scale, density=not self.extended, no_plot=no_plot)
part_ret.append(ret)
if not no_plot:
ax.grid(grid)
txt = _param_text(describe(self), arg, error)
if not no_plot and print_par:
ax.text(parmloc[0], parmloc[1], txt, ha="left", va="top", transform=ax.transAxes)
return (data_ret, error_ret, total_ret, part_ret)
def draw_x2(self,
minuit=None,
ax=None,
parmloc=(0.05, 0.95),
print_par=True,
args=None,
errors=None,
grid=True,
parts=False,
no_plot=False):
data_ret = None
error_ret = None
total_ret = None
part_ret = []
ax = plt.gca() if ax is None and not no_plot else ax
arg, error = _get_args_and_errors(self, minuit, args, errors)
x = self.x
y = self.y
data_err = self.error
data_ret = x, y
if data_err is None:
if not no_plot: ax.plot(x, y, '+')
err_ret = (np.ones(len(self.x)), np.ones(len(self.x)))
else:
if not no_plot: ax.errorbar(x, y, data_err, fmt='.')
err_ret = (data_err, data_err)
draw_arg = [('lw', 2)]
draw_arg.append(('color', 'r'))
total_ret = draw_pdf_with_midpoints(self.f,
arg,
x,
ax=ax,
no_plot=no_plot,
**dict(draw_arg))
if not no_plot:
ax.grid(grid)
txt = _param_text(describe(self), arg, error)
chi2 = self(*arg)
if self.ndof > 0:
txt += u'chi2/ndof = %5.4g(%5.4g/%d)' % (chi2 / self.ndof, chi2,
self.ndof)
else:
txt += u'chi2/ndof = (%5.4g/%d)' % (chi2, self.ndof)
if parts:
f_parts = getattr(self.f, 'parts', None)
if f_parts is not None:
for p in f_parts():
tmp = draw_pdf_with_midpoints(p,
arg,
x,
ax=ax,
no_plot=no_plot,
**dict(draw_arg))
part_ret.append(tmp)
if print_par and not no_plot:
ax.text(parmloc[0],
parmloc[1],
txt,
ha='left',
va='top',
transform=ax.transAxes)
return (data_ret, error_ret, total_ret, part_ret)
def draw_bx2(self,
minuit=None,
parmloc=(0.05, 0.95),
nfbins=500,
ax=None,
print_par=True,
args=None,
errors=None,
parts=False,
grid=True,
no_plot=False):
data_ret = None
error_ret = None
total_ret = None
part_ret = []
ax = plt.gca() if ax is None and not no_plot else ax
arg, error = _get_args_and_errors(self, minuit, args, errors)
m = mid(self.edges)
if not no_plot:
ax.errorbar(m, self.h, self.err, fmt='.')
data_ret = (self.edges, self.h)
error_ret = (self.err, self.err)
bound = (self.edges[0], self.edges[-1])
scale = nfbins / float(self.bins) #scale back to bins
draw_arg = [('lw', 2)]
if not parts:
draw_arg.append(('color', 'r'))
total_ret = draw_pdf(self.f,
arg,
bins=nfbins,
bound=bound,
ax=ax,
density=False,
scale=scale,
no_plot=no_plot,
**dict(draw_arg))
if parts:
f_parts = getattr(self.f, 'parts', None)
if f_parts is not None:
for p in f_parts():
tmp = draw_pdf(p,
arg,
bound=bound,
bins=nfbins,
ax=ax,
density=False,
scale=scale,
no_plot=no_plot)
part_ret.append(tmp)
if not no_plot:
ax.grid(grid)
txt = _param_text(describe(self), arg, error)
chi2 = self(*arg)
if self.ndof > 0:
txt += u'chi2/ndof = %5.4g(%5.4g/%d)' % (chi2 / self.ndof, chi2,
self.ndof)
else:
txt += u'chi2/ndof = (%5.4g/%d)' % (chi2, self.ndof)
if print_par and not no_plot:
ax.text(parmloc[0],
parmloc[1],
txt,
ha='left',
va='top',
transform=ax.transAxes)
return (data_ret, error_ret, total_ret, part_ret)
def handle_console_cmd(cons, cmd, envs):
threading.current_thread().name = 'MainThread'
dev = envs['dev']
ob = envs['ob']
st = envs['state']
tlib = envs['tlib']
env = envs['env']
if 'app' in envs:
app = envs['app']
else:
app = 'cui'
def save_stat():
tlib.save_stat(os.path.join("../stat/", "%s.txt" % app))
if cmd == 'q':
save_stat()
dev.finish()
sys.exit(0)
elif cmd == 'sense':
scr = ob.grab_state(dev, no_verify=True, no_print=True)
if scr.get('guess_descs') is None:
util.print_tree(scr.get('tree'))
else:
util.print_tree(scr.get('tree'), scr.get('guess_descs'),
scr.get('guess_score'))
st.merge(scr)
return
elif cmd == 'tags':
config.show_guess_tags = True
scr = ob.grab_state(dev, no_verify=True, no_print=True)
util.print_tree(scr.get('tree'), scr.get('guess_descs'), scr.get('guess_score'))
st.merge(scr)
config.show_guess_tags = False
return
elif cmd == 'tree':
scr = ob.grab_state(dev, no_img=True)
util.print_tree(scr.get('tree'))
st.merge(scr)
return
elif cmd.startswith('app '):
app = cmd.split(' ')[1]
load_app(app, ob, tlib, envs)
return
elif cmd == 'load':
tlib = testlib.collect_pieces("../tlib/")
envs['tlib'] = tlib
load_app(app, ob, tlib, envs)
return
elif cmd == 'reload':
value.init_params("../etc/", app)
try:
if app:
os.remove("../model/screen_%s" % app)
os.remove("../model/element_%s" % app)
else:
os.remove("../model/screen")
os.remove("../model/element")
except:
pass
ob.load("../model/", "../guis/", "../guis-extra/", config.extra_screens,
config.extra_element_scrs)
return
elif cmd == 'tlib':
tlib = testlib.collect_pieces("../tlib/")
envs['tlib'] = tlib
return
elif cmd == 'test':
tlib = testlib.collect_pieces("../tlib/")
tlib.set_app(app)
tlib.add_test(microtest.init_test(appdb.get_app(app)))
envs['tlib'] = tlib
config.show_guess_tags = True
scr = ob.grab_state(dev)
config.show_guess_tags = False
st.merge(scr)
begin_state = st.to_essential(tlib.essential_props())
tests = tlib.usable_tests(st)
tests.sort(key=lambda test: test.prio)
for i in range(len(tests)):
testinfo = tlib.get_testinfo(tests[i])
print("%2d. %s: %s [%d %d]" % (i, tests[i].feature_name, tests[i].name,
testinfo.succs, testinfo.fails))
if len(tests) == 0:
print("no test available!")
return
if len(tests) == 1:
tid = 0
else:
tid = input("which one? ")
try:
tid = int(tid)
except:
return
test = tests[tid]
ret = test.attempt(dev, ob, st, tlib, environ.empty)
util.print_tree(st.get('tree'), st.get('guess_descs'))
end_state = st.to_essential(tlib.essential_props())
if ret:
print("test SUCC")
tlib.clear_stat(test)
tlib.mark_succ(test, begin_state, end_state)
else:
print("test FAIL")
tlib.mark_fail(test, begin_state)
save_stat()
return
elif cmd == 'save':
save_stat()
return
elif cmd == 'stat':
tlib.print_stat()
return
elif cmd == 'items':
scr = ob.grab_state(dev, no_img=True)
util.printitems(scr.get('items'))
return
elif cmd.startswith('item'):
itemid = int(cmd.split(' ')[1])
scr = ob.grab_state(dev, no_img=True)
items = scr.get('items')
if itemid in items:
print(util.describe(items[itemid]))
else:
print("no such item: %d" % itemid)
return
elif cmd == 'debug':
logging.basicConfig(level=logging.DEBUG)
return
elif cmd == 'idle':
dev.wait_idle()
return
elif cmd == 'dialog':
scr = ob.grab_state(dev)
ret = tlib.handle_dialog(scr, dev, ob)
if ret:
print("dialog handled")
else:
print("dialog not handled")
return
elif cmd.startswith('find '):
tag = cmd.split(' ', 1)[1]
con = concept.parse(tag)
if con is None:
print("invalid locator")
return
scr = ob.grab_state(dev)
widgets = con.locate(scr, ob, environ.Environment())
if widgets == []:
print("can't find")
else:
for widget in widgets:
print("FOUND: %s" % widget)
print(" content:", widget.content())
return
elif cmd == 'perf':
perfmon.print_stat()
return
elif cmd == 'clear':
init = tlib.find_test('meta', 'start app')
st.reset()
init.attempt(dev, ob, st, tlib, None)
ob.update_state(dev, st)
tlib.mark_succ(init, state.init_state, st)
return
elif cmd.startswith('set'):
parts = cmd.split(' ', 2)
if len(parts) == 2:
key = parts[1]
val = "1"
else:
(key, val) = parts[1:]
st.set(key, val)
return
elif cmd.startswith('del'):
parts = cmd.split(' ', 1)
key = parts[1]
st.remove(key)
return
elif cmd == 'dump':
filename = util.choose_filename("../cap/", "page%d")
logger.info("dumping to page %s", filename)
dev.dump(filename)
#sense.dump_page(dev, "../cap/")
return
elif cmd == 'list':
scr = ob.grab_state(dev, no_img=True)
tree = scr.get('tree')
treeinfo = analyze.collect_treeinfo(tree)
for root in sorted(treeinfo['listlike']):
print("ROOT:", util.describe_node(tree[root]))
for item in sorted(treeinfo['itemlike']):
if listinfo.get_lca(tree, [root, item]) == root:
print(" NODE:", util.describe_node(tree[item]))
for field in sorted(treeinfo['dupid']):
if listinfo.get_lca(tree, [item, field]) == item:
print(" FIELD:", util.describe_node(tree[field]))
return
elif cmd == 'webon':
config.GRAB_WEBVIEW = True
return
elif cmd == 'weboff':
config.GRAB_WEBVIEW = False
return
elif cmd.startswith('ob '):
prop = cmd.split(' ', 1)[1]
wd = watchdog.Watchdog(100000)
smgr = statemgr.StateMgr(tlib, None, dev, ob, wd)
ret = smgr.observe_prop(prop, st)
if ret:
print("prop %s = %s" % (prop, st.get(prop, '')))
else:
print("observe error")
return
elif cmd.startswith('clean '):
parts = cmd.split(' ', 2)
if len(parts) == 2:
prop = parts[1]
val = "1"
else:
(prop, val) = parts[1:]
wd = watchdog.Watchdog(100000)
smgr = statemgr.StateMgr(tlib, None, dev, ob, wd)
ret = smgr.cleanup_prop(prop, val, st)
if ret:
print("prop %s cleaned" % prop)
else:
print("clean error")
return
elif cmd.startswith('oc '):
prop = cmd.split(' ', 1)[1]
wd = watchdog.Watchdog(100000)
smgr = statemgr.StateMgr(tlib, None, dev, ob, wd)
ret = smgr.observe_and_clean(prop, st)
if ret:
print("prop %s is clean now" % prop)
else:
print("observe/clean error")
return
elif cmd == 'dbg':
print(tlib)
return
elif cmd.startswith('skip '):
name = cmd.split(' ', 1)[1]
add_skip(app, name)
print("skipping %s" % name)
return
elif cmd == 'skip':
skips = load_skip(app)
for skip in skips:
print("now skipping %s" % skip)
return
elif cmd.startswith('noskip '):
name = cmd.split(' ', 1)[1]
del_skip(app, name)
print("not skipping %s" % name)
return
elif cmd == 'src':
if st.get('xml') is not None:
print(st.get('xml'))
if st.get('src') is not None:
print(st.get('src'))
return
elif cmd == 'install':
apputils.install(dev, app)
return
elif cmd == 'uninstall':
apputils.uninstall(dev, app)
return
op = operation.parse_line(cmd)
if op is None:
print("unknown op")
return
ret = op.do(dev, ob, st, env, tlib)
if not ret:
print("op failed")
def guess(filename, lasttag=None):
basename = filename.replace('.xml', '')
(items, descs) = analyze.load_case(filename)
# if descs:
# return
appname = basename.split('/')[-1].split('.')[0].split('_')[0]
scrname = basename.split('/')[-1].split('.')[0].split('_')[-1]
imgfile = filename.replace('.xml', '.png')
if not appname in tag_db:
tag_db[appname] = {}
if appname in tag_db:
for tag in tag_db[appname]:
xid = tag_db[appname][tag]
for itemid in items:
if items[itemid]['id'] == xid:
descs[itemid] = tag
print("%s Current:" % basename)
util.printdesc(items, descs)
# analyze.printitems(items)
tree = analyze.analyze([filename])[0]
util.print_tree(tree, descs)
viewproc = subprocess.Popen([config.picviewer_path, basename + ".png"])
prompt = ''
while True:
print(tags.tag['universal'])
if scrname in tags.tag:
print(tags.tag[scrname])
line = input(prompt + "> ")
parts = line.split(' ')
cmd = parts[0]
if cmd == '':
util.save_desc(basename, descs)
break
elif cmd[0] == 'q':
viewproc.kill()
sys.exit(0)
elif cmd[0] == 'l':
listid = int(parts[1])
listtype = parts[2]
guess_list(listid, listtype, items, descs)
util.printdesc(items, descs)
elif cmd[0] == 'a':
auto_guess(filename, items, descs)
util.print_tree(tree, descs)
elif cmd[0] == 'y':
util.save_desc(basename, descs)
break
elif cmd[0] >= '0' and cmd[0] <= '9':
if len(parts) > 1:
lasttag = parts[1]
util.mark_item(parts, items, descs, scrname, tag_db[appname])
util.print_tree(tree, descs)
else:
if lasttag is None:
print(util.describe(items[int(cmd)]))
else:
util.mark_item([cmd, lasttag], items, descs, scrname,
tag_db[appname])
util.print_tree(tree, descs)
elif cmd[0] == 'd':
if cmd == 'dr':
recursive = True
else:
recursive = False
util.remove_mark(parts[1:], items, descs, recursive)
util.print_tree(tree, descs)
elif cmd[0] == ';':
util.print_tree(tree, descs)
elif cmd[0] == '!':
util.printitems(items)
elif cmd[0] == 'c':
element_clas = elements.getmodel("../model/", "../guis/", None)
imgdata = skimage.io.imread(imgfile, as_grey=True)
imgdata = skimage.transform.resize(imgdata,
(config.height, config.width))
treeinfo = analyze.collect_treeinfo(tree)
for itemid in tree:
(guess_element,
score) = element_clas.classify(scrname, tree, itemid, imgdata,
treeinfo)
if guess_element != 'NONE':
descs[itemid] = guess_element
util.print_tree(tree, descs)
elif cmd[0] == '?':
print(util.describe(items[int(parts[1])]))
else:
print("Unknown command")
viewproc.kill()
