def frame_progress(self, length, width=None, symbol="#"): """ Draw (lazy) progress bar for the DataFrame: >>> f = DataFrame ( ... ) >>> p = f.ProgressBar ( 1000 ) ## number of elements! >>> .... """ cnt = self.Count() if not isatty(): return cnt left = "[ " right = " ]" minw = len(left) + len(right) + 3 + 1 + 1 + 1 length = length if isinstance(length, integer_types) and 0 < length else len(self) width = width if isinstance( width, integer_types) and 10 + minw <= width else terminal_size()[1] width = max(10, width - minw) nchunks = max(250, width + 1) csize, rr = divmod(length, nchunks) if rr: nchunks += 1 symbol = allright(symbol) fun = Ostap.Utils.frame_progress(nchunks, width, symbol, ' ', left, right) cnt.OnPartialResultSlot(csize, fun) return cnt
def frame_progress(self, length, width=None, symbol="#"): """ Draw (lazy) progress bar for the DataFrame: >>> f = DataFrame ( ... ) >>> p = f.ProgressBar ( 1000 ) ## number of elements! >>> .... """ cnt = self.Count() if not isatty(): return cnt length = length if isinstance(length, integer_types) and 0 < length else len(self) width = width if isinstance( width, integer_types) and 10 < width else terminal_size()[1] if width < 16: width = 16 nchunks = width - 14 csize = max(int(length / nchunks), 1) left = "[ " right = " ]" symbol = allright(symbol) fun = Ostap.Utils.frame_progress(csize, nchunks, symbol, ' ', left, right) cnt.OnPartialResultSlot(csize, fun) return cnt
def table(rows, title='', prefix=''): """Format the list of rows as a table. - Each row is a sequence of column cells. - The first row defines the column headers. When terminaltables package is accessible, use it to get nice pretty tables, otherwise use some home-made primitive replacement - see https://pypi.org/project/terminaltables - see https://github.com/Robpol86/terminaltables >>> table_data = [ ... ( 'Name' , 'Occupation' , 'Note' ) , ... ( 'Alice' , '?' , '---' ) , ... ( 'Bob' , 'unemployed' , '' ) ] >>> t = table ( table_data , 'Title' ) >>> print (t) """ from ostap.utils.basic import isatty title = allright(decolorize(title)) if rows: rows = list(rows) header_row = rows[0] header_row = [infostr(decolorize(c)) for c in header_row] rows[0] = header_row rows = tuple(rows) if terminaltables and isatty(): table_instance = terminaltables.SingleTable(rows, title) table_instance.justify_columns[0] = 'left' table_instance.justify_columns[2] = 'right' return add_prefix(table_instance.table, prefix) elif terminaltables: title = allright(title) table_instance = terminaltables.AsciiTable(rows, title) table_instance.justify_columns[0] = 'left' table_instance.justify_columns[2] = 'right' t = table_instance.table return add_prefix(table_instance.table, prefix) ## use the local replacement return the_table(rows, title, prefix)
def table(rows, title='', prefix='', alignment=(), wrap_width=-1, indent=wrap_indent): """Format the list of rows as a table. - Each row is a sequence of column cells. - The first row defines the column headers. When terminaltables package is accessible, use it to get nice pretty tables, otherwise use some home-made primitive replacement - see https://pypi.org/project/terminaltables - see https://github.com/Robpol86/terminaltables >>> table_data = [ ... ( 'Name' , 'Occupation' , 'Note' ) , ... ( 'Alice' , '?' , '---' ) , ... ( 'Bob' , 'unemployed' , '' ) ] >>> t = table ( table_data , 'Title' ) >>> print (t) """ from ostap.utils.basic import isatty title = allright(decolorize(title)) if rows: rows = list(rows) header_row = rows[0] header_row = [infostr(decolorize(c)) for c in header_row] rows[0] = header_row rows = tuple(rows) rows = [list(row) for row in rows] if not terminaltables: ## use the local replacement return the_table(rows, title, prefix, alignment=alignment) if isatty(): title = allright(title) table_instance = terminaltables.SingleTable(rows, title) else: title = allright(title) table_instance = terminaltables.AsciiTable(rows, title) cw = table_instance.column_widths nc = len(cw) wraps = [i for (i, a) in enumerate(alignment) if a in wrapped] if wraps: from terminaltables.width_and_alignment import max_dimensions widths = max_dimensions(table_instance.table_data, table_instance.padding_left, table_instance.padding_right)[2] widths = sum(l for (i, l) in enumerate(widths) if not i in wraps) widths += nc + 1 + len(prefix) + 4 + 2 * len(wraps) _, w = terminal_size() ww = w - widths ww, _ = divmod(ww, len(wraps)) if 12 < ww and ww < wrap_width: wrap_width = ww elif 12 < ww and wrap_width <= 0: wrap_width = ww if wrap_width < 12: wrap_width = max_width nw = len(wraps) for i, a in zip(range(nc), alignment): if a and isinstance(a, str): al = a.lower() if al in left: table_instance.justify_columns[i] = 'left' elif al in right: table_instance.justify_columns[i] = 'right' elif al in center: table_instance.justify_columns[i] = 'center' elif al in wrapped: maxw = table_instance.column_max_width(i) if 15 < wrap_width * nw < maxw: maxw = (wrap_width - 3) * nw if 1 < nw else wrap_width if maxw < 15: maxw = (wrap_width - 3) * nw if 1 < nw else wrap_width if maxw < 15: maxw = (max_width - 3) * nw if 1 < nw else max_width width = maxw / nw if 1 < nw else maxw for l, line in enumerate(table_instance.table_data): if width < len(line[i]): table_instance.table_data[l][i] = textwrap.fill( indent + line[i], wrap_width) return add_prefix(table_instance.table, prefix)
def the_table(rows, title='', prefix=''): """Format the list of rows as a table (home-made primitive) - Each row is a sequence of column cells. - The first row defines the column headers. >>> table_data = [ ... ( 'Name' , 'Occupation' , 'Note' ) , ... ( 'Alice' , '?' , '---' ) , ... ( 'Bob' , 'unemployed' , '' ) ] >>> t = the_table ( table_data , 'Title' ) >>> print (t) """ ## calculate the number of columns nc = 0 for row in rows: nc = max(nc, len(row)) ## calculate the maximum width for columns widths = {} for row in rows: cols = [c for c in row] while len(cols) < nc: cols.append('') for i, c in enumerate(cols): if not i in widths: widths[i] = 1 widths[i] = max(widths[i], len(decolorize(c))) totwidth = 0 for c in widths: totwidth += widths[c] totwidth += (nc - 1) * 3 if totwidth < len(title): delta = 1 + (len(title) - totwidth) // nc for c in widths: widths[c] += delta hformats = ["{:^%d}" % widths[c] for c in range(nc)] rformats = [" {:^%d} " % widths[c] for c in range(nc)] # the first column is left adjusted rformats[0] = rformats[0].replace('^', '<') # the last column is right adjusted rformats[-1] = rformats[-1].replace('^', '>') seps = ['-' * (widths[c] + 2) for c in range(nc)] sepline = '+' + "+".join(seps) + '+' table = [] if title: sline = '+' + '-' * (len(sepline) - 2) + '+' tfmt = "{:^%d}" % (len(sepline) - 4) t = '| ' + allright(tfmt.format(decolorize(title))) + ' |' table.append(sline) table.append(t) table.append(sepline) for i, row in enumerate(rows): cols = [c for c in row] while len(cols) < nc: cols.append('') if 0 == i: table.append('| ' + ' | '.join([ infostr(f.format(decolorize(i))) for (f, i) in zip(hformats, cols) ]) + ' |') table.append(sepline) else: table.append('|' + '|'.join( [f.format(decolorize(i)) for (f, i) in zip(rformats, cols)]) + '|') table.append(sepline) return prefix + ('\n' + prefix).join(table) if prefix else '\n'.join(table)
def _rfr_table_ ( r , title = '' , prefix = '' , more_vars = {} ) : """ print RooFitResult as a table >>> result = ... >>> result.table() """ from ostap.fitting.utils import fit_status, cov_qual rows = [] ## 1. fit status status = r.status() if status : row = attention ( 'Status' ) , '' , attention ( fit_status ( status ) ) , '' rows.append ( row ) else : row = 'Status' , '' , allright ( fit_status ( status ) ) , '' rows.append ( row ) ## 2. minumum NLL s , n = pretty_float ( r.minNll() ) if n : n = '[10^%+d]' % n else : n = '' rows.append ( ( "Minimized FCN/NLL value" , n , ' ' + s , '' ) ) s , n = pretty_float ( r.edm () ) if n : n = '[10^%+d]' % n else : n = '' rows.append ( ( 'Estimated distance to minimum' , n , ' ' + s , '' ) ) cq = r.covQual() cn = '' if -1 == cq : cn = cov_qual ( cq ) elif 3 == cq : cn = allright ( cov_qual ( cq ) ) elif cq in ( 0 , 1 , 2 ) : cn = attention ( cov_qual ( cq ) ) else : cn = cov_qual ( cq ) rows.append ( ( 'Covariance matrix quality' , '' , ' ' + cn , '' ) ) for i in range ( r.numStatusHistory() ) : label = r.statusLabelHistory ( i ) code = r.statusCodeHistory ( i ) row = 'Status: %s '% label , '' , '%d' % code if not code in ( 0 , -1 ) : row = attention ( row [ 0 ] ) , row [ 1 ] , ' ' + attention ( row [ 2 ] ) , '' else : row = row [ 0 ] , row [ 1 ] , ' ' + allright ( row [ 2 ] ) , '' rows.append ( row ) nbadnll = r.numInvalidNLL() if 0 < nbadnll : rows.append ( ( 'Invalid FCN/NLL evaluations' , '' , ' %d' % nbadnll , '' ) ) with_globcorr = not ( (6,24) <= root_info < (6,28) ) if with_globcorr : rows = [ ( '', 'Unit', 'Value' , 'Global/max correlation [%]') ] + rows else : rows = [ ( '', 'Unit', 'Value' , 'Max correlation [%]') ] + rows pars_all = r.params ( float_only = False ) pars_float = r.params ( float_only = True ) ## constant/fix parameters crows = [] for p in pars_all : if p in pars_float : continue v , a = pars_all [ p ] s , n = pretty_float ( v.value() ) if n : n = '[10^%+d]' % n else : n = '' row = p , n , ' ' + s + ' (fix)' , '' crows.append ( row ) ## floating parameters max_corr = False frows = [] for p in pars_float : v , a = pars_float [ p ] if not a.hasAsymError() : s , n = pretty_ve ( v ) else : s , n = pretty_2ve ( a.getVal() , a.getAsymErrorHi() , a.getAsymErrorLo() ) if n : n = '[10^%+d]' % n else : n = '' if 0 <= cq and 1 < len ( pars_float ) : mxr , mxv = r.max_cor ( p ) if with_globcorr : gc = -1.0 gc = r.globalCorr ( p ) if 3 == cq else -1.00 if 0 <= gc : cc = '% +5.1f/(% +5.1f,%s)' % ( gc*100 , mxr*100 , mxv ) else : cc = '% +5.1f : %-s' % ( mxr * 100 , mxv ) if 0.95 < abs ( mxr ) or 0.95 < gc : cc = attention ( cc ) else : cc = '% +5.1f : %-s' % ( mxr * 100 , mxv ) if 0.95 < abs ( mxr ) : cc = attention ( cc ) max_corr = True row = p , n , s , cc else : row = p , n , s frows.append ( row ) ## more parameters mrows = [] for p in sorted ( more_vars ) : func = more_vars [ p ] v = func ( r ) s , n = pretty_ve ( v ) if n : n = '[10^%+d]' % n else : n = '' cc = 'derived' row = p , n , s , cc mrows.append ( row ) crows.sort() frows.sort() all = rows + crows + frows + mrows import ostap.logger.table as T return T.table ( all , title = title if title else r.GetTitle() , prefix = prefix , alignment = 'llll' )
def makeWeights( dataset, plots=[], database="weights.db", compare=None, ## comparison function delta=0.01, ## delta for ``mean'' weight variation minmax=0.03, ## delta for ``minmax'' weight variation power=None, ## auto-determination debug=True, ## save intermediate information in DB make_plots=False, ## make plots tag="Reweighting"): """The main function: perform one re-weighting iteration and reweight ``MC''-data set to looks as ``data''(reference) dataset >>> results = makeWeights ( ... dataset , ## data source to be reweighted (DataSet, TTree, abstract source) ... plots , ## reweighting plots ... database , ## datadabse to store/update reweigting results ... delta , ## stopping criteria for `mean` weight variation ... minmax , ## stopping criteria for `min/max` weight variation ... power , ## effective power to apply to the weigths ... debug = True , ## store debuig information in database ... make_plots = True , ## produce useful comparison plots ... tag = 'RW' ) ## tag for better printout If `make_plots = False`, it returns the tuple of active reweitings: >>> active = makeWeights ( ... , make_plots = False , ... ) Otherwise it also returns list of comparison plots >>> active, cmp_plots = makeWeights ( ... , make_plots = True , ... ) >>> for item in cmp_plots : ... what = item.what ... hdata = item.data ... hmc = item.mc ... hweight = item.weight If no more rewighting iteratios required, <code>active</code> is an empty tuple """ assert 0 < delta, "makeWeights(%s): Invalid value for ``delta'' %s" % ( tag, delta) assert 0 < minmax, "makeWeights(%s): Invalid value for ``minmax'' %s" % ( tag, minmax) from ostap.logger.colorized import allright, attention, infostr from ostap.utils.basic import isatty nplots = len(plots) ## if 1 < nplots : ## import math ## fudge_factor = math.sqrt ( 1.0 / max ( 2.0 , nplots - 1.0 ) ) ## delta = delta * fudge_factor ## minmax = minmax * fudge_factor ## list of plots to compare cmp_plots = [] ## reweighting summary table header = ('Reweighting', 'wmin/wmax', 'OK?', 'wrms[%]', 'OK?', 'chi2/ndf', 'ww', 'exp') rows = {} save_to_db = [] ## number of active plots for reweighting for wplot in plots: what = wplot.what ## variable/function to plot/compare how = wplot.how ## weight and/or additional cuts address = wplot.address ## address in database hdata0 = wplot.data ## original "DATA" object hmc0 = wplot.mc_histo ## original "MC" histogram ww = wplot.w ## relative weight projector = wplot.projector ## projector for MC data ignore = wplot.ignore ## ignore for weigtht building? # # normalize the data # hdata = hdata0 if isinstance(hdata, ROOT.TH1): hdata = hdata.density() # ===================================================================== ## make a plot on (MC) data with the weight # ===================================================================== hmc0 = projector(dataset, hmc0, what, how) st = hmc0.stat() mnmx = st.minmax() if iszero(mnmx[0]): logger.warning("%s: statistic goes to zero %s/``%s''" % (tag, st, address)) elif mnmx[0] <= 0: logger.warning("%s: statistic is negative %s/``%s''" % (tag, st, address)) # ===================================================================== ## normalize MC # ===================================================================== hmc = hmc0.density() # ===================================================================== ## calculate the reweighting factor : a bit conservative (?) # this is the only important line # ===================================================================== # try to exploit finer binning if/when possible hboth = isinstance(hmc, ROOT.TH1) and isinstance(hdata, ROOT.TH1) if hboth and 1 == hmc.dim () and 1 == hdata.dim () and \ len ( hmc ) >= len( hdata ) : w = (1.0 / hmc) * hdata ## NB! elif hboth and 2 == hmc.dim () and 2 == hdata.dim () and \ ( hmc.binsx() >= hdata.binsx() ) and \ ( hmc.binsy() >= hdata.binsy() ) : w = (1.0 / hmc) * hdata ## NB! elif hboth and 3 == hmc.dim () and 3 == hdata.dim () and \ ( hmc.binsx() >= hdata.binsx() ) and \ ( hmc.binsy() >= hdata.binsy() ) and \ ( hmc.binsz() >= hdata.binsz() ) : w = (1.0 / hmc) * hdata ## NB! else: w = hdata / hmc ## NB! # ===================================================================== ## scale & get the statistics of weights w /= w.stat().mean().value() cnt = w.stat() # mnw, mxw = cnt.minmax() wvar = cnt.rms() / cnt.mean() good1 = wvar.value() <= delta good2 = abs(mxw - mnw) <= minmax good = good1 and good2 ## small variance? # c2ndf = 0 for i in w: c2ndf += w[i].chi2(1.0) c2ndf /= (len(w) - 1) ## build the row in the summary table row = address , \ '%-5.3f/%5.3f' % ( cnt.minmax()[0] , cnt.minmax()[1] ) , \ allright ( '+' ) if good2 else attention ( '-' ) , \ (wvar * 100).toString('%6.2f+-%-6.2f') , \ allright ( '+' ) if good1 else attention ( '-' ) , '%6.2f' % c2ndf ## make plots at the start of each iteration? if make_plots: item = ComparisonPlot(what, hdata, hmc, w) cmp_plots.append(item) row = tuple(list(row) + ['%4.3f' % ww if 1 != ww else '']) rows[address] = row # ## make decision based on the variance of weights # mnw, mxw = cnt.minmax() if (not good) and (not ignore): ## small variance? save_to_db.append((address, ww, hdata0, hmc0, hdata, hmc, w)) # ===================================================================== ## make a comparison (if needed) # ===================================================================== if compare: compare(hdata0, hmc0, address) active = tuple([p[0] for p in save_to_db]) nactive = len(active) if power and callable(power): eff_exp = power(nactive) elif isinstance(power, num_types) and 0 < power <= 1.5: eff_exp = 1.0 * power elif 1 == nactive and 1 < len(plots): eff_exp = 0.95 elif 1 == nactive: eff_exp = 1.00 else: eff_exp = 1.10 / max(nactive, 1) while database and save_to_db: entry = save_to_db.pop() address, ww, hd0, hm0, hd, hm, weight = entry cnt = weight.stat() mnw, mxw = cnt.minmax() ## avoid too large or too small weights for i in weight: w = weight[i] if w.value() < 0.5: weight[i] = VE(0.5, w.cov2()) elif w.value() > 2.0: weight[i] = VE(2.0, w.cov2()) if 1 < nactive and 1 != ww: eff_exp *= ww logger.info("%s: apply ``effective exponent'' of %.3f for ``%s''" % (tag, eff_exp, address)) if 1 != eff_exp and 0 < eff_exp: weight = weight**eff_exp row = list(rows[address]) row.append('%4.3f' % eff_exp) rows[address] = tuple(row) with DBASE.open(database) as db: db[address] = db.get(address, []) + [weight] if debug: addr = address + ':REWEIGHTING' db[addr] = db.get(addr, []) + list(entry[2:]) del hd0, hm0, hd, hm, weight, entry table = [header] for row in rows: table.append(rows[row]) import ostap.logger.table as Table logger.info( '%s, active:#%d \n%s ' % (tag, nactive, Table.table(table, title=tag, prefix='# ', alignment='lccccccc'))) cmp_plots = tuple(cmp_plots) return (active, cmp_plots) if make_plots else active
def _fit_table_ ( rfit , title = '' , prefix = '' ) : """Print <code>TFitResult</code> as a table """ from ostap.fitting.utils import fit_status, cov_qual from ostap.logger.colorized import attention, allright from ostap.logger.utils import pretty_float, pretty_ve, pretty_2ve, fmt_pretty_ve header = ( '', 'Unit' , 'Value' ) rows = [] ## 0. minimized type row = "Minimizer Type" , '' , rfit.MinimizerType() rows.append ( row ) ## 0. minimized type v = rfit.IsValid() if v : row = "Valid" , '' , 'True' else : row = "Valid" , '' , attention ( 'False') rows.append ( row ) ## 1. fit status status = rfit.Status() if status : row = attention ( 'Status' ) , '' , attention ( fit_status ( status ) ) rows.append ( row ) else : row = 'Status' , '' , allright ( fit_status ( status ) ) rows.append ( row ) ## 4. covariance status cq = rfit.CovMatrixStatus() cn = '' if -1 == cq : cn = cov_qual ( cq ) elif 3 == cq : cn = allright ( cov_qual ( cq ) ) elif cq in ( 0 , 1 , 2 ) : cn = attention ( cov_qual ( cq ) ) else : cn = cov_qual ( cq ) rows.append ( ( 'Covariance matrix quality' , '' , ' ' + cn ) ) ## 3-6. chi2,nDoF,chi2/nDoF,minFCN chi2 = rfit.Chi2 () s , n = pretty_float ( chi2 ) if n : n = '[10^%+d]' % n else : n = '' rows.append ( ( "Chi2" , n , ' ' + s ) ) ## ndf = rfit.Ndf() rows.append ( ( "nDoF" , '' , ' ' + '%d' % ndf ) ) ## c2ndf = rfit.Chi2 () / ndf s , n = pretty_float ( c2ndf ) if n : n = '[10^%+d]' % n else : n = '' rows.append ( ( "Chi2/nDoF" , n , ' ' + s ) ) ## minfcn = rfit.MinFcnValue() s , n = pretty_float ( minfcn ) if n : n = '[10^%+d]' % n else : n = '' rows.append ( ( "Minimal FCN" , n , ' ' + s ) ) ## 7.Probability in %[%] prob = rfit.Prob() / 100 rows.append ( ( "Probability" , '[%]' , ' %5.3e' % prob ) ) ## 8. distrance to minimum edm = rfit.Edm() s , n = pretty_float ( edm ) if n : n = '[10^%+d]' % n else : n = '' rows.append ( ( "Estimated distance to minimum" , n , ' ' + s ) ) ncalls = rfit.NCalls() rows.append ( ( "FCN calls" , '' , ' ' + '%d' % ncalls ) ) ## has_minos = False for i in rfit : if not rfit.HasMinosError( i ) : continue has_minos = True break if has_minos : rows = [ row + ('','','') for row in rows ] header = header + ( 'neg-minos' , 'pos-minos' , 'Global corr.' ) else : rows = [ row + ('',) for row in rows ] header = header + ( 'Global corr.' , ) for i in rfit : pname = rfit.GetParameterName ( i ) value = rfit.Value ( i ) fixed = rfit.IsParameterFixed ( i ) fmte = '' if fixed : v = value s , n = pretty_float ( v ) s = s + '(fixed)' nv = n else : error = rfit.Error ( i ) v = VE ( value , error * error ) ## fmt , fmtv , fmte , n = fmt_pretty_ve ( v ) s = fmt % ( value / 10**n , error / 10**n ) nv = n if n : n = '[10^%+d]' % n else : n = '' pname = "%-2d: %s"% ( i , pname ) row = pname , n , ' ' + s if not fixed and rfit.HasMinosError( i ) : if fmte : error_low = fmte % ( rfit.LowerError ( i ) / 10**nv ) error_up = fmte % ( rfit.UpperError ( i ) / 10**nv ) else : error_low = "%+8g" % ( rfit.LowerError ( i ) / 10**nv ) error_up = "%+8g" % ( rfit.UpperError ( i ) / 10**nv ) else : error_low = '' error_up = '' if has_minos : row = row + ( error_low , error_up ) gcc = rfit.GlobalCC ( i ) * 100 gcc = '%+5.1f%%' % gcc row = row + ( gcc, ) rows.append ( row ) if not title : title = rfit.GetTitle() import ostap.logger.table as T rows = [ header ] + rows return T.table ( rows , title = title , prefix = prefix )
from ostap.core.meta_info import root_info from ostap.logger.colorized import allright, attention # ============================================================================= # logging # ============================================================================= from ostap.logger.logger import getLogger if '__main__' == __name__ : logger = getLogger( 'ostap.fitting.minuit' ) else : logger = getLogger( __name__ ) # ============================================================================= logger.debug ( 'Some useful decorations for (T)Minuit functions') # ============================================================================= partypes = integer_types # ============================================================================= ## return codes from MINUIT commands return_codes = { 0 : allright ( 'command executed normally' ) , 1 : attention ( 'command is blank, ignored' ) , 2 : attention ( 'command line unreadable, ignored' ) , 3 : attention ( 'unknown command, ignored' ) , 4 : attention ( 'abnormal termination (e.g., MIGRAD not converged)' ), 5 : 'command is a request to read PARAMETER definitions' , 6 : "'SET INPUT' command" , 7 : "'SET TITLE' command" , 8 : "'SET COVAR' command" , 9 : 'reserved' , 10 : 'END command' , 11 : 'EXIT or STOP command' , 12 : 'RETURN command' , } # ============================================================================= ## get the parameter from (T)Minuit
def build_bar(self): """Figure new percent complete, and rebuild the bar string base on self.amount. """ diff = float(self.amount - self.min) done = (diff / float(self.span)) * 100.0 percent_done = int(round(done)) # figure the proper number of 'character' make up the bar all_full = self.width - 2 num_hashes = int(round((percent_done * all_full) / 100)) if 100 <= done and self.__end is None: self.__end = time.time() if self.__end is None and num_hashes == self._hashes: return False eta = '' leta = len(eta) if self.__end is None and 6 < num_hashes and 1 < done: now = time.time() feta = int((100 - done) * (now - self.__start) / done) h, _ = divmod(feta, 3600) m, s = divmod(feta - h * 3600, 60) if h: eta = 'ETA %02d:%02d:%02d ' % (h, m, s) elif m: eta = 'ETA %02d:%02d ' % (m, s) elif s >= 1: eta = 'ETA %02d ' % s leta = len(eta) elif (not self.__end is None) and 5 < num_hashes: now = self.__end feta = int(now - self.__start) h, _ = divmod(feta, 3600) m, s = divmod(feta - h * 3600, 60) if h: eta = '%02d:%02d:%02d ' % (h, m, s) elif m: eta = '%02d:%02d ' % (m, s) elif s >= 1: eta = '%ds ' % s leta = len(eta) if self.mode == 'dynamic': # build a progress bar with self.char (to create a dynamic bar # where the percent string moves along with the bar progress. if eta and leta < num_hashes: self.bar = allright(eta + self.char * (num_hashes - leta)) else: self.bar = allright(self.char * num_hashes) else: # build a progress bar with self.char and spaces (to create a # fixed bar (the percent string doesn't move) if eta and leta + 1 < num_hashes: self.bar = allright(eta + self.char * (num_hashes - leta)) + ' ' * (all_full - num_hashes) else: self.bar = allright( self.char * num_hashes) + ' ' * (all_full - num_hashes) percent_str = str(percent_done) + "%" self.bar = '[ ' + self.bar + ' ] ' + infostr(percent_str) self._hashes = num_hashes self._done = done return True
self.output.flush() self.silent = True def __enter__(self): self.show() return self def __exit__(self, *_): self.end() def __del__(self): self.end() # ============================================================================= _bar_ = (allright('Running ... | ') + '\r', allright('Running ... / ') + '\r', allright('Running ... - ') + '\r', allright('Running ... \\ ') + '\r') _lbar = len(_bar_) _done_ = infostr('Done %-d') + '\n' # ============================================================================= ## @class RunningBar # - RunningBar # @code # with RunningBar () as bar : # for i in xrange(2000) : # ... # bar += 1 # @endcode # With helper function:
def makeWeights( dataset, plots=[], database="weights.db", compare=None, ## comparison function delta=0.001, ## delta for ``mean'' weight variation minmax=0.05, ## delta for ``minmax'' weight variation power=0, ## auto-determination debug=True, ## save intermediate information in DB tag="Reweighting"): assert 0 < delta, "makeWeights(%s): Invalid value for ``delta'' %s" % ( tag, delta) assert 0 < minmax, "makeWeights(%s): Invalid value for ``minmax'' %s" % ( tag, minmax) from ostap.logger.colorized import allright, attention, infostr from ostap.utils.basic import isatty power = power if power >= 1 else len(plots) nplots = len(plots) if 1 < nplots: import math fudge_factor = math.sqrt(1.0 / max(2.0, nplots - 1.0)) delta = delta * fudge_factor minmax = minmax * fudge_factor save_to_db = [] ## number of active plots for reweighting for wplot in plots: what = wplot.what ## variable/function to plot/compare how = wplot.how ## weight and/or additional cuts address = wplot.address ## address in database hdata0 = wplot.data ## original "DATA" object hmc0 = wplot.mc_histo ## original "MC" histogram ww = wplot.w ## relative weight # # normailze the data # hdata = hdata0 if isinstance(hdata, ROOT.TH1): hdata = hdata.density() # ===================================================================== ## make a plot on (MC) data with the weight # ===================================================================== dataset.project(hmc0, what, how) st = hmc0.stat() mnmx = st.minmax() if iszero(mnmx[0]): logger.warning("Reweighting: statistic goes to zero %s/``%s''" % (st, address)) # ===================================================================== ## normalize MC # ===================================================================== hmc = hmc0.density() # ===================================================================== ## calculate the reweighting factor : a bit conservative (?) # this is the only important line # ===================================================================== # try to exploit finer binning if/when possible if isinstance ( hmc , ( ROOT.TH1F , ROOT.TH1D ) ) and \ isinstance ( hdata , ( ROOT.TH1F , ROOT.TH1D ) ) and \ len ( hmc ) >= len( hdata ) : w = (1.0 / hmc) * hdata ## NB! ## elif isinstance ( hmc , ( ROOT.TH2F , ROOT.TH2D ) ) and \ ## isinstance ( hdata , ( ROOT.TH2F , ROOT.TH2D ) ) and \ ## len ( hmc.GetXaxis() ) >= len( hdata.GetXaxis () ) and \ ## len ( hmc.GetYaxis() ) >= len( hdata.GetYaxis () ) : w = ( 1.0 / hmc ) * hdata ## NB! ## elif isinstance ( hmc , ( ROOT.TH3F , ROOT.TH3D ) ) and \ ## isinstance ( hdata , ( ROOT.TH3F , ROOT.TH3D ) ) and \ ## len ( hmc.GetXaxis() ) >= len( hdata.GetXaxis () ) and \ ## len ( hmc.GetYaxis() ) >= len( hdata.GetYaxis () ) and \ ## len ( hmc.GetZaxis() ) >= len( hdata.GetZaxis () ) : w = ( 1.0 / hmc ) * hdata ## NB! else: w = hdata / hmc ## NB! # ===================================================================== ## scale & get the statistics of weights w /= w.stat().mean().value() cnt = w.stat() # mnw, mxw = cnt.minmax() wvar = cnt.rms() / cnt.mean() good1 = wvar.value() <= delta good2 = abs(mxw - mnw) <= minmax good = good1 and good2 ## small variance? # afunc1 = allright if good1 else attention afunc2 = allright if good2 else attention # message = "%s: %24s:" % (tag, address) message += ' ' + 'mean=%12s' % cnt.mean().toString('(%4.2f+-%4.2f)') message += ' ' + afunc2('min/max=%-5.3f/%5.3f' % (cnt.minmax()[0], cnt.minmax()[1])) message += ' ' + afunc1('rms=%s[%%]' % (wvar * 100).toString('(%4.2f+-%4.2f)')) logger.info(message) # ## make decision based on the variance of weights # mnw, mxw = cnt.minmax() if good: ## small variance? message = "%s: No more reweights for %s" % (tag, address) message += ' ' + allright("min/max/rms=%+3.1f/%+3.1f/%3.1f[%%]" % ((mnw - 1) * 100, (mxw - 1) * 100, 100 * wvar)) logger.info(message) del w, hdata, hmc else: save_to_db.append((address, ww, hdata0, hmc0, hdata, hmc, w)) # ===================================================================== ## make a comparison (if needed) # ===================================================================== if compare: compare(hdata0, hmc0, address) ## for single reweighting ## if 1 == nplots : power = 1 ## if power != nplots : # logger.info ( "%s: ``power'' is %g/#%d" % ( tag , power , nplots ) ) active = [p[0] for p in save_to_db] all = [p.address for p in plots] for i, a in enumerate(all): if a in active: if isatty(): all[i] = attention(a) else: all[i] = '*' + a + '*' else: if isatty(): all[i] = allright(a) logger.info("%s: reweights are: %s" % (tag, (', '.join(all)))) ## if len ( active ) != nplots : ## if database and save_to_db : ## power += ( nplots - len ( active ) ) ## logger.info ("%s: ``power'' is changed to %g" % ( tag , power ) ) nactive = len(active) while database and save_to_db: entry = save_to_db.pop() address, ww, hd0, hm0, hd, hm, weight = entry ## eff_exp = 1.0 / power ## eff_exp = 0.95 / ( 1.0 * nactive ) ** 0.5 cnt = weight.stat() mnw, mxw = cnt.minmax() if 0.95 < mnw and mxw < 1.05: eff_exp = 0.75 if 1 < nactive else 1.50 elif 0.90 < mnw and mxw < 1.10: eff_exp = 0.70 if 1 < nactive else 1.30 elif 0.80 < mnw and mxw < 1.20: eff_exp = 0.65 if 1 < nactive else 1.25 elif 0.70 < mnw and mxw < 1.30: eff_exp = 0.60 if 1 < nactive else 1.15 elif 0.50 < mnw and mxw < 1.50: eff_exp = 0.55 if 1 < nactive else 1.10 else: eff_exp = 0.50 if 1 < nactive else 1.0 ## print 'effective exponent is:', eff_exp , address , mnw , mxw , (1.0/mnw)*mnw**eff_exp , (1.0/mxw)*mxw**eff_exp if 1 < nactive and 1 != ww: eff_exp *= ww logger.info("%s: apply ``effective exponent'' of %.3f for ``%s''" % (tag, eff_exp, address)) if 1 != eff_exp and 0 < eff_exp: weight = weight**eff_exp ## print 'WEIGHT stat', eff_exp, weight.stat() ## hmmmm... needed ? yes! #if 1 < power : weight = weight ** ( 1.0 / power ) ## relative importance #if 1 != ww : # logger.info ("%s: apply ``relative importance factor'' of %.3g for ``'%s'" % ( tag , ww , address ) ) # weight = weight ** ww with DBASE.open(database) as db: db[address] = db.get(address, []) + [weight] if debug: addr = address + ':REWEIGHTING' db[addr] = db.get(addr, []) + list(entry[2:]) del hd0, hm0, hd, hm, weight, entry return active
## variables to be used in MC-dataset variables = [ Variable('x', 'x-var', 0, 20), Variable('y', 'y-var', 0, 15), ] selector = SelectorWithVars(variables, '0<x && x<20 && 0<y && y<20', silence=True) mctree.process(selector, silent=True) mcds_ = selector.data ## dataset # ============================================================================= ## start reweighting iterations: for iter in range(1, maxIter + 1): tag = 'Reweighting iteration #%d' % iter logger.info(allright(tag)) with timing(tag + ': prepare MC-dataset:', logger=logger): # ========================================================================= ## 0) The weighter object weighter = Weight(dbname, weightings) # ========================================================================= ## 1a) create new "weighted" mcdataset mcds = mcds_.Clone() with timing(tag + ': add weight to MC-dataset', logger=logger): ## 1b) add "weight" variable to dataset mcds.add_reweighting(weighter, name='weight') if 1 == iter % 10: logger.info((tag + ' MCDATA:\n%s') % mcds)
def the_table(rows, title='', prefix='', alignment=(), wrap_width=-1, indent=wrap_indent): """Format the list of rows as a table (home-made primitive) - Each row is a sequence of column cells. - The first row defines the column headers. >>> table_data = [ ... ( 'Name' , 'Occupation' , 'Note' ) , ... ( 'Alice' , '?' , '---' ) , ... ( 'Bob' , 'unemployed' , '' ) ] >>> t = the_table ( table_data , 'Title' ) >>> print (t) """ ## calculate the number of columns rows = list(rows) nc = 0 for row in rows: nc = max(nc, len(row)) wraps = [] for i, a in zip(range(nc), alignment): if a and isinstance(a, str): al = a.lower() if al in left: pass elif al in right: pass elif al in wrapped: wraps.append(i) ## calculate the maximum width for columns widths = {} for k, row in enumerate(rows): cols = [c for c in row] while len(cols) < nc: cols.append('') for i, c in enumerate(cols): if not i in widths: widths[i] = 1 widths[i] = max(widths[i], len(decolorize(c))) cols = tuple(cols) rows[k] = cols ## play with wrapped columns while wraps: twidth = 1 + len(prefix) for k in widths: if not k in wraps: twidth += widths[k] + 2 twidth += nc + 1 _, w = terminal_size() if w <= twidth: break nw = len(wraps) ww = (w - twidth) - 2 * nw ww, _ = divmod(ww, nw) if 12 < wrap_width and wrap_width < ww: ww = wrap_width if ww < 15: break lw = len(wraps) wraps = [i for i in wraps if ww <= widths[i]] if len(wraps) == lw: break for i in wraps: widths[i] = min(ww, widths[i]) hformats = ["{:^%d}" % widths[c] for c in range(nc)] rformats = [" {:^%d} " % widths[c] for c in range(nc)] for i, a in zip(range(nc), alignment): if a and isinstance(a, str): al = a.lower() if al in left or al in wrapped: hformats[i] = hformats[i].replace('^', '<') rformats[i] = rformats[i].replace('^', '<') elif al in right: hformats[i] = hformats[i].replace('^', '>') rformats[i] = rformats[i].replace('^', '>') if wraps: rows_ = rows rows = [] for row in rows_: cells = [] for i, c in enumerate(row): if i in wraps and wrap_width < len(c): cells.append(textwrap.wrap(indent + c, widths[i])) else: cells.append([c]) nr = 0 for c in cells: nr = max(nr, len(c)) for l in cells: while len(l) < nr: l.insert(0, '') l.append('') for r in range(nr): new_row = [] for i, c in enumerate(cells): lc = len(c) if r < lc: new_row.append(c[r]) else: new_row.append('') rows.append(new_row) totwidth = 0 for c in widths: totwidth += widths[c] totwidth += (nc - 1) * 3 if totwidth < len(title): delta = 1 + (len(title) - totwidth) // nc for c in widths: widths[c] += delta seps = ['-' * (widths[c] + 2) for c in range(nc)] sepline = '+' + "+".join(seps) + '+' table = [] if title: sline = '+' + '-' * (len(sepline) - 2) + '+' tfmt = "{:^%d}" % (len(sepline) - 4) t = '| ' + allright(tfmt.format(decolorize(title))) + ' |' table.append(sline) table.append(t) table.append(sepline) for i, row in enumerate(rows): cols = [c for c in row] while len(cols) < nc: cols.append('') if 0 == i: table.append('| ' + ' | '.join([ infostr(f.format(decolorize(i))) for (f, i) in zip(hformats, cols) ]) + ' |') table.append(sepline) else: table.append('|' + '|'.join( [f.format(decolorize(i)) for (f, i) in zip(rformats, cols)]) + '|') table.append(sepline) return prefix + ('\n' + prefix).join(table) if prefix else '\n'.join(table)
def _mn_table_ ( self , title = '' , prefix = '' ) : """Build parameter table from (T)Minuit """ header = ( 'Parameter' , '' , 'Value' ) rows = [] from ostap.fitting.utils import fit_status , cov_qual from ostap.logger.utils import pretty_float, pretty_ve, pretty_2ve status = self.GetStatus() if status : status = fit_status ( status ) row = ' Status' , '' , status rows.append ( row ) stat = _mn_stat_ ( self ) istat = stat.pop ( 'ISTAT' , None ) if not istat is None : cq = '' if -1 == istat : cq = cov_qual ( istat ) elif 3 == istat : cq = allright ( cov_qual ( istat ) ) elif istat in ( 0 , 1 , 2 ) : cq = attentiont ( cov_qual ( istat ) ) else : cq = cov_qual ( istat ) row = 'Covariance matrix quality' , '' , cq rows.append ( row ) fmin = stat.pop ( 'FMIN' , None ) if not fmin is None : s , n = pretty_float ( fmin ) if n : n = '[10^%+d]' % n else : n = '' row = 'Minimized FCN value' , n , s rows.append ( row ) fedm = stat.pop ( 'FEDM' , None ) if not fedm is None : s , n = pretty_float ( fedm ) if n : n = '[10^%+d]' % n else : n = '' row = 'Estimated distance to minimum' , n , s rows.append ( row ) errdef = stat.pop ( 'ERRDEF' , None ) ## needed ? has_limits = False has_minos = False val = ctypes.c_double ( 0 ) err = ctypes.c_double ( 0 ) low = ctypes.c_double ( 0 ) high = ctypes.c_double ( 0 ) idx = ctypes.c_int ( 0 ) dct_pars = {} ## loop over all parameters for i in self : name = ROOT.TString() self.mnpout ( i , name , val , err , low , high , idx ) if not 0 <= idx.value : continue dct = {} dct [ 'name' ] = '%-2d: %s' % ( i , str ( name ).strip() ) dct [ 'value'] = val.value if low.value < high.value : dct [ 'low' ] = low .value dct [ 'high'] = high.value has_limits = True if 0 <= err.value : dct [ 'error' ] = err.value mn_plus , mn_minus = _mn_minerr_ ( self , str ( name ) ) if 0 < mn_plus or mn_minus < 0 : dct [ 'minos+' ] = mn_plus dct [ 'minos-' ] = mn_minus has_minos = True dct_pars[ i ] = dct if has_minos : ## some parameters have MINOS errors, add columns header = ( header ) + ( 'neg-minos' , 'pos-minos' ) rows = [ r + ( '','' ) for r in rows ] if has_limits : ## some parameters have LIMITS, add columns header = ( header ) + ( 'low limit' , 'high limit' ) rows = [ r + ( '','' ) for r in rows ] for p in dct_pars : pdict = dct_pars [ p ] row = [] row.append ( pdict.pop ( 'name' ) ) value = pdict.pop ( 'value' ) error = pdict.pop ( 'error', None ) if error is None : s , n = pretty_float ( value ) s = "%s(fixed)" % s else : s , n = pretty_ve ( VE ( value , error * error ) ) if n : row.append ( '[10^%+d]' % n ) else : row.append ( '' ) row.append ( s ) if has_minos : mn_plus = pdict.pop ( 'minos+' , None ) mn_minus = pdict.pop ( 'minos-' , None ) if mn_plus is None : mn_plus = '' else : mn_plus = '%8f' % ( mn_plus * 10** n ) if mn_minus is None : mn_minus = '' else : mn_minus = '%8f' % ( mn_minus * 10** n ) row.append ( mn_minus ) row.append ( mn_plus ) if has_limits : low = pdict.pop ( 'low' , None ) high = pdict.pop ( 'high' , None ) if low is None : low = '' else : low = '%8f' % ( low * 10 ** n ) if high is None : high = '' else : high = '%8f' % ( high * 10 ** n ) row.append ( low ) row.append ( high ) row = tuple ( row ) rows.append ( row ) rows = [ header ] + rows from ostap.logger.table import table return table ( rows , title = title , prefix = prefix )
# ============================================================================= ## variables to be used in MC-dataset variables = [ Variable('x', 'x-var', 0, 20), Variable('y', 'y-var', 0, 15), ] selector = SelectorWithVars(variables, '0<x && x<20 && 0<y && y<20', silence=True) mctree.process(selector, silent=True) mcds_ = selector.data ## dataset # ============================================================================= ## start reweighting iterations: for iter in range(1, maxIter + 1): logger.info(allright('Reweighting iteration %d ' % iter)) with timing('Prepare MC-dataset:', logger=logger): # ========================================================================= ## 0) The weighter object weighter = Weight(dbname, weightings) # ========================================================================= ## 1a) create new "weighted" mcdataset mcds = mcds_.Clone() with timing('Add weight to MC-dataset', logger=logger): ## 1b) add "weight" variable to dataset mcds.add_reweighting(weighter, name='weight') logger.info('MCDATA:\n%s' % mcds)
def _rfr_table_(r, title='', prefix=''): """ print RooFitResult as a table >>> result = ... >>> result.table() """ from ostap.fitting.utils import fit_status, cov_qual rows = [] if r.status(): row = attention(' Status'), '', attention(fit_status(r.status())), '' rows.append(row) s, n = pretty_float(r.minNll()) if n: n = '[10^%+d]' % n else: n = '' rows.append(("Minimized FCN/NLL value", n, ' ' + s, '')) s, n = pretty_float(r.edm()) if n: n = '[10^%+d]' % n else: n = '' rows.append(('Estimated distance to minimum', n, ' ' + s, '')) cq = r.covQual() cn = '' if -1 == cq: cn = cov_qual(cq) elif 3 == cq: cn = allright(cov_qual(cq)) elif cq in (0, 1, 2): cn = attention(cov_qual(cq)) else: cn = cov_qual(cq) rows.append(('Covariance matrix quality', '', ' ' + cn, '')) for i in range(r.numStatusHistory()): label = r.statusLabelHistory(i) code = r.statusCodeHistory(i) row = 'Status: %s ' % label, '', '%d' % code if not code in (0, -1): row = attention(row[0]), row[1], ' ' + attention(row[2]), '' else: row = row[0], row[1], ' ' + allright(row[2]), '' rows.append(row) nbadnll = r.numInvalidNLL() if 0 < nbadnll: rows.append(('Invalid FCN/NLL evaluations', '', ' %d' % nbadnll, '')) rows = [('', 'Unit', 'Value', 'Global/max correlation')] + rows pars_all = r.params(float_only=False) pars_float = r.params(float_only=True) ## constant/fix parameters crows = [] for p in pars_all: if p in pars_float: continue v, a = pars_all[p] s, n = pretty_float(v.value()) if n: n = '[10^%+d]' % n else: n = '' row = p, n, ' ' + s, '' crows.append(row) ## floating parameters frows = [] for p in pars_float: v, a = pars_float[p] if not a.hasAsymError(): s, n = pretty_ve(v) else: s, n = pretty_2ve(a.getVal(), a.getAsymErrorHi(), a.getAsymErrorLo()) if n: n = '[10^%+d]' % n else: n = '' cc = 'Not available' if 0 <= cq: mxr, mxv = r.max_cor(p) gc = r.globalCorr(p) cc = '%+5.3f/(%+5.3f,%s)' % (gc, mxr, mxv) if 0.95 < abs(gc) or 0.95 < abs(mxr): cc = attention(cc) row = p, n, s, cc frows.append(row) crows.sort() frows.sort() all = rows + crows + frows import ostap.logger.table as T all = T.align_column(all, 0, 'left') all = T.align_column(all, 1, 'left') all = T.align_column(all, 2, 'left') all = T.align_column(all, 3, 'left') for l in range(len(rows), len(all)): line = all[l] line = list(line) line[0] = allright(line[0]) all[l] = tuple(line) if title: return T.table(all, title=title, prefix=prefix) else: return T.table(all, title=r.GetTitle(), prefix=prefix)