def format_results(self, results): ''' Show query search results ''' expe = self.expe startchar, endchar = colored('splitme', 'bold').split('splitme') for rank, hit in enumerate(results): shortpath = hit['shortpath'] fullpath = hit['fullpath'] score = hit.score print('%d:' % (rank + 1), colored(shortpath, 'green'), '%.2f' % score) if 'title' in hit: print('Title: %s: ' % (colored(colored(hit['title'], 'bold'), 'red'))) if expe.highlight: text = extract_pdf(fullpath, page_limit=42) if text: fragments = hit.highlights('content', text=text, top=expe.number_highlight) fragments = ' '.join(fragments.split()) fragments = textwrap.fill(' ' * 4 + fragments, width=84, subsequent_indent=' ' * 4) print(fragments) print()
def highlight_table(self, array, highlight_dim=1): hack_float = np.vectorize(lambda x: '{:.3f}'.format(float(x))) table = np.char.array(hack_float(array), itemsize=42) # vectorize for i, col in enumerate(array.argmax(1)): table[i, col] = colored(table[i, col], 'bold') for i, col in enumerate(array.argmin(1)): table[i, col] = colored(table[i, col], 'magenta') return table
def pvalue(self): ''' Compute Goodness of fit statistics ''' expe = self.expe frontend = self.frontend data = frontend.data d, dc = degree_hist(adj_to_degree(data), filter_zeros=True) gof = gofit(d, dc) if not hasattr(self.gramexp, 'Table'): corpuses = self.specname(self.gramexp.get_set('corpus')) Meas = ['pvalue', 'alpha', 'x_min', 'n_tail'] Table = np.empty((len(corpuses), len(Meas))) Table = np.column_stack((corpuses, Table)) self.gramexp.Table = Table self.gramexp.Meas = Meas else: Table = self.gramexp.Table Meas = self.gramexp.Meas for i, v in enumerate(Meas): Table[self.corpus_pos, i + 1] = gof[v] if self._it == self.expe_size - 1: tablefmt = 'latex' print(colored('\nPvalue Table:', 'green')) print( self.tabulate(Table, headers=Meas, tablefmt=tablefmt, floatfmt='.3f'))
def format_token(self, text, token, replace=False): # Use the get_text function to get the text corresponding to the # token tokentext = ws.highlight.get_text(text, token, replace) # Return the text as you want it to appear in the highlighted # string return "%s" % colored(tokentext, 'bold')
def highlight_table(self, array, highlight_dim=1, fmt=None, **kwargs): hack_float = np.vectorize(lambda x: '{:.3f}'.format(float(x))) table = np.char.array(hack_float(array), itemsize=42) if fmt == 'latex': _wrap = lambda x: '\\textbf{%s}' % x # @debug @perso if 'wsim' in kwargs.get('z', ''): _fun = array.argmin else: _fun = array.argmax for i, col in enumerate(_fun(1)): table[i, col] = _wrap(table[i, col]) else: for i, col in enumerate(array.argmax(1)): table[i, col] = colored(table[i, col], 'bold') for i, col in enumerate(array.argmin(1)): table[i, col] = colored(table[i, col], 'magenta') return table
def log_tables(self, **kwargs): for _title, _table in self.gramexp._tables.items(): arr = _table.array.astype(str) table = np.column_stack((_table.column, arr)) Table = tabulate(table, headers=_table.headers, tablefmt=_table.tablefmt, floatfmt=_table.floatfmt) self.log.info(colored('\n%s Table:' % (_title), 'green')) print(Table)
def stats(self, fmt='simple'): ''' Show data stats ''' expe = self.expe frontend = FrontendManager.load(expe) if self.is_first_expe(): # Warning order sensitive @deprecated Table. #corpuses = self.specname(self.gramexp.get_set('corpus')) corpuses = self.specname(self.gramexp.get_list('corpus')) Meas = [ 'num_nodes', 'num_edges', 'density', 'is_directed', 'modularity', 'diameter', 'clustering_coefficient', 'net_type', 'feat_len' ] Meas_ = [ 'nodes', 'edges', 'density', 'directed', 'modularity', 'diameter', 'cluster-coef', 'weights', 'feat-len' ] Table = np.zeros((len(corpuses), len(Meas))) * np.nan Table = np.column_stack((corpuses, Table)) self.D.Table = Table self.D.Meas = Meas self.D.Meas_ = Meas_ Table = self.D.Table Meas = self.D.Meas for i, v in enumerate(Meas): if frontend.data is None: Table[self.corpus_pos, 1:] = np.nan break value = getattr(frontend, v)() value = value if value is not None else np.nan Table[self.corpus_pos, i + 1] = value if hasattr(frontend, '_check'): frontend._check() if self.is_last_expe(): tablefmt = 'latex' if fmt == 'tex' else fmt precision = '.5f' print(colored('\nStats Table :', 'green')) Meas_ = self.D.Meas_ print( self.tabulate(Table, headers=Meas_, tablefmt=tablefmt, floatfmt=precision))
def _future_stats(self): ''' Show data stats ''' expe = self.expe corpuses = self.specname(self.gramexp.get_list('corpus')) if not corpuses: corpuses = [ 'manufacturing', 'fb_uc', 'blogs', 'emaileu', 'propro', 'euroroad', 'generator7', 'generator12', 'generator10', 'generator4' ] Meas = ['num_nodes', 'num_edges', 'density'] Meas += [ 'is_symmetric', 'modularity', 'clustering_coefficient', 'net_type', 'feat_len' ] Table = np.zeros((len(corpuses), len(Meas))) * np.nan Table = np.column_stack((corpuses, Table)) for _corpus_cpt, corpus_name in enumerate(corpuses): expe.update(corpus=corpus_name) # @Heeere: big problme of data management: # if data_type is set heren input_path os wrong !? # how to corretcly manage this, think about it. # pmk looks in pmk-temp here. expe.update(_data_type='networks') frontend = FrontendManager.load(expe) for i, v in enumerate(Meas): if frontend.data is None: Table[self.corpus_pos, 1:] = 'none' break Table[self.corpus_pos, i + 1] = getattr(frontend, v)() tablefmt = 'simple' # 'latex' print(colored('\nStats Table :', 'green')) print( self.tabulate(Table, headers=Meas, tablefmt=tablefmt, floatfmt='.3f'))
def stats(self): ''' Show data stats ''' expe = self.expe frontend = FrontendManager.load(expe) try: #@ugly debug Table = self.gramexp.Table Meas = self.gramexp.Meas except AttributeError: # Warning order sensitive @deprecated Table. #corpuses = self.specname(self.gramexp.get_set('corpus')) corpuses = self.specname(self.gramexp.get_list('corpus')) Meas = ['num_nodes', 'num_edges', 'density'] Meas += [ 'is_symmetric', 'modularity', 'clustering_coefficient', 'net_type', 'feat_len' ] Table = np.zeros((len(corpuses), len(Meas))) * np.nan Table = np.column_stack((corpuses, Table)) self.gramexp.Table = Table self.gramexp.Meas = Meas #print (frontend.get_data_prop()) for i, v in enumerate(Meas): if frontend.data is None: Table[self.corpus_pos, 1:] = np.nan break value = getattr(frontend, v)() value = value if value is not None else np.nan Table[self.corpus_pos, i + 1] = value if self._it == self.expe_size - 1: tablefmt = 'simple' # 'latex' print(colored('\nStats Table :', 'green')) print( self.tabulate(Table, headers=Meas, tablefmt=tablefmt, floatfmt='.3f'))
def __init__(self, pt, expe, expdesign, gramexp): ''' Sandbox class for scripts/actions. Parameters ---------- pt: int Positional indicator for current run expe: ExpSpace Current spec (or equivalently accessible with `self.s` expdesign: ExpDesign Current design class gramexp: Global pmk object Methods ------- log_expe: none overwrite the header message before each run. is_first_expe: none return true if the current run is the first. is_last_expe: none return true if the current run is the last. get_expe_len: int the total number of expe get_expe_it: int the current expe iteration load_frontend: none load the data frontend for the current expe. load_model: none the model for the current expe. load_data: matrix load data based on extension Decorator -------- plot: none plot control on expe table: none table control on expe ''' self._expdesign = expdesign # Global self.expe_size = len(gramexp) self.gramexp = gramexp self.D = self.gramexp.D # Global conteneur # Local self.pt = pt self.expe = expe self.spec = expe self.s = expe # Plot utils from pymake.plot import _linestyle, _markers, _colors self.linestyles = _linestyle.copy() self.markers = _markers.copy() self.colors = _colors.copy() # to exploit / Vizu self._it = pt['expe'] self.corpus_pos = pt.get('corpus') self.model_pos = pt.get('model') self.output_path = self.expe['_output_path'] self.input_path = self.expe['_input_path'] if expe.get('_expe_silent'): self.log_silent() else: self.log.info('-' * 10) self.log.info( ''.join([colored('Expe %d/%d', 'red'), ' : ', self.log_expe()]) % ( self._it + 1, self.expe_size, )) self.log.info('-' * 10) self._set_measures()
def wrapper(*args, **kwargs): self = args[0] discr_args = [] # discriminant keys (to distinguish filename) if len(args) == 1: x, y, z = 'corpus', 'model', '_entropy' else: x, y, z = args[1].split(':') if len(args) > 2: discr_args = args[2].split('/') if discr_args: groups = discr_args # or None if args not in expe (tex option...) ggroup = self._file_part( [self.expe.get(g) for g in groups], sep='-') or None else: groups = None ggroup = None _z = z.split('-') if not hasattr(self.gramexp, '_tables'): tables = dict() if groups: gset = product(*filter( None, [self.gramexp.get_set(g) for g in groups])) else: gset = [None] for g in gset: gg = '-'.join(map(str, g)) if g else None tables[gg] = ExpSpace() array, floc = self.gramexp.get_array_loc(x, y, _z) tables[gg].array = array tables[gg].floc = floc self.gramexp._tables = tables frame = self.gramexp._tables[ggroup] array = frame.array floc = frame.floc for z in _z: kernel = fun(self, array, floc, x, y, z, **kwargs) if self._it == self.expe_size - 1: for ggroup in list(self.gramexp._tables): _table = self.gramexp._tables.pop(ggroup) array = _table.array for zpos, z in enumerate(_z): # Format table #tablefmt = 'latex' # 'simple' tablefmt = 'latex' if 'tex' in discr_args else 'simple' Meas = self.specname(self.gramexp.get_set(y)) arr = self.highlight_table(array[:, :, zpos]) table = np.column_stack( (self.specname(self.gramexp.get_set(x)), arr)) Table = tabulate(table, headers=Meas, tablefmt=tablefmt, floatfmt='.3f') gg = z + '-' + ggroup if ggroup else z self.gramexp._tables[gg] = ExpSpace({ 'table': Table, 'base': '_'.join((fun.__name__, str(self.expe[x]), str(self.expe[y]))), 'args': discr_args, #'args':self.gramexp.get_nounique_keys(x, y), }) print(colored('\n%s Table:' % (gg), 'green')) print(Table) if self.expe._write: tablefmt_ext = dict(simple='md', latex='tex') self.write_frames(self.gramexp._tables, ext=tablefmt_ext[tablefmt]) return kernel
def __init__(self, pt, expe, expdesign, gramexp): ''' Sandbox class for scripts/actions. Parameters ---------- pt: int Positional indicator for current run expe: ExpSpace Current spec expdesign: ExpDesign Current design class gramexp: Global pmk object Methods ------- log_expe: overwrite the header message before each run. is_first_expe: return true if the current run is the first. is_last_expe: return true if the current run is the last. load_frontend: load the data frontend for the current expe. load_model: the model for the current expe. Decorator -------- plot: plot control on expe table: table control on expe ''' self._expdesign = expdesign # @debug this, I dont know whyiam in lib/package sometimes, annoying ! os.chdir(os.getenv('PWD')) # Global self.expe_size = len(gramexp) self.gramexp = gramexp self.D = self.gramexp.D # Global conteneur # Local self.pt = pt self.expe = expe # Plot utils from pymake.plot import _linestyle, _markers, _colors self.linestyles = _linestyle.copy() self.markers = _markers.copy() self.colors = _colors.copy() # to exploit / Vizu self._it = pt['expe'] self.corpus_pos = pt.get('corpus') self.model_pos = pt.get('model') self.output_path = self.expe['_output_path'] self.input_path = self.expe['_input_path'] if expe.get('_expe_silent'): self.log_silent() else: self.log.info('-' * 10) self.log.info( ''.join([colored('Expe %d/%d', 'red'), ' : ', self.log_expe()]) % ( self._it + 1, self.expe_size, )) self.log.info('-' * 10)