def format_results(self, results):
''' Show query search results '''
expe = self.expe
startchar, endchar = colored('splitme', 'bold').split('splitme')
for rank, hit in enumerate(results):
shortpath = hit['shortpath']
fullpath = hit['fullpath']
score = hit.score
print('%d:' % (rank + 1), colored(shortpath, 'green'),
'%.2f' % score)
if 'title' in hit:
print('Title: %s: ' %
(colored(colored(hit['title'], 'bold'), 'red')))
if expe.highlight:
text = extract_pdf(fullpath, page_limit=42)
if text:
fragments = hit.highlights('content',
text=text,
top=expe.number_highlight)
fragments = ' '.join(fragments.split())
fragments = textwrap.fill(' ' * 4 + fragments,
width=84,
subsequent_indent=' ' * 4)
print(fragments)
print()
def highlight_table(self, array, highlight_dim=1):
hack_float = np.vectorize(lambda x: '{:.3f}'.format(float(x)))
table = np.char.array(hack_float(array), itemsize=42)
# vectorize
for i, col in enumerate(array.argmax(1)):
table[i, col] = colored(table[i, col], 'bold')
for i, col in enumerate(array.argmin(1)):
table[i, col] = colored(table[i, col], 'magenta')
return table
def pvalue(self):
''' Compute Goodness of fit statistics '''
expe = self.expe
frontend = self.frontend
data = frontend.data
d, dc = degree_hist(adj_to_degree(data), filter_zeros=True)
gof = gofit(d, dc)
if not hasattr(self.gramexp, 'Table'):
corpuses = self.specname(self.gramexp.get_set('corpus'))
Meas = ['pvalue', 'alpha', 'x_min', 'n_tail']
Table = np.empty((len(corpuses), len(Meas)))
Table = np.column_stack((corpuses, Table))
self.gramexp.Table = Table
self.gramexp.Meas = Meas
else:
Table = self.gramexp.Table
Meas = self.gramexp.Meas
for i, v in enumerate(Meas):
Table[self.corpus_pos, i + 1] = gof[v]
if self._it == self.expe_size - 1:
tablefmt = 'latex'
print(colored('\nPvalue Table:', 'green'))
print(
self.tabulate(Table,
headers=Meas,
tablefmt=tablefmt,
floatfmt='.3f'))
def format_token(self, text, token, replace=False):
# Use the get_text function to get the text corresponding to the
# token
tokentext = ws.highlight.get_text(text, token, replace)
# Return the text as you want it to appear in the highlighted
# string
return "%s" % colored(tokentext, 'bold')
def highlight_table(self, array, highlight_dim=1, fmt=None, **kwargs):
hack_float = np.vectorize(lambda x: '{:.3f}'.format(float(x)))
table = np.char.array(hack_float(array), itemsize=42)
if fmt == 'latex':
_wrap = lambda x: '\\textbf{%s}' % x
# @debug @perso
if 'wsim' in kwargs.get('z', ''):
_fun = array.argmin
else:
_fun = array.argmax
for i, col in enumerate(_fun(1)):
table[i, col] = _wrap(table[i, col])
else:
for i, col in enumerate(array.argmax(1)):
table[i, col] = colored(table[i, col], 'bold')
for i, col in enumerate(array.argmin(1)):
table[i, col] = colored(table[i, col], 'magenta')
return table
def log_tables(self, **kwargs):
for _title, _table in self.gramexp._tables.items():
arr = _table.array.astype(str)
table = np.column_stack((_table.column, arr))
Table = tabulate(table,
headers=_table.headers,
tablefmt=_table.tablefmt,
floatfmt=_table.floatfmt)
self.log.info(colored('\n%s Table:' % (_title), 'green'))
print(Table)
def stats(self, fmt='simple'):
''' Show data stats '''
expe = self.expe
frontend = FrontendManager.load(expe)
if self.is_first_expe():
# Warning order sensitive @deprecated Table.
#corpuses = self.specname(self.gramexp.get_set('corpus'))
corpuses = self.specname(self.gramexp.get_list('corpus'))
Meas = [
'num_nodes', 'num_edges', 'density', 'is_directed',
'modularity', 'diameter', 'clustering_coefficient', 'net_type',
'feat_len'
]
Meas_ = [
'nodes', 'edges', 'density', 'directed', 'modularity',
'diameter', 'cluster-coef', 'weights', 'feat-len'
]
Table = np.zeros((len(corpuses), len(Meas))) * np.nan
Table = np.column_stack((corpuses, Table))
self.D.Table = Table
self.D.Meas = Meas
self.D.Meas_ = Meas_
Table = self.D.Table
Meas = self.D.Meas
for i, v in enumerate(Meas):
if frontend.data is None:
Table[self.corpus_pos, 1:] = np.nan
break
value = getattr(frontend, v)()
value = value if value is not None else np.nan
Table[self.corpus_pos, i + 1] = value
if hasattr(frontend, '_check'):
frontend._check()
if self.is_last_expe():
tablefmt = 'latex' if fmt == 'tex' else fmt
precision = '.5f'
print(colored('\nStats Table :', 'green'))
Meas_ = self.D.Meas_
print(
self.tabulate(Table,
headers=Meas_,
tablefmt=tablefmt,
floatfmt=precision))
def _future_stats(self):
''' Show data stats '''
expe = self.expe
corpuses = self.specname(self.gramexp.get_list('corpus'))
if not corpuses:
corpuses = [
'manufacturing', 'fb_uc', 'blogs', 'emaileu', 'propro',
'euroroad', 'generator7', 'generator12', 'generator10',
'generator4'
]
Meas = ['num_nodes', 'num_edges', 'density']
Meas += [
'is_symmetric', 'modularity', 'clustering_coefficient', 'net_type',
'feat_len'
]
Table = np.zeros((len(corpuses), len(Meas))) * np.nan
Table = np.column_stack((corpuses, Table))
for _corpus_cpt, corpus_name in enumerate(corpuses):
expe.update(corpus=corpus_name)
# @Heeere: big problme of data management:
# if data_type is set heren input_path os wrong !?
# how to corretcly manage this, think about it.
# pmk looks in pmk-temp here.
expe.update(_data_type='networks')
frontend = FrontendManager.load(expe)
for i, v in enumerate(Meas):
if frontend.data is None:
Table[self.corpus_pos, 1:] = 'none'
break
Table[self.corpus_pos, i + 1] = getattr(frontend, v)()
tablefmt = 'simple' # 'latex'
print(colored('\nStats Table :', 'green'))
print(
self.tabulate(Table,
headers=Meas,
tablefmt=tablefmt,
floatfmt='.3f'))
def stats(self):
''' Show data stats '''
expe = self.expe
frontend = FrontendManager.load(expe)
try:
#@ugly debug
Table = self.gramexp.Table
Meas = self.gramexp.Meas
except AttributeError:
# Warning order sensitive @deprecated Table.
#corpuses = self.specname(self.gramexp.get_set('corpus'))
corpuses = self.specname(self.gramexp.get_list('corpus'))
Meas = ['num_nodes', 'num_edges', 'density']
Meas += [
'is_symmetric', 'modularity', 'clustering_coefficient',
'net_type', 'feat_len'
]
Table = np.zeros((len(corpuses), len(Meas))) * np.nan
Table = np.column_stack((corpuses, Table))
self.gramexp.Table = Table
self.gramexp.Meas = Meas
#print (frontend.get_data_prop())
for i, v in enumerate(Meas):
if frontend.data is None:
Table[self.corpus_pos, 1:] = np.nan
break
value = getattr(frontend, v)()
value = value if value is not None else np.nan
Table[self.corpus_pos, i + 1] = value
if self._it == self.expe_size - 1:
tablefmt = 'simple' # 'latex'
print(colored('\nStats Table :', 'green'))
print(
self.tabulate(Table,
headers=Meas,
tablefmt=tablefmt,
floatfmt='.3f'))
def __init__(self, pt, expe, expdesign, gramexp):
''' Sandbox class for scripts/actions.
Parameters
----------
pt: int
Positional indicator for current run
expe: ExpSpace
Current spec (or equivalently accessible with `self.s`
expdesign: ExpDesign
Current design class
gramexp: Global pmk object
Methods
-------
log_expe: none
overwrite the header message before each run.
is_first_expe: none
return true if the current run is the first.
is_last_expe: none
return true if the current run is the last.
get_expe_len: int
the total number of expe
get_expe_it: int
the current expe iteration
load_frontend: none
load the data frontend for the current expe.
load_model: none
the model for the current expe.
load_data: matrix
load data based on extension
Decorator
--------
plot: none
plot control on expe
table: none
table control on expe
'''
self._expdesign = expdesign
# Global
self.expe_size = len(gramexp)
self.gramexp = gramexp
self.D = self.gramexp.D # Global conteneur
# Local
self.pt = pt
self.expe = expe
self.spec = expe
self.s = expe
# Plot utils
from pymake.plot import _linestyle, _markers, _colors
self.linestyles = _linestyle.copy()
self.markers = _markers.copy()
self.colors = _colors.copy()
# to exploit / Vizu
self._it = pt['expe']
self.corpus_pos = pt.get('corpus')
self.model_pos = pt.get('model')
self.output_path = self.expe['_output_path']
self.input_path = self.expe['_input_path']
if expe.get('_expe_silent'):
self.log_silent()
else:
self.log.info('-' * 10)
self.log.info(
''.join([colored('Expe %d/%d', 'red'), ' : ',
self.log_expe()]) % (
self._it + 1,
self.expe_size,
))
self.log.info('-' * 10)
self._set_measures()
def wrapper(*args, **kwargs):
self = args[0]
discr_args = [] # discriminant keys (to distinguish filename)
if len(args) == 1:
x, y, z = 'corpus', 'model', '_entropy'
else:
x, y, z = args[1].split(':')
if len(args) > 2:
discr_args = args[2].split('/')
if discr_args:
groups = discr_args
# or None if args not in expe (tex option...)
ggroup = self._file_part(
[self.expe.get(g) for g in groups], sep='-') or None
else:
groups = None
ggroup = None
_z = z.split('-')
if not hasattr(self.gramexp, '_tables'):
tables = dict()
if groups:
gset = product(*filter(
None, [self.gramexp.get_set(g) for g in groups]))
else:
gset = [None]
for g in gset:
gg = '-'.join(map(str, g)) if g else None
tables[gg] = ExpSpace()
array, floc = self.gramexp.get_array_loc(x, y, _z)
tables[gg].array = array
tables[gg].floc = floc
self.gramexp._tables = tables
frame = self.gramexp._tables[ggroup]
array = frame.array
floc = frame.floc
for z in _z:
kernel = fun(self, array, floc, x, y, z, **kwargs)
if self._it == self.expe_size - 1:
for ggroup in list(self.gramexp._tables):
_table = self.gramexp._tables.pop(ggroup)
array = _table.array
for zpos, z in enumerate(_z):
# Format table
#tablefmt = 'latex' # 'simple'
tablefmt = 'latex' if 'tex' in discr_args else 'simple'
Meas = self.specname(self.gramexp.get_set(y))
arr = self.highlight_table(array[:, :, zpos])
table = np.column_stack(
(self.specname(self.gramexp.get_set(x)), arr))
Table = tabulate(table,
headers=Meas,
tablefmt=tablefmt,
floatfmt='.3f')
gg = z + '-' + ggroup if ggroup else z
self.gramexp._tables[gg] = ExpSpace({
'table':
Table,
'base':
'_'.join((fun.__name__, str(self.expe[x]),
str(self.expe[y]))),
'args':
discr_args,
#'args':self.gramexp.get_nounique_keys(x, y),
})
print(colored('\n%s Table:' % (gg), 'green'))
print(Table)
if self.expe._write:
tablefmt_ext = dict(simple='md', latex='tex')
self.write_frames(self.gramexp._tables,
ext=tablefmt_ext[tablefmt])
return kernel
def __init__(self, pt, expe, expdesign, gramexp):
''' Sandbox class for scripts/actions.
Parameters
----------
pt: int
Positional indicator for current run
expe: ExpSpace
Current spec
expdesign: ExpDesign
Current design class
gramexp: Global pmk object
Methods
-------
log_expe:
overwrite the header message before each run.
is_first_expe:
return true if the current run is the first.
is_last_expe:
return true if the current run is the last.
load_frontend:
load the data frontend for the current expe.
load_model:
the model for the current expe.
Decorator
--------
plot:
plot control on expe
table:
table control on expe
'''
self._expdesign = expdesign
# @debug this, I dont know whyiam in lib/package sometimes, annoying !
os.chdir(os.getenv('PWD'))
# Global
self.expe_size = len(gramexp)
self.gramexp = gramexp
self.D = self.gramexp.D # Global conteneur
# Local
self.pt = pt
self.expe = expe
# Plot utils
from pymake.plot import _linestyle, _markers, _colors
self.linestyles = _linestyle.copy()
self.markers = _markers.copy()
self.colors = _colors.copy()
# to exploit / Vizu
self._it = pt['expe']
self.corpus_pos = pt.get('corpus')
self.model_pos = pt.get('model')
self.output_path = self.expe['_output_path']
self.input_path = self.expe['_input_path']
if expe.get('_expe_silent'):
self.log_silent()
else:
self.log.info('-' * 10)
self.log.info(
''.join([colored('Expe %d/%d', 'red'), ' : ',
self.log_expe()]) % (
self._it + 1,
self.expe_size,
))
self.log.info('-' * 10)
