def examples_dataframe(fpath: str) -> DataFrame:
""" Loads protobuf into a DataFrame """
d: dict = {'idx': [], 'data': [], 'isin': []}
print(f"Reading {fpath}")
def _addval(v, isin: int, idx: int) -> None:
nonlocal d
d['idx'].append(idx)
if isinstance(v,IVal) and \
isinstance(v.val, int):
d['data'].append(v.val)
d['isin'].append(isin)
else:
d['data'].append(None)
d['isin'].append(isin)
with open(fpath, 'rb') as f:
_next = fd2examples(f)
try:
idx = 0
while True:
example = _next()
for k, v in example.inp.items():
_addval(v, isin=1, idx=idx)
_addval(example.out, isin=0, idx=idx)
idx += 1
except KeyboardInterrupt:
raise
except Exception as e:
print(e)
pass
df = DataFrame(d)
print(f"Number of examples: {idx}")
return df
def _make(b: Build):
build_setoutpaths(b, 1)
rref = run_dataset2(what=2,
maxitems=mklens(b).N.val,
index=index,
interactive=False)
assert isfile(mklens(rref).ref_df.examples.syspath)
system(
f"cp {mklens(rref).ref_df.ref_data.inputs.syspath} {mklens(b).out_inputs.syspath}"
)
df = pd.read_csv(mklens(rref).df.syspath)
makedirs(mklens(b).out_barplots.syspath)
allowed = stabilize(
df,
npoints=mklens(b).npoints.val,
path_barplot=lambda i: join(
mklens(b).out_barplots.syspath, f'_plot_{i:03d}.png'),
path_traceplot=mklens(b).out_traceplot.syspath,
min_allow_size=mklens(b).min_allow_size.val)
with open(mklens(rref).ref_df.examples.syspath, 'rb') as f:
with open(mklens(b).out_examples.syspath, 'wb') as f_o:
_read = fd2examples(f)
_write = examples2fd(f_o)
try:
idx = 0
while True:
example = _read()
if idx in allowed:
_write(example)
idx += 1
except KeyboardInterrupt:
raise
except Exception as e:
print(e)
pass
def test_example():
e=Example(inp=mkmap({Ref('a'):ival(0)}),
expr=ap(ref('a'),ref('b')),
out=IVal(33))
with open('/tmp/binfile','wb') as f:
add=examples2fd(f)
add(e)
add(e)
with open('/tmp/binfile','rb') as f:
_next=fd2examples(f)
n=_next()
assert n==e
n=_next()
assert n==e
def loadsection(si:int)->List[Example]:
print(f"Loading section {si}")
acc:list=[]
try:
with open(mklens(load3(si,False)).out_examples.syspath, 'rb') as ef:
_next=fd2examples(ef)
e=_next()
print(type(e))
s=pickle_dumps(e, protocol=HIGHEST_PROTOCOL)
print(s)
# while True:
# acc.append(_next())
except KeyboardInterrupt:
raise
except LookupError:
pass
return acc[:10]