def _make(b: Build):
build_setoutpaths(b, 1)
df = pd.read_csv(mklens(b).df.syspath)
vis_bars(df,
plot_fpath=mklens(b).out_plot.syspath,
async_plot=None,
plot_title=mklens(b).title.val)
def _make(b: Build):
build_setoutpaths(b, 1)
rref = run_dataset2(what=2,
maxitems=mklens(b).N.val,
index=index,
interactive=False)
assert isfile(mklens(rref).ref_df.examples.syspath)
system(
f"cp {mklens(rref).ref_df.ref_data.inputs.syspath} {mklens(b).out_inputs.syspath}"
)
df = pd.read_csv(mklens(rref).df.syspath)
makedirs(mklens(b).out_barplots.syspath)
allowed = stabilize(
df,
npoints=mklens(b).npoints.val,
path_barplot=lambda i: join(
mklens(b).out_barplots.syspath, f'_plot_{i:03d}.png'),
path_traceplot=mklens(b).out_traceplot.syspath,
min_allow_size=mklens(b).min_allow_size.val)
with open(mklens(rref).ref_df.examples.syspath, 'rb') as f:
with open(mklens(b).out_examples.syspath, 'wb') as f_o:
_read = fd2examples(f)
_write = examples2fd(f_o)
try:
idx = 0
while True:
example = _read()
if idx in allowed:
_write(example)
idx += 1
except KeyboardInterrupt:
raise
except Exception as e:
print(e)
pass
def _make(b:Build):
build_setoutpaths(b, 1)
builtin_inputs:dict={
# Ref("i0"):IVal(0),
# Ref("i1"):IVal(1)
}
IMEMs:List[IMem] = [mkmap(_union(builtin_inputs,
{Ref(f"i{i+len(builtin_inputs.keys())}"): IVal(randint(range_min,range_max))
for i in range(num_inputs)}))
for _ in range(batch_size)]
writejson(mklens(b).out_inputs.syspath, [imem2json(M) for M in IMEMs])
def _make(b:Build):
build_setoutpaths(b, 1)
WLIB = mkwlib(lib_impl, Wdef)
IMEMs = [json2imem(j) for j in readjson(mklens(b).inputs.syspath)]
print(f"Inputs: {IMEMs}")
i = 0
# acc:List[Expr] = []
g = genexpr(WLIB, IMEMs)
written_bytes = 0
written_items = 0
time_start = time()
acci=set()
hb=time()
with open(mklens(b).out_examples.syspath,'wb') as f:
_add=examples2fd(f)
while time()<time_start+mklens(b).time2run_sec.val and \
written_items<mklens(b).maxitems.val:
# gt0=time()
r,mem,imems,exprw = next(g)
# gt1=time()
# print('gen time', gt1-gt0)
assert isinstance(imems[0][r], IVal), f"{imems[0][r]}"
i += 1
gi = 0
gexpr = gengather(r,mem)
# gg0=time()
exprs=[]
for expr in gexpr:
exprs.append(expr)
if gather_depth is not None:
exprs=exprs[-gather_depth:]
else:
exprs=[exprs[randint(0,len(exprs)-1)]]
for expr in exprs:
er=extrefs(expr)
ds=decls(expr)
# acc.append(expr)
for j in range(len(IMEMs)):
if len(ds)>0:
# print(gi, j, list(inps.keys()), print_expr(expr))
inps:IMem = TMap({k:imems[j][k] for k in er})
acci |= set(inps.values())
written_bytes+=_add(Example(inps,expr,imems[j][r]))
written_items+=1
hb2=time()
if written_items%100 == 0:
print(f".. NW {written_items} W {exprw[r]} DEP {depth(expr)} "
f"LAST_REF {r} WRBYTES {written_bytes // (1024) }K "
f"INPSZ {len(acci)} TIME {hb2-hb} "
f"VM {virtual_memory().used // 1024 // 1024}M")
hb=hb2
gi+=1
def _make(b:Build):
build_setoutpaths(b, 1)
WLIB = mkwlib(lib_impl, Wdef)
IMEMs = [json2imem(j) for j in readjson(mklens(b).inputs.syspath)]
print(f"Inputs: {IMEMs}")
i = 0
acc:List[Expr] = []
g = genexpr(WLIB, IMEMs)
written_bytes = 0
time_start = time()
with open(mklens(b).out_examples.syspath,'wb') as f:
_add=examples2fd(f)
while time()<time_start+mklens(b).time2run_sec.val:
r,mem,vals,w = next(g)
ival = vals[0]
assert isinstance(ival[r], IVal)
expr = gather(r,mem)
acc.append(expr)
i += 1
for j in range(len(IMEMs)):
written_bytes+=_add(Example(IMEMs[j],expr,vals[j][r]))
if i%300 == 0:
print(f".. i {i} W {w} LAST_REF {r} WRBYTES {written_bytes}.. ")
def _make(b: Build):
build_setoutpaths(b, 1)
df = examples_dataframe(mklens(b).examples.syspath)
df.to_csv(mklens(b).out_df.syspath)