def _make(b: Build):
build_setoutpaths(b, 1)
df = pd.read_csv(mklens(b).df.syspath)
vis_bars(df,
plot_fpath=mklens(b).out_plot.syspath,
async_plot=None,
plot_title=mklens(b).title.val)
def sample(N:int=5)->List[Example]:
""" Samples `N` examples from the dataset """
acc:List[Example]=[]
while len(acc)<N:
i=int(choice(SECTIONS, size=1)[0])
n=0
print('Loading section', i)
try:
with open(mklens(load3(i,False)).out_examples.syspath, 'rb') as ef:
_,_skip=fd2examples2(ef)
try:
while True:
_skip()
n+=1
except IndexError:
pass
except LookupError:
print('Woops, no realization')
continue
x=randint(0,n-1)
print('.. it contains', n, 'records, we take', x)
with open(mklens(load3(i,False)).out_examples.syspath, 'rb') as ef:
_next,_skip=fd2examples2(ef)
for i in range(x-1):
_skip()
acc.append(_next())
return acc
def _config():
name = mklens(ref_df).name.val + '-vis'
df = mklens(ref_df).out_df.refpath
out_plot = [promise, 'plot.png']
title = mklens(ref_df).ref_data.name.val
version = 3
return locals()
def _make(b: Build):
build_setoutpaths(b, 1)
rref = run_dataset2(what=2,
maxitems=mklens(b).N.val,
index=index,
interactive=False)
assert isfile(mklens(rref).ref_df.examples.syspath)
system(
f"cp {mklens(rref).ref_df.ref_data.inputs.syspath} {mklens(b).out_inputs.syspath}"
)
df = pd.read_csv(mklens(rref).df.syspath)
makedirs(mklens(b).out_barplots.syspath)
allowed = stabilize(
df,
npoints=mklens(b).npoints.val,
path_barplot=lambda i: join(
mklens(b).out_barplots.syspath, f'_plot_{i:03d}.png'),
path_traceplot=mklens(b).out_traceplot.syspath,
min_allow_size=mklens(b).min_allow_size.val)
with open(mklens(rref).ref_df.examples.syspath, 'rb') as f:
with open(mklens(b).out_examples.syspath, 'wb') as f_o:
_read = fd2examples(f)
_write = examples2fd(f_o)
try:
idx = 0
while True:
example = _read()
if idx in allowed:
_write(example)
idx += 1
except KeyboardInterrupt:
raise
except Exception as e:
print(e)
pass
def _config(): nonlocal Wdef,lib_methods,time2run_sec num_inputs = mklens(ref_inputs).num_inputs.val batch_size = mklens(ref_inputs).batch_size.val inputs = mklens(ref_inputs).out_inputs.refpath out_examples = [promise, 'examples.bin'] version = ['0'] return locals()
def _config():
nonlocal Wdef,lib_methods,time2run_sec,maxitems,gather_depth
num_inputs = mklens(ref_inputs).num_inputs.val
batch_size = mklens(ref_inputs).batch_size.val
name=f'dataset2-ni{num_inputs}-bs{batch_size}-Wd{Wdef}-mi{maxitems}'
inputs = mklens(ref_inputs).out_inputs.refpath
out_examples = [promise, 'examples.bin']
version = ['21']
return locals()
def _make(b:Build):
build_setoutpaths(b, 1)
WLIB = mkwlib(lib_impl, Wdef)
IMEMs = [json2imem(j) for j in readjson(mklens(b).inputs.syspath)]
print(f"Inputs: {IMEMs}")
i = 0
# acc:List[Expr] = []
g = genexpr(WLIB, IMEMs)
written_bytes = 0
written_items = 0
time_start = time()
acci=set()
hb=time()
with open(mklens(b).out_examples.syspath,'wb') as f:
_add=examples2fd(f)
while time()<time_start+mklens(b).time2run_sec.val and \
written_items<mklens(b).maxitems.val:
# gt0=time()
r,mem,imems,exprw = next(g)
# gt1=time()
# print('gen time', gt1-gt0)
assert isinstance(imems[0][r], IVal), f"{imems[0][r]}"
i += 1
gi = 0
gexpr = gengather(r,mem)
# gg0=time()
exprs=[]
for expr in gexpr:
exprs.append(expr)
if gather_depth is not None:
exprs=exprs[-gather_depth:]
else:
exprs=[exprs[randint(0,len(exprs)-1)]]
for expr in exprs:
er=extrefs(expr)
ds=decls(expr)
# acc.append(expr)
for j in range(len(IMEMs)):
if len(ds)>0:
# print(gi, j, list(inps.keys()), print_expr(expr))
inps:IMem = TMap({k:imems[j][k] for k in er})
acci |= set(inps.values())
written_bytes+=_add(Example(inps,expr,imems[j][r]))
written_items+=1
hb2=time()
if written_items%100 == 0:
print(f".. NW {written_items} W {exprw[r]} DEP {depth(expr)} "
f"LAST_REF {r} WRBYTES {written_bytes // (1024) }K "
f"INPSZ {len(acci)} TIME {hb2-hb} "
f"VM {virtual_memory().used // 1024 // 1024}M")
hb=hb2
gi+=1
def test_lens_closures():
with setup_storage('test_lens_closures'):
def _setting(m:Manager)->DRef:
n1=mkstage(m, {'name':'1', 'x':33, 'promise':[promise,'artifact']})
n2=mkstage(m, {'name':'2', 'papa':n1, 'dict':{'d1':1} })
n3=mkstage(m, {'name':'3', 'maman':n2 })
return n3
clo=instantiate(_setting)
assert isclosure(clo)
rref=realize(mklens(clo).maman.papa.closure)
assert mklens(rref).x.val==33
assert open(mklens(rref).promise.syspath).read()=='0'
def test_redefine() -> None:
with setup_storage('test_redefine'):
def _setting(m: Manager) -> DRef:
return mknode(m, {
'name': 'foo',
'bar': 'baz',
'output': [promise, 'f']
}, {Name('f'): bytes(('umgh').encode('utf-8'))})
def _nc(c):
mklens(c).bar.val = 42
_setting2 = redefine(_setting, new_config=_nc)
rref = realize(instantiate(_setting2))
assert mklens(rref).bar.val == 42
assert tryread(mklens(rref).output.syspath) == 'umgh'
def print_size()->None:
total=0
for index in SECTIONS:
try:
sz=stat(mklens(load3(index,False)).out_examples.syspath).st_size
total+=sz
print(f"Index {index}: {sz/1024/1024:.1f}M")
except LookupError:
print(f"Woops, no dataset index {index}")
print(f"Total: {total/1024/1024:.1f}M")
def test_recursive_realize_with_another_manager() -> None:
with setup_storage2('test_recursive_realize_with_another_manager') as (T,
S):
rref_inner = None
def _setup_inner(m):
return mkstage(m, {'foo': 'bar'})
def _setup_outer(m):
nonlocal rref_inner
rref_inner = realize(instantiate(_setup_inner, S=S))
return mkstage(m, {'baz': mklens(rref_inner, S=S).foo.val})
rref = realize(instantiate(_setup_outer, S=S))
assert rref_inner is not None
assert len(store_deepdeps([rref2dref(rref)], S=S)) == 0
assert len(store_deepdeps([rref2dref(rref_inner)], S=S)) == 0
assert mklens(rref_inner, S=S).foo.val == 'bar'
assert mklens(rref, S=S).baz.val == 'bar'
def _make(b:Build):
build_setoutpaths(b, 1)
builtin_inputs:dict={
# Ref("i0"):IVal(0),
# Ref("i1"):IVal(1)
}
IMEMs:List[IMem] = [mkmap(_union(builtin_inputs,
{Ref(f"i{i+len(builtin_inputs.keys())}"): IVal(randint(range_min,range_max))
for i in range(num_inputs)}))
for _ in range(batch_size)]
writejson(mklens(b).out_inputs.syspath, [imem2json(M) for M in IMEMs])
def _make(b: Build) -> None:
o: Path = build_outpath(b)
with TemporaryDirectory() as tmp:
copytree(mklens(b).src.syspath, join(tmp, 'src'))
dirrw(Path(join(tmp, 'src')))
cwd = getcwd()
try:
chdir(join(tmp, 'src'))
system('./configure --prefix=/usr')
system('make')
system(f'make install DESTDIR={o}')
finally:
chdir(cwd)
def _realize(b:Build):
o=build_outpath(b)
c=build_cattrs(b)
assert isrefpath(mklens(b).maman.promise.refpath)
assert isfile(mklens(b).papa.promise.syspath)
assert o in mklens(b).promise.syspath
assert o == mklens(b).syspath
assert mklens(b).papa.name.val == '2'
assert mklens(b).papa.dref == c.papa
with open(mklens(b).promise.syspath,'w') as f:
f.write('chickenpoop')
def _make(b:Build):
build_setoutpaths(b, 1)
WLIB = mkwlib(lib_impl, Wdef)
IMEMs = [json2imem(j) for j in readjson(mklens(b).inputs.syspath)]
print(f"Inputs: {IMEMs}")
i = 0
acc:List[Expr] = []
g = genexpr(WLIB, IMEMs)
written_bytes = 0
time_start = time()
with open(mklens(b).out_examples.syspath,'wb') as f:
_add=examples2fd(f)
while time()<time_start+mklens(b).time2run_sec.val:
r,mem,vals,w = next(g)
ival = vals[0]
assert isinstance(ival[r], IVal)
expr = gather(r,mem)
acc.append(expr)
i += 1
for j in range(len(IMEMs)):
written_bytes+=_add(Example(IMEMs[j],expr,vals[j][r]))
if i%300 == 0:
print(f".. i {i} W {w} LAST_REF {r} WRBYTES {written_bytes}.. ")
def loadsection(si:int)->List[Example]:
print(f"Loading section {si}")
acc:list=[]
try:
with open(mklens(load3(si,False)).out_examples.syspath, 'rb') as ef:
_next=fd2examples(ef)
e=_next()
print(type(e))
s=pickle_dumps(e, protocol=HIGHEST_PROTOCOL)
print(s)
# while True:
# acc.append(_next())
except KeyboardInterrupt:
raise
except LookupError:
pass
return acc[:10]
def test_fetchlocal2():
with setup_storage('test_fetclocal') as tmp:
mockdata = join(tmp, 'mockdata')
with open(mockdata, 'w') as f:
f.write('dogfood')
wanted_sha256 = pipe_stdout([SHA256SUM, "mockdata"],
cwd=tmp).split()[0]
rref = realize(
instantiate(fetchlocal,
path=mockdata,
sha256=wanted_sha256,
mode='as-is'))
assert isrref(rref)
assert isfile(join(store_rref2path(rref), 'mockdata'))
assert isfile(mklens(rref).out_path.syspath)
def test_lens():
with setup_storage('test_lens'):
def _setting(m:Manager)->DRef:
n1=mkstage(m, {'name':'1', 'promise':[promise,'artifact']})
n2=mkstage(m, {'name':'2', 'promise':[promise,'artifact'],
'dict':{'d1':1} })
def _realize(b:Build):
o=build_outpath(b)
c=build_cattrs(b)
assert isrefpath(mklens(b).maman.promise.refpath)
assert isfile(mklens(b).papa.promise.syspath)
assert o in mklens(b).promise.syspath
assert o == mklens(b).syspath
assert mklens(b).papa.name.val == '2'
assert mklens(b).papa.dref == c.papa
with open(mklens(b).promise.syspath,'w') as f:
f.write('chickenpoop')
return mkdrv(m,
mkconfig({'name':'3', 'maman':n1, 'papa':n2,
'promise':[promise,'artifact'],
}),
matcher=match_some(),
realizer=build_wrapper(_realize))
clo=instantiate(_setting)
assert isrefpath(mklens(clo.dref).maman.promise.refpath)
assert isdir(mklens(clo.dref).syspath)
rref=realize(clo)
assert_valid_rref(rref)
assert isrefpath(mklens(rref).maman.promise.refpath)
assert isfile(mklens(rref).maman.promise.syspath)
assert mklens(rref).rref == rref
assert isrefpath(mklens(rref).papa.promise.refpath)
assert mklens(rref).papa.dict.d1.val == 1
assert mklens(rref).dref == clo.dref
assert isdir(mklens(rref).syspath)
try:
print(mklens(clo.dref).maman.promise.syspath)
raise ShouldHaveFailed()
except AssertionError:
pass
try:
print(mklens(rref).papa.dict.d1.get('xxx'))
raise ShouldHaveFailed()
except AssertionError:
pass
try:
print(mklens(rref).papa.dict.d1.syspath)
raise ShouldHaveFailed()
except AssertionError:
pass
try:
mklens(rref).papa.dict.d1.val=42 # can't mutate rref
raise ShouldHaveFailed()
except AssertionError:
pass
d={'foo':'foo','bar':'bar'}
mklens(d).foo.val='zzz'
assert d['foo']=='zzz'
try:
mklens(d).x.val=42 # can't set new values
raise ShouldHaveFailed()
except AssertionError:
pass
mklens(d).bar.val+='33'
assert d['bar']=='bar33'
def _make(b: Build):
build_setoutpaths(b, 1)
df = examples_dataframe(mklens(b).examples.syspath)
df.to_csv(mklens(b).out_df.syspath)
def _setup_outer(m):
nonlocal rref_inner
rref_inner = realize(instantiate(_setup_inner, S=S))
return mkstage(m, {'baz': mklens(rref_inner, S=S).foo.val})
def hello_config() -> Config:
name = 'hello-bin'
src = mklens(hello_src).src.refpath
return mkconfig(locals())
def load(what: int) -> DataFrame:
rref = run(what, interactive=False)
df = pd.read_csv(mklens(rref).df.syspath)
return df
def _config():
nonlocal ref_data
name = mklens(ref_data).name.val + '-df'
examples = mklens(ref_data).out_examples.refpath
out_df = [promise, 'df.csv']
return locals()
def _nc(c):
mklens(c).bar.val = 42
