def test_realize_readonly() -> None:
with setup_storage2('test_realize_readonly') as (T, S):
rref1 = realize(instantiate(mkstage, {'a': '1'}, S=S))
try:
with open(join(store_rref2path(rref1, S), 'newfile'), 'w') as f:
f.write('foo')
raise ShouldHaveFailed('No write-protection??')
except OSError:
pass
try:
rmtree(store_rref2path(rref1, S))
raise ShouldHaveFailed('No remove-protection??')
except OSError:
pass
def _realize(b: Build):
with open(join(build_outpath(b), 'exe'), 'w') as f:
f.write('#!/bin/sh\necho "Fooo"')
chmod(join(build_outpath(b), 'exe'), S_IWRITE | S_IREAD | S_IEXEC)
rref2 = realize(
instantiate(mkdrv,
Config({}),
match_some(),
build_wrapper(_realize),
S=S))
assert pipe_stdout([join(store_rref2path(rref2,S),'exe')])=='Fooo\n', \
"Did we lost exec permission?"
def _setting(m: Manager, fullfill: bool) -> DRef:
n1 = mkstage(m, {'name': '1', 'promise': [promise, 'artifact']})
def _realize(b: Build):
o = build_outpath(b)
c = build_cattrs(b)
assert b.dref in c.promise
assert n1 in store_cattrs(c.maman, S).promise
assert build_path(b, c.promise) == join(o, 'uber-artifact')
assert build_path(b,
store_cattrs(c.maman,
S).promise) == build_path(
b, c.maman_promise)
if fullfill:
with open(build_path(b, c.promise), 'w') as f:
f.write('chickenpoop')
return mkdrv(m,
mkconfig({
'name': '2',
'maman': n1,
'promise': [promise, 'uber-artifact'],
'maman_promise': [n1, 'artifact']
}),
matcher=match_some(),
realizer=build_wrapper(_realize))
def _setting(m: Manager, assume_realized: bool) -> DRef:
def _realize(b: Build):
if assume_realized:
raise ShouldHaveFailed('Should not call the real realizer')
return build_outpath(b)
return mkdrv(m, mkconfig({'name': '1'}), match_some(),
build_wrapper(_realize))
def stage_datasetS(m: Manager,
index: int = 0,
Nvalid: int = 5000,
npoints: int = 100) -> DRef:
""" Creates stabilized dataset """
def _config():
name = 'dataset3'
nonlocal index, npoints
N = int(5 * Nvalid / 3)
min_allow_size = Nvalid
out_examples = [promise, 'examples.bin']
out_barplots = [promise, 'barplots']
out_traceplot = [promise, 'traceplot.png']
out_inputs = [promise, 'inputs.json']
version = 2
return locals()
def _make(b: Build):
build_setoutpaths(b, 1)
rref = run_dataset2(what=2,
maxitems=mklens(b).N.val,
index=index,
interactive=False)
assert isfile(mklens(rref).ref_df.examples.syspath)
system(
f"cp {mklens(rref).ref_df.ref_data.inputs.syspath} {mklens(b).out_inputs.syspath}"
)
df = pd.read_csv(mklens(rref).df.syspath)
makedirs(mklens(b).out_barplots.syspath)
allowed = stabilize(
df,
npoints=mklens(b).npoints.val,
path_barplot=lambda i: join(
mklens(b).out_barplots.syspath, f'_plot_{i:03d}.png'),
path_traceplot=mklens(b).out_traceplot.syspath,
min_allow_size=mklens(b).min_allow_size.val)
with open(mklens(rref).ref_df.examples.syspath, 'rb') as f:
with open(mklens(b).out_examples.syspath, 'wb') as f_o:
_read = fd2examples(f)
_write = examples2fd(f_o)
try:
idx = 0
while True:
example = _read()
if idx in allowed:
_write(example)
idx += 1
except KeyboardInterrupt:
raise
except Exception as e:
print(e)
pass
return mkdrv(m, mkconfig(_config()), match_only(), build_wrapper(_make))
def stage_df(m: Manager, ref_data: DRef):
def _config():
nonlocal ref_data
name = mklens(ref_data).name.val + '-df'
examples = mklens(ref_data).out_examples.refpath
out_df = [promise, 'df.csv']
return locals()
def _make(b: Build):
build_setoutpaths(b, 1)
df = examples_dataframe(mklens(b).examples.syspath)
df.to_csv(mklens(b).out_df.syspath)
return mkdrv(m, mkconfig(_config()), match_only(), build_wrapper(_make))
def breed_node(m:Manager)->DRef:
def _config()->Config:
name='ultimatum'
nepoch=30000
n=300
nrounds=10*30
cutoff=0.1
version=6
nrunners=10
return Config(locals())
def _build(b:Build)->None:
c=build_cattrs(b)
p=Pool()
p.starmap(_build_process,[(c,o) for o in build_outpaths(b,nouts=c.nrunners)],1)
return mkdrv(m, config=_config(), matcher=match_all(), realizer=build_wrapper(_build))
def stage_vis(m: Manager, ref_df: DRef):
def _config():
name = mklens(ref_df).name.val + '-vis'
df = mklens(ref_df).out_df.refpath
out_plot = [promise, 'plot.png']
title = mklens(ref_df).ref_data.name.val
version = 3
return locals()
def _make(b: Build):
build_setoutpaths(b, 1)
df = pd.read_csv(mklens(b).df.syspath)
vis_bars(df,
plot_fpath=mklens(b).out_plot.syspath,
async_plot=None,
plot_title=mklens(b).title.val)
return mkdrv(m, mkconfig(_config()), match_latest(), build_wrapper(_make))
def stage_hello(m: Manager) -> DRef:
def _config() -> Config:
name: str = 'hello-bin'
src: RefPath = [hello_src, f'hello-{hello_version}']
return locals()
def _make(b: Build) -> None:
o: Path = build_outpath(b)
with TemporaryDirectory() as tmp:
copytree(mklens(b).src.syspath, join(tmp, 'src'))
dirrw(Path(join(tmp, 'src')))
cwd = getcwd()
try:
chdir(join(tmp, 'src'))
system('./configure --prefix=/usr')
system('make')
system(f'make install DESTDIR={o}')
finally:
chdir(cwd)
return mkdrv(m, mkconfig(_config()), match_latest(), build_wrapper(_make))
def stage_inputs(m:Manager, num_inputs:int=4, batch_size:int=5, index:int=1)->DRef:
range_min = -100
range_max = 100
def _config():
name = 'dataset-inputs'
nonlocal num_inputs, batch_size, range_min, range_max, index
out_inputs = [promise, 'inputs.json']
version = 4
return locals()
def _make(b:Build):
build_setoutpaths(b, 1)
builtin_inputs:dict={
# Ref("i0"):IVal(0),
# Ref("i1"):IVal(1)
}
IMEMs:List[IMem] = [mkmap(_union(builtin_inputs,
{Ref(f"i{i+len(builtin_inputs.keys())}"): IVal(randint(range_min,range_max))
for i in range(num_inputs)}))
for _ in range(batch_size)]
writejson(mklens(b).out_inputs.syspath, [imem2json(M) for M in IMEMs])
return mkdrv(m, mkconfig(_config()), match_only(), build_wrapper(_make))
def stage_dataset1(m:Manager, ref_inputs:DRef)->DRef:
Wdef = 5
lib_impl = lib_arith
lib_methods = [mn.val for mn in lib_impl.keys()]
time2run_sec = int(0.5*60)
def _config():
nonlocal Wdef,lib_methods,time2run_sec
num_inputs = mklens(ref_inputs).num_inputs.val
batch_size = mklens(ref_inputs).batch_size.val
inputs = mklens(ref_inputs).out_inputs.refpath
out_examples = [promise, 'examples.bin']
version = ['0']
return locals()
def _make(b:Build):
build_setoutpaths(b, 1)
WLIB = mkwlib(lib_impl, Wdef)
IMEMs = [json2imem(j) for j in readjson(mklens(b).inputs.syspath)]
print(f"Inputs: {IMEMs}")
i = 0
acc:List[Expr] = []
g = genexpr(WLIB, IMEMs)
written_bytes = 0
time_start = time()
with open(mklens(b).out_examples.syspath,'wb') as f:
_add=examples2fd(f)
while time()<time_start+mklens(b).time2run_sec.val:
r,mem,vals,w = next(g)
ival = vals[0]
assert isinstance(ival[r], IVal)
expr = gather(r,mem)
acc.append(expr)
i += 1
for j in range(len(IMEMs)):
written_bytes+=_add(Example(IMEMs[j],expr,vals[j][r]))
if i%300 == 0:
print(f".. i {i} W {w} LAST_REF {r} WRBYTES {written_bytes}.. ")
return mkdrv(m, mkconfig(_config()), match_only(), build_wrapper(_make))
def _setting(m:Manager)->DRef:
n1=mkstage(m, {'name':'1', 'promise':[promise,'artifact']})
n2=mkstage(m, {'name':'2', 'promise':[promise,'artifact'],
'dict':{'d1':1} })
def _realize(b:Build):
o=build_outpath(b)
c=build_cattrs(b)
assert isrefpath(mklens(b).maman.promise.refpath)
assert isfile(mklens(b).papa.promise.syspath)
assert o in mklens(b).promise.syspath
assert o == mklens(b).syspath
assert mklens(b).papa.name.val == '2'
assert mklens(b).papa.dref == c.papa
with open(mklens(b).promise.syspath,'w') as f:
f.write('chickenpoop')
return mkdrv(m,
mkconfig({'name':'3', 'maman':n1, 'papa':n2,
'promise':[promise,'artifact'],
}),
matcher=match_some(),
realizer=build_wrapper(_realize))
def summarize_node(m:Manager)->DRef:
def _config()->Config:
return Config({
'name':'analyzer',
'version':3,
'history':[breed_node(m), 'history.json']
})
def _build(b:Build)->None:
chdir(build_outpath(b))
c=build_cattrs(b)
fig=plt.figure()
ax=fig.add_subplot(1, 1, 1)
ax.set_ylabel('mean strategies')
ax.grid()
for nhist,histpath in enumerate(build_paths(b, c.history)):
epoches:List[float]=[]; pmeans:List[float]=[]; rmeans:List[float]=[]
with open(histpath,'r') as f:
epoches,pmeans,rmeans=json_loads(f.read())
ax.plot(epoches,pmeans,label=f'pmeans{nhist}',color='blue')
ax.plot(epoches,rmeans,label=f'rmeans{nhist}',color='orange')
plt.savefig('figure.png')
return mkdrv(m, config=_config(), matcher=match_only(), realizer=build_wrapper(_build))
def model(m)->DRef: return mkdrv(m, mnist_config(), mnist_match(), build_wrapper(mnist_build))
def model_best(m)->DRef:
return mkdrv(m, mnist_config(), match_best('accuracy.txt'), build_wrapper(mnist_build))
def hello_realize(b: Build) -> None:
c: Any = build_cattrs(b)
o: Path = build_outpath(b)
with TemporaryDirectory() as tmp:
copytree(mklens(b).src.syspath, join(tmp, 'src'))
dirrw(Path(join(tmp, 'src')))
cwd = getcwd()
try:
chdir(join(tmp, 'src'))
system(f'./configure --prefix=/usr')
system(f'make')
system(f'make install DESTDIR={o}')
finally:
chdir(cwd)
from pylightnix import mkdrv, build_wrapper, match_only
hello:DRef = \
instantiate_inplace(mkdrv, hello_config(), match_only(), build_wrapper(hello_realize))
print(hello)
rref: RRef = realize_inplace(hello)
print(rref)
hello_bin = join(rref2path(rref), 'usr/bin/hello')
print(Popen([hello_bin], stdout=PIPE, shell=True).stdout.read()) # type:ignore
def stage_dataset2(m:Manager, ref_inputs:DRef,
gather_depth:Optional[int]=999,
Wdef:int=5,
maxtime_sec:Optional[int]=None,
maxitems:Optional[int]=None)->DRef:
lib_impl = lib_arith
lib_methods = [mn.val for mn in lib_impl.keys()]
assert (maxtime_sec is not None) or (maxitems is not None), \
"At least one stop criteria should be defined"
time2run_sec:int = maxsize if maxtime_sec is None else maxtime_sec
maxitems:int = maxsize if maxitems is None else maxitems
def _config():
nonlocal Wdef,lib_methods,time2run_sec,maxitems,gather_depth
num_inputs = mklens(ref_inputs).num_inputs.val
batch_size = mklens(ref_inputs).batch_size.val
name=f'dataset2-ni{num_inputs}-bs{batch_size}-Wd{Wdef}-mi{maxitems}'
inputs = mklens(ref_inputs).out_inputs.refpath
out_examples = [promise, 'examples.bin']
version = ['21']
return locals()
def _make(b:Build):
build_setoutpaths(b, 1)
WLIB = mkwlib(lib_impl, Wdef)
IMEMs = [json2imem(j) for j in readjson(mklens(b).inputs.syspath)]
print(f"Inputs: {IMEMs}")
i = 0
# acc:List[Expr] = []
g = genexpr(WLIB, IMEMs)
written_bytes = 0
written_items = 0
time_start = time()
acci=set()
hb=time()
with open(mklens(b).out_examples.syspath,'wb') as f:
_add=examples2fd(f)
while time()<time_start+mklens(b).time2run_sec.val and \
written_items<mklens(b).maxitems.val:
# gt0=time()
r,mem,imems,exprw = next(g)
# gt1=time()
# print('gen time', gt1-gt0)
assert isinstance(imems[0][r], IVal), f"{imems[0][r]}"
i += 1
gi = 0
gexpr = gengather(r,mem)
# gg0=time()
exprs=[]
for expr in gexpr:
exprs.append(expr)
if gather_depth is not None:
exprs=exprs[-gather_depth:]
else:
exprs=[exprs[randint(0,len(exprs)-1)]]
for expr in exprs:
er=extrefs(expr)
ds=decls(expr)
# acc.append(expr)
for j in range(len(IMEMs)):
if len(ds)>0:
# print(gi, j, list(inps.keys()), print_expr(expr))
inps:IMem = TMap({k:imems[j][k] for k in er})
acci |= set(inps.values())
written_bytes+=_add(Example(inps,expr,imems[j][r]))
written_items+=1
hb2=time()
if written_items%100 == 0:
print(f".. NW {written_items} W {exprw[r]} DEP {depth(expr)} "
f"LAST_REF {r} WRBYTES {written_bytes // (1024) }K "
f"INPSZ {len(acci)} TIME {hb2-hb} "
f"VM {virtual_memory().used // 1024 // 1024}M")
hb=hb2
gi+=1
# gg1=time()
# print('gather time', gg1-gg0)
return mkdrv(m, mkconfig(_config()), match_only(), build_wrapper(_make))
def evolution_stage(m:Manager)->DRef:
return mkdrv(m, config=evolution_config(),
matcher=match_latest(n=10),
realizer=build_wrapper(evolution_realize))
def summary_stage(m:Manager)->DRef:
return mkdrv(m, config=summary_config(evolution_stage(m)),
matcher=match_latest(),
realizer=build_wrapper(summary_realize))
