def test_realize_readonly() -> None: with setup_storage2('test_realize_readonly') as (T, S): rref1 = realize(instantiate(mkstage, {'a': '1'}, S=S)) try: with open(join(store_rref2path(rref1, S), 'newfile'), 'w') as f: f.write('foo') raise ShouldHaveFailed('No write-protection??') except OSError: pass try: rmtree(store_rref2path(rref1, S)) raise ShouldHaveFailed('No remove-protection??') except OSError: pass def _realize(b: Build): with open(join(build_outpath(b), 'exe'), 'w') as f: f.write('#!/bin/sh\necho "Fooo"') chmod(join(build_outpath(b), 'exe'), S_IWRITE | S_IREAD | S_IEXEC) rref2 = realize( instantiate(mkdrv, Config({}), match_some(), build_wrapper(_realize), S=S)) assert pipe_stdout([join(store_rref2path(rref2,S),'exe')])=='Fooo\n', \ "Did we lost exec permission?"
def _setting(m: Manager, fullfill: bool) -> DRef: n1 = mkstage(m, {'name': '1', 'promise': [promise, 'artifact']}) def _realize(b: Build): o = build_outpath(b) c = build_cattrs(b) assert b.dref in c.promise assert n1 in store_cattrs(c.maman, S).promise assert build_path(b, c.promise) == join(o, 'uber-artifact') assert build_path(b, store_cattrs(c.maman, S).promise) == build_path( b, c.maman_promise) if fullfill: with open(build_path(b, c.promise), 'w') as f: f.write('chickenpoop') return mkdrv(m, mkconfig({ 'name': '2', 'maman': n1, 'promise': [promise, 'uber-artifact'], 'maman_promise': [n1, 'artifact'] }), matcher=match_some(), realizer=build_wrapper(_realize))
def _setting(m: Manager, assume_realized: bool) -> DRef: def _realize(b: Build): if assume_realized: raise ShouldHaveFailed('Should not call the real realizer') return build_outpath(b) return mkdrv(m, mkconfig({'name': '1'}), match_some(), build_wrapper(_realize))
def stage_datasetS(m: Manager, index: int = 0, Nvalid: int = 5000, npoints: int = 100) -> DRef: """ Creates stabilized dataset """ def _config(): name = 'dataset3' nonlocal index, npoints N = int(5 * Nvalid / 3) min_allow_size = Nvalid out_examples = [promise, 'examples.bin'] out_barplots = [promise, 'barplots'] out_traceplot = [promise, 'traceplot.png'] out_inputs = [promise, 'inputs.json'] version = 2 return locals() def _make(b: Build): build_setoutpaths(b, 1) rref = run_dataset2(what=2, maxitems=mklens(b).N.val, index=index, interactive=False) assert isfile(mklens(rref).ref_df.examples.syspath) system( f"cp {mklens(rref).ref_df.ref_data.inputs.syspath} {mklens(b).out_inputs.syspath}" ) df = pd.read_csv(mklens(rref).df.syspath) makedirs(mklens(b).out_barplots.syspath) allowed = stabilize( df, npoints=mklens(b).npoints.val, path_barplot=lambda i: join( mklens(b).out_barplots.syspath, f'_plot_{i:03d}.png'), path_traceplot=mklens(b).out_traceplot.syspath, min_allow_size=mklens(b).min_allow_size.val) with open(mklens(rref).ref_df.examples.syspath, 'rb') as f: with open(mklens(b).out_examples.syspath, 'wb') as f_o: _read = fd2examples(f) _write = examples2fd(f_o) try: idx = 0 while True: example = _read() if idx in allowed: _write(example) idx += 1 except KeyboardInterrupt: raise except Exception as e: print(e) pass return mkdrv(m, mkconfig(_config()), match_only(), build_wrapper(_make))
def stage_df(m: Manager, ref_data: DRef): def _config(): nonlocal ref_data name = mklens(ref_data).name.val + '-df' examples = mklens(ref_data).out_examples.refpath out_df = [promise, 'df.csv'] return locals() def _make(b: Build): build_setoutpaths(b, 1) df = examples_dataframe(mklens(b).examples.syspath) df.to_csv(mklens(b).out_df.syspath) return mkdrv(m, mkconfig(_config()), match_only(), build_wrapper(_make))
def breed_node(m:Manager)->DRef: def _config()->Config: name='ultimatum' nepoch=30000 n=300 nrounds=10*30 cutoff=0.1 version=6 nrunners=10 return Config(locals()) def _build(b:Build)->None: c=build_cattrs(b) p=Pool() p.starmap(_build_process,[(c,o) for o in build_outpaths(b,nouts=c.nrunners)],1) return mkdrv(m, config=_config(), matcher=match_all(), realizer=build_wrapper(_build))
def stage_vis(m: Manager, ref_df: DRef): def _config(): name = mklens(ref_df).name.val + '-vis' df = mklens(ref_df).out_df.refpath out_plot = [promise, 'plot.png'] title = mklens(ref_df).ref_data.name.val version = 3 return locals() def _make(b: Build): build_setoutpaths(b, 1) df = pd.read_csv(mklens(b).df.syspath) vis_bars(df, plot_fpath=mklens(b).out_plot.syspath, async_plot=None, plot_title=mklens(b).title.val) return mkdrv(m, mkconfig(_config()), match_latest(), build_wrapper(_make))
def stage_hello(m: Manager) -> DRef: def _config() -> Config: name: str = 'hello-bin' src: RefPath = [hello_src, f'hello-{hello_version}'] return locals() def _make(b: Build) -> None: o: Path = build_outpath(b) with TemporaryDirectory() as tmp: copytree(mklens(b).src.syspath, join(tmp, 'src')) dirrw(Path(join(tmp, 'src'))) cwd = getcwd() try: chdir(join(tmp, 'src')) system('./configure --prefix=/usr') system('make') system(f'make install DESTDIR={o}') finally: chdir(cwd) return mkdrv(m, mkconfig(_config()), match_latest(), build_wrapper(_make))
def stage_inputs(m:Manager, num_inputs:int=4, batch_size:int=5, index:int=1)->DRef: range_min = -100 range_max = 100 def _config(): name = 'dataset-inputs' nonlocal num_inputs, batch_size, range_min, range_max, index out_inputs = [promise, 'inputs.json'] version = 4 return locals() def _make(b:Build): build_setoutpaths(b, 1) builtin_inputs:dict={ # Ref("i0"):IVal(0), # Ref("i1"):IVal(1) } IMEMs:List[IMem] = [mkmap(_union(builtin_inputs, {Ref(f"i{i+len(builtin_inputs.keys())}"): IVal(randint(range_min,range_max)) for i in range(num_inputs)})) for _ in range(batch_size)] writejson(mklens(b).out_inputs.syspath, [imem2json(M) for M in IMEMs]) return mkdrv(m, mkconfig(_config()), match_only(), build_wrapper(_make))
def stage_dataset1(m:Manager, ref_inputs:DRef)->DRef: Wdef = 5 lib_impl = lib_arith lib_methods = [mn.val for mn in lib_impl.keys()] time2run_sec = int(0.5*60) def _config(): nonlocal Wdef,lib_methods,time2run_sec num_inputs = mklens(ref_inputs).num_inputs.val batch_size = mklens(ref_inputs).batch_size.val inputs = mklens(ref_inputs).out_inputs.refpath out_examples = [promise, 'examples.bin'] version = ['0'] return locals() def _make(b:Build): build_setoutpaths(b, 1) WLIB = mkwlib(lib_impl, Wdef) IMEMs = [json2imem(j) for j in readjson(mklens(b).inputs.syspath)] print(f"Inputs: {IMEMs}") i = 0 acc:List[Expr] = [] g = genexpr(WLIB, IMEMs) written_bytes = 0 time_start = time() with open(mklens(b).out_examples.syspath,'wb') as f: _add=examples2fd(f) while time()<time_start+mklens(b).time2run_sec.val: r,mem,vals,w = next(g) ival = vals[0] assert isinstance(ival[r], IVal) expr = gather(r,mem) acc.append(expr) i += 1 for j in range(len(IMEMs)): written_bytes+=_add(Example(IMEMs[j],expr,vals[j][r])) if i%300 == 0: print(f".. i {i} W {w} LAST_REF {r} WRBYTES {written_bytes}.. ") return mkdrv(m, mkconfig(_config()), match_only(), build_wrapper(_make))
def _setting(m:Manager)->DRef: n1=mkstage(m, {'name':'1', 'promise':[promise,'artifact']}) n2=mkstage(m, {'name':'2', 'promise':[promise,'artifact'], 'dict':{'d1':1} }) def _realize(b:Build): o=build_outpath(b) c=build_cattrs(b) assert isrefpath(mklens(b).maman.promise.refpath) assert isfile(mklens(b).papa.promise.syspath) assert o in mklens(b).promise.syspath assert o == mklens(b).syspath assert mklens(b).papa.name.val == '2' assert mklens(b).papa.dref == c.papa with open(mklens(b).promise.syspath,'w') as f: f.write('chickenpoop') return mkdrv(m, mkconfig({'name':'3', 'maman':n1, 'papa':n2, 'promise':[promise,'artifact'], }), matcher=match_some(), realizer=build_wrapper(_realize))
def summarize_node(m:Manager)->DRef: def _config()->Config: return Config({ 'name':'analyzer', 'version':3, 'history':[breed_node(m), 'history.json'] }) def _build(b:Build)->None: chdir(build_outpath(b)) c=build_cattrs(b) fig=plt.figure() ax=fig.add_subplot(1, 1, 1) ax.set_ylabel('mean strategies') ax.grid() for nhist,histpath in enumerate(build_paths(b, c.history)): epoches:List[float]=[]; pmeans:List[float]=[]; rmeans:List[float]=[] with open(histpath,'r') as f: epoches,pmeans,rmeans=json_loads(f.read()) ax.plot(epoches,pmeans,label=f'pmeans{nhist}',color='blue') ax.plot(epoches,rmeans,label=f'rmeans{nhist}',color='orange') plt.savefig('figure.png') return mkdrv(m, config=_config(), matcher=match_only(), realizer=build_wrapper(_build))
def model(m)->DRef: return mkdrv(m, mnist_config(), mnist_match(), build_wrapper(mnist_build))
def model_best(m)->DRef: return mkdrv(m, mnist_config(), match_best('accuracy.txt'), build_wrapper(mnist_build))
def hello_realize(b: Build) -> None: c: Any = build_cattrs(b) o: Path = build_outpath(b) with TemporaryDirectory() as tmp: copytree(mklens(b).src.syspath, join(tmp, 'src')) dirrw(Path(join(tmp, 'src'))) cwd = getcwd() try: chdir(join(tmp, 'src')) system(f'./configure --prefix=/usr') system(f'make') system(f'make install DESTDIR={o}') finally: chdir(cwd) from pylightnix import mkdrv, build_wrapper, match_only hello:DRef = \ instantiate_inplace(mkdrv, hello_config(), match_only(), build_wrapper(hello_realize)) print(hello) rref: RRef = realize_inplace(hello) print(rref) hello_bin = join(rref2path(rref), 'usr/bin/hello') print(Popen([hello_bin], stdout=PIPE, shell=True).stdout.read()) # type:ignore
def stage_dataset2(m:Manager, ref_inputs:DRef, gather_depth:Optional[int]=999, Wdef:int=5, maxtime_sec:Optional[int]=None, maxitems:Optional[int]=None)->DRef: lib_impl = lib_arith lib_methods = [mn.val for mn in lib_impl.keys()] assert (maxtime_sec is not None) or (maxitems is not None), \ "At least one stop criteria should be defined" time2run_sec:int = maxsize if maxtime_sec is None else maxtime_sec maxitems:int = maxsize if maxitems is None else maxitems def _config(): nonlocal Wdef,lib_methods,time2run_sec,maxitems,gather_depth num_inputs = mklens(ref_inputs).num_inputs.val batch_size = mklens(ref_inputs).batch_size.val name=f'dataset2-ni{num_inputs}-bs{batch_size}-Wd{Wdef}-mi{maxitems}' inputs = mklens(ref_inputs).out_inputs.refpath out_examples = [promise, 'examples.bin'] version = ['21'] return locals() def _make(b:Build): build_setoutpaths(b, 1) WLIB = mkwlib(lib_impl, Wdef) IMEMs = [json2imem(j) for j in readjson(mklens(b).inputs.syspath)] print(f"Inputs: {IMEMs}") i = 0 # acc:List[Expr] = [] g = genexpr(WLIB, IMEMs) written_bytes = 0 written_items = 0 time_start = time() acci=set() hb=time() with open(mklens(b).out_examples.syspath,'wb') as f: _add=examples2fd(f) while time()<time_start+mklens(b).time2run_sec.val and \ written_items<mklens(b).maxitems.val: # gt0=time() r,mem,imems,exprw = next(g) # gt1=time() # print('gen time', gt1-gt0) assert isinstance(imems[0][r], IVal), f"{imems[0][r]}" i += 1 gi = 0 gexpr = gengather(r,mem) # gg0=time() exprs=[] for expr in gexpr: exprs.append(expr) if gather_depth is not None: exprs=exprs[-gather_depth:] else: exprs=[exprs[randint(0,len(exprs)-1)]] for expr in exprs: er=extrefs(expr) ds=decls(expr) # acc.append(expr) for j in range(len(IMEMs)): if len(ds)>0: # print(gi, j, list(inps.keys()), print_expr(expr)) inps:IMem = TMap({k:imems[j][k] for k in er}) acci |= set(inps.values()) written_bytes+=_add(Example(inps,expr,imems[j][r])) written_items+=1 hb2=time() if written_items%100 == 0: print(f".. NW {written_items} W {exprw[r]} DEP {depth(expr)} " f"LAST_REF {r} WRBYTES {written_bytes // (1024) }K " f"INPSZ {len(acci)} TIME {hb2-hb} " f"VM {virtual_memory().used // 1024 // 1024}M") hb=hb2 gi+=1 # gg1=time() # print('gather time', gg1-gg0) return mkdrv(m, mkconfig(_config()), match_only(), build_wrapper(_make))
def evolution_stage(m:Manager)->DRef: return mkdrv(m, config=evolution_config(), matcher=match_latest(n=10), realizer=build_wrapper(evolution_realize))
def summary_stage(m:Manager)->DRef: return mkdrv(m, config=summary_config(evolution_stage(m)), matcher=match_latest(), realizer=build_wrapper(summary_realize))