def test_config_ro(): d = {'a': 1, 'b': 33} c = mkconfig(d) cro = config_cattrs(RConfig( config_dict(c))) # We sure we don't have promises here for k in d.keys(): assert getattr(cro, k) == d[k]
def _setting(m: Manager, fullfill: bool) -> DRef: n1 = mkstage(m, {'name': '1', 'promise': [promise, 'artifact']}) def _realize(b: Build): o = build_outpath(b) c = build_cattrs(b) assert b.dref in c.promise assert n1 in store_cattrs(c.maman, S).promise assert build_path(b, c.promise) == join(o, 'uber-artifact') assert build_path(b, store_cattrs(c.maman, S).promise) == build_path( b, c.maman_promise) if fullfill: with open(build_path(b, c.promise), 'w') as f: f.write('chickenpoop') return mkdrv(m, mkconfig({ 'name': '2', 'maman': n1, 'promise': [promise, 'uber-artifact'], 'maman_promise': [n1, 'artifact'] }), matcher=match_some(), realizer=build_wrapper(_realize))
def mnist_config(mnist:DRef)->Config: name = 'convnn_mnist' dataset:RefPath = [mnist, 'mnist.npz'] learning_rate = 1e-3 num_epoches = 1 version = 7 return mkconfig(locals())
def _setting(m: Manager, assume_realized: bool) -> DRef: def _realize(b: Build): if assume_realized: raise ShouldHaveFailed('Should not call the real realizer') return build_outpath(b) return mkdrv(m, mkconfig({'name': '1'}), match_some(), build_wrapper(_realize))
def evolution_config()->Config: name = 'ultimatum' nepoch = 30000 n = 300 nrounds = 10*30 cutoff = 0.1 version = 6 nrunners = 10 return mkconfig(locals())
def stage_datasetS(m: Manager, index: int = 0, Nvalid: int = 5000, npoints: int = 100) -> DRef: """ Creates stabilized dataset """ def _config(): name = 'dataset3' nonlocal index, npoints N = int(5 * Nvalid / 3) min_allow_size = Nvalid out_examples = [promise, 'examples.bin'] out_barplots = [promise, 'barplots'] out_traceplot = [promise, 'traceplot.png'] out_inputs = [promise, 'inputs.json'] version = 2 return locals() def _make(b: Build): build_setoutpaths(b, 1) rref = run_dataset2(what=2, maxitems=mklens(b).N.val, index=index, interactive=False) assert isfile(mklens(rref).ref_df.examples.syspath) system( f"cp {mklens(rref).ref_df.ref_data.inputs.syspath} {mklens(b).out_inputs.syspath}" ) df = pd.read_csv(mklens(rref).df.syspath) makedirs(mklens(b).out_barplots.syspath) allowed = stabilize( df, npoints=mklens(b).npoints.val, path_barplot=lambda i: join( mklens(b).out_barplots.syspath, f'_plot_{i:03d}.png'), path_traceplot=mklens(b).out_traceplot.syspath, min_allow_size=mklens(b).min_allow_size.val) with open(mklens(rref).ref_df.examples.syspath, 'rb') as f: with open(mklens(b).out_examples.syspath, 'wb') as f_o: _read = fd2examples(f) _write = examples2fd(f_o) try: idx = 0 while True: example = _read() if idx in allowed: _write(example) idx += 1 except KeyboardInterrupt: raise except Exception as e: print(e) pass return mkdrv(m, mkconfig(_config()), match_only(), build_wrapper(_make))
def stage_df(m: Manager, ref_data: DRef): def _config(): nonlocal ref_data name = mklens(ref_data).name.val + '-df' examples = mklens(ref_data).out_examples.refpath out_df = [promise, 'df.csv'] return locals() def _make(b: Build): build_setoutpaths(b, 1) df = examples_dataframe(mklens(b).examples.syspath) df.to_csv(mklens(b).out_df.syspath) return mkdrv(m, mkconfig(_config()), match_only(), build_wrapper(_make))
def stage_vis(m: Manager, ref_df: DRef): def _config(): name = mklens(ref_df).name.val + '-vis' df = mklens(ref_df).out_df.refpath out_plot = [promise, 'plot.png'] title = mklens(ref_df).ref_data.name.val version = 3 return locals() def _make(b: Build): build_setoutpaths(b, 1) df = pd.read_csv(mklens(b).df.syspath) vis_bars(df, plot_fpath=mklens(b).out_plot.syspath, async_plot=None, plot_title=mklens(b).title.val) return mkdrv(m, mkconfig(_config()), match_latest(), build_wrapper(_make))
def stage_inputs(m:Manager, num_inputs:int=4, batch_size:int=5, index:int=1)->DRef: range_min = -100 range_max = 100 def _config(): name = 'dataset-inputs' nonlocal num_inputs, batch_size, range_min, range_max, index out_inputs = [promise, 'inputs.json'] version = 4 return locals() def _make(b:Build): build_setoutpaths(b, 1) builtin_inputs:dict={ # Ref("i0"):IVal(0), # Ref("i1"):IVal(1) } IMEMs:List[IMem] = [mkmap(_union(builtin_inputs, {Ref(f"i{i+len(builtin_inputs.keys())}"): IVal(randint(range_min,range_max)) for i in range(num_inputs)})) for _ in range(batch_size)] writejson(mklens(b).out_inputs.syspath, [imem2json(M) for M in IMEMs]) return mkdrv(m, mkconfig(_config()), match_only(), build_wrapper(_make))
def stage_hello(m: Manager) -> DRef: def _config() -> Config: name: str = 'hello-bin' src: RefPath = [hello_src, f'hello-{hello_version}'] return locals() def _make(b: Build) -> None: o: Path = build_outpath(b) with TemporaryDirectory() as tmp: copytree(mklens(b).src.syspath, join(tmp, 'src')) dirrw(Path(join(tmp, 'src'))) cwd = getcwd() try: chdir(join(tmp, 'src')) system('./configure --prefix=/usr') system('make') system(f'make install DESTDIR={o}') finally: chdir(cwd) return mkdrv(m, mkconfig(_config()), match_latest(), build_wrapper(_make))
def stage_dataset1(m:Manager, ref_inputs:DRef)->DRef: Wdef = 5 lib_impl = lib_arith lib_methods = [mn.val for mn in lib_impl.keys()] time2run_sec = int(0.5*60) def _config(): nonlocal Wdef,lib_methods,time2run_sec num_inputs = mklens(ref_inputs).num_inputs.val batch_size = mklens(ref_inputs).batch_size.val inputs = mklens(ref_inputs).out_inputs.refpath out_examples = [promise, 'examples.bin'] version = ['0'] return locals() def _make(b:Build): build_setoutpaths(b, 1) WLIB = mkwlib(lib_impl, Wdef) IMEMs = [json2imem(j) for j in readjson(mklens(b).inputs.syspath)] print(f"Inputs: {IMEMs}") i = 0 acc:List[Expr] = [] g = genexpr(WLIB, IMEMs) written_bytes = 0 time_start = time() with open(mklens(b).out_examples.syspath,'wb') as f: _add=examples2fd(f) while time()<time_start+mklens(b).time2run_sec.val: r,mem,vals,w = next(g) ival = vals[0] assert isinstance(ival[r], IVal) expr = gather(r,mem) acc.append(expr) i += 1 for j in range(len(IMEMs)): written_bytes+=_add(Example(IMEMs[j],expr,vals[j][r])) if i%300 == 0: print(f".. i {i} W {w} LAST_REF {r} WRBYTES {written_bytes}.. ") return mkdrv(m, mkconfig(_config()), match_only(), build_wrapper(_make))
def _setting(m:Manager)->DRef: n1=mkstage(m, {'name':'1', 'promise':[promise,'artifact']}) n2=mkstage(m, {'name':'2', 'promise':[promise,'artifact'], 'dict':{'d1':1} }) def _realize(b:Build): o=build_outpath(b) c=build_cattrs(b) assert isrefpath(mklens(b).maman.promise.refpath) assert isfile(mklens(b).papa.promise.syspath) assert o in mklens(b).promise.syspath assert o == mklens(b).syspath assert mklens(b).papa.name.val == '2' assert mklens(b).papa.dref == c.papa with open(mklens(b).promise.syspath,'w') as f: f.write('chickenpoop') return mkdrv(m, mkconfig({'name':'3', 'maman':n1, 'papa':n2, 'promise':[promise,'artifact'], }), matcher=match_some(), realizer=build_wrapper(_realize))
def mnist_config()->Config: learning_rate = 1e-3 num_epoches = 1 dataset:RefPath = [mnist_dataset, 'mnist.npz'] accuracy:PromisePath = [promise, 'accuracy.txt'] return mkconfig(locals())
def test_config2() -> None: try: c = mkconfig({'a': (3, 1)}) raise ShouldHaveFailed(f'RConfig {c} is surprizingly valid') except AssertionError: pass
def hello_config() -> Config: name = 'hello-bin' src = mklens(hello_src).src.refpath return mkconfig(locals())
def stage_dataset2(m:Manager, ref_inputs:DRef, gather_depth:Optional[int]=999, Wdef:int=5, maxtime_sec:Optional[int]=None, maxitems:Optional[int]=None)->DRef: lib_impl = lib_arith lib_methods = [mn.val for mn in lib_impl.keys()] assert (maxtime_sec is not None) or (maxitems is not None), \ "At least one stop criteria should be defined" time2run_sec:int = maxsize if maxtime_sec is None else maxtime_sec maxitems:int = maxsize if maxitems is None else maxitems def _config(): nonlocal Wdef,lib_methods,time2run_sec,maxitems,gather_depth num_inputs = mklens(ref_inputs).num_inputs.val batch_size = mklens(ref_inputs).batch_size.val name=f'dataset2-ni{num_inputs}-bs{batch_size}-Wd{Wdef}-mi{maxitems}' inputs = mklens(ref_inputs).out_inputs.refpath out_examples = [promise, 'examples.bin'] version = ['21'] return locals() def _make(b:Build): build_setoutpaths(b, 1) WLIB = mkwlib(lib_impl, Wdef) IMEMs = [json2imem(j) for j in readjson(mklens(b).inputs.syspath)] print(f"Inputs: {IMEMs}") i = 0 # acc:List[Expr] = [] g = genexpr(WLIB, IMEMs) written_bytes = 0 written_items = 0 time_start = time() acci=set() hb=time() with open(mklens(b).out_examples.syspath,'wb') as f: _add=examples2fd(f) while time()<time_start+mklens(b).time2run_sec.val and \ written_items<mklens(b).maxitems.val: # gt0=time() r,mem,imems,exprw = next(g) # gt1=time() # print('gen time', gt1-gt0) assert isinstance(imems[0][r], IVal), f"{imems[0][r]}" i += 1 gi = 0 gexpr = gengather(r,mem) # gg0=time() exprs=[] for expr in gexpr: exprs.append(expr) if gather_depth is not None: exprs=exprs[-gather_depth:] else: exprs=[exprs[randint(0,len(exprs)-1)]] for expr in exprs: er=extrefs(expr) ds=decls(expr) # acc.append(expr) for j in range(len(IMEMs)): if len(ds)>0: # print(gi, j, list(inps.keys()), print_expr(expr)) inps:IMem = TMap({k:imems[j][k] for k in er}) acci |= set(inps.values()) written_bytes+=_add(Example(inps,expr,imems[j][r])) written_items+=1 hb2=time() if written_items%100 == 0: print(f".. NW {written_items} W {exprw[r]} DEP {depth(expr)} " f"LAST_REF {r} WRBYTES {written_bytes // (1024) }K " f"INPSZ {len(acci)} TIME {hb2-hb} " f"VM {virtual_memory().used // 1024 // 1024}M") hb=hb2 gi+=1 # gg1=time() # print('gather time', gg1-gg0) return mkdrv(m, mkconfig(_config()), match_only(), build_wrapper(_make))
def summary_config(evolution:DRef)->Config: name = 'summary' history_refpath = [evolution, 'history.json'] return mkconfig(locals())
def _config() -> Config: c = deepcopy(config) c['artifact'] = [promise_strength, 'artifact'] return mkconfig(c)