def _make(b: Build): build_setoutpaths(b, 1) df = pd.read_csv(mklens(b).df.syspath) vis_bars(df, plot_fpath=mklens(b).out_plot.syspath, async_plot=None, plot_title=mklens(b).title.val)
def sample(N:int=5)->List[Example]: """ Samples `N` examples from the dataset """ acc:List[Example]=[] while len(acc)<N: i=int(choice(SECTIONS, size=1)[0]) n=0 print('Loading section', i) try: with open(mklens(load3(i,False)).out_examples.syspath, 'rb') as ef: _,_skip=fd2examples2(ef) try: while True: _skip() n+=1 except IndexError: pass except LookupError: print('Woops, no realization') continue x=randint(0,n-1) print('.. it contains', n, 'records, we take', x) with open(mklens(load3(i,False)).out_examples.syspath, 'rb') as ef: _next,_skip=fd2examples2(ef) for i in range(x-1): _skip() acc.append(_next()) return acc
def _config(): name = mklens(ref_df).name.val + '-vis' df = mklens(ref_df).out_df.refpath out_plot = [promise, 'plot.png'] title = mklens(ref_df).ref_data.name.val version = 3 return locals()
def _make(b: Build): build_setoutpaths(b, 1) rref = run_dataset2(what=2, maxitems=mklens(b).N.val, index=index, interactive=False) assert isfile(mklens(rref).ref_df.examples.syspath) system( f"cp {mklens(rref).ref_df.ref_data.inputs.syspath} {mklens(b).out_inputs.syspath}" ) df = pd.read_csv(mklens(rref).df.syspath) makedirs(mklens(b).out_barplots.syspath) allowed = stabilize( df, npoints=mklens(b).npoints.val, path_barplot=lambda i: join( mklens(b).out_barplots.syspath, f'_plot_{i:03d}.png'), path_traceplot=mklens(b).out_traceplot.syspath, min_allow_size=mklens(b).min_allow_size.val) with open(mklens(rref).ref_df.examples.syspath, 'rb') as f: with open(mklens(b).out_examples.syspath, 'wb') as f_o: _read = fd2examples(f) _write = examples2fd(f_o) try: idx = 0 while True: example = _read() if idx in allowed: _write(example) idx += 1 except KeyboardInterrupt: raise except Exception as e: print(e) pass
def _config(): nonlocal Wdef,lib_methods,time2run_sec num_inputs = mklens(ref_inputs).num_inputs.val batch_size = mklens(ref_inputs).batch_size.val inputs = mklens(ref_inputs).out_inputs.refpath out_examples = [promise, 'examples.bin'] version = ['0'] return locals()
def _config(): nonlocal Wdef,lib_methods,time2run_sec,maxitems,gather_depth num_inputs = mklens(ref_inputs).num_inputs.val batch_size = mklens(ref_inputs).batch_size.val name=f'dataset2-ni{num_inputs}-bs{batch_size}-Wd{Wdef}-mi{maxitems}' inputs = mklens(ref_inputs).out_inputs.refpath out_examples = [promise, 'examples.bin'] version = ['21'] return locals()
def _make(b:Build): build_setoutpaths(b, 1) WLIB = mkwlib(lib_impl, Wdef) IMEMs = [json2imem(j) for j in readjson(mklens(b).inputs.syspath)] print(f"Inputs: {IMEMs}") i = 0 # acc:List[Expr] = [] g = genexpr(WLIB, IMEMs) written_bytes = 0 written_items = 0 time_start = time() acci=set() hb=time() with open(mklens(b).out_examples.syspath,'wb') as f: _add=examples2fd(f) while time()<time_start+mklens(b).time2run_sec.val and \ written_items<mklens(b).maxitems.val: # gt0=time() r,mem,imems,exprw = next(g) # gt1=time() # print('gen time', gt1-gt0) assert isinstance(imems[0][r], IVal), f"{imems[0][r]}" i += 1 gi = 0 gexpr = gengather(r,mem) # gg0=time() exprs=[] for expr in gexpr: exprs.append(expr) if gather_depth is not None: exprs=exprs[-gather_depth:] else: exprs=[exprs[randint(0,len(exprs)-1)]] for expr in exprs: er=extrefs(expr) ds=decls(expr) # acc.append(expr) for j in range(len(IMEMs)): if len(ds)>0: # print(gi, j, list(inps.keys()), print_expr(expr)) inps:IMem = TMap({k:imems[j][k] for k in er}) acci |= set(inps.values()) written_bytes+=_add(Example(inps,expr,imems[j][r])) written_items+=1 hb2=time() if written_items%100 == 0: print(f".. NW {written_items} W {exprw[r]} DEP {depth(expr)} " f"LAST_REF {r} WRBYTES {written_bytes // (1024) }K " f"INPSZ {len(acci)} TIME {hb2-hb} " f"VM {virtual_memory().used // 1024 // 1024}M") hb=hb2 gi+=1
def test_lens_closures(): with setup_storage('test_lens_closures'): def _setting(m:Manager)->DRef: n1=mkstage(m, {'name':'1', 'x':33, 'promise':[promise,'artifact']}) n2=mkstage(m, {'name':'2', 'papa':n1, 'dict':{'d1':1} }) n3=mkstage(m, {'name':'3', 'maman':n2 }) return n3 clo=instantiate(_setting) assert isclosure(clo) rref=realize(mklens(clo).maman.papa.closure) assert mklens(rref).x.val==33 assert open(mklens(rref).promise.syspath).read()=='0'
def test_redefine() -> None: with setup_storage('test_redefine'): def _setting(m: Manager) -> DRef: return mknode(m, { 'name': 'foo', 'bar': 'baz', 'output': [promise, 'f'] }, {Name('f'): bytes(('umgh').encode('utf-8'))}) def _nc(c): mklens(c).bar.val = 42 _setting2 = redefine(_setting, new_config=_nc) rref = realize(instantiate(_setting2)) assert mklens(rref).bar.val == 42 assert tryread(mklens(rref).output.syspath) == 'umgh'
def print_size()->None: total=0 for index in SECTIONS: try: sz=stat(mklens(load3(index,False)).out_examples.syspath).st_size total+=sz print(f"Index {index}: {sz/1024/1024:.1f}M") except LookupError: print(f"Woops, no dataset index {index}") print(f"Total: {total/1024/1024:.1f}M")
def test_recursive_realize_with_another_manager() -> None: with setup_storage2('test_recursive_realize_with_another_manager') as (T, S): rref_inner = None def _setup_inner(m): return mkstage(m, {'foo': 'bar'}) def _setup_outer(m): nonlocal rref_inner rref_inner = realize(instantiate(_setup_inner, S=S)) return mkstage(m, {'baz': mklens(rref_inner, S=S).foo.val}) rref = realize(instantiate(_setup_outer, S=S)) assert rref_inner is not None assert len(store_deepdeps([rref2dref(rref)], S=S)) == 0 assert len(store_deepdeps([rref2dref(rref_inner)], S=S)) == 0 assert mklens(rref_inner, S=S).foo.val == 'bar' assert mklens(rref, S=S).baz.val == 'bar'
def _make(b:Build): build_setoutpaths(b, 1) builtin_inputs:dict={ # Ref("i0"):IVal(0), # Ref("i1"):IVal(1) } IMEMs:List[IMem] = [mkmap(_union(builtin_inputs, {Ref(f"i{i+len(builtin_inputs.keys())}"): IVal(randint(range_min,range_max)) for i in range(num_inputs)})) for _ in range(batch_size)] writejson(mklens(b).out_inputs.syspath, [imem2json(M) for M in IMEMs])
def _make(b: Build) -> None: o: Path = build_outpath(b) with TemporaryDirectory() as tmp: copytree(mklens(b).src.syspath, join(tmp, 'src')) dirrw(Path(join(tmp, 'src'))) cwd = getcwd() try: chdir(join(tmp, 'src')) system('./configure --prefix=/usr') system('make') system(f'make install DESTDIR={o}') finally: chdir(cwd)
def _realize(b:Build): o=build_outpath(b) c=build_cattrs(b) assert isrefpath(mklens(b).maman.promise.refpath) assert isfile(mklens(b).papa.promise.syspath) assert o in mklens(b).promise.syspath assert o == mklens(b).syspath assert mklens(b).papa.name.val == '2' assert mklens(b).papa.dref == c.papa with open(mklens(b).promise.syspath,'w') as f: f.write('chickenpoop')
def _make(b:Build): build_setoutpaths(b, 1) WLIB = mkwlib(lib_impl, Wdef) IMEMs = [json2imem(j) for j in readjson(mklens(b).inputs.syspath)] print(f"Inputs: {IMEMs}") i = 0 acc:List[Expr] = [] g = genexpr(WLIB, IMEMs) written_bytes = 0 time_start = time() with open(mklens(b).out_examples.syspath,'wb') as f: _add=examples2fd(f) while time()<time_start+mklens(b).time2run_sec.val: r,mem,vals,w = next(g) ival = vals[0] assert isinstance(ival[r], IVal) expr = gather(r,mem) acc.append(expr) i += 1 for j in range(len(IMEMs)): written_bytes+=_add(Example(IMEMs[j],expr,vals[j][r])) if i%300 == 0: print(f".. i {i} W {w} LAST_REF {r} WRBYTES {written_bytes}.. ")
def loadsection(si:int)->List[Example]: print(f"Loading section {si}") acc:list=[] try: with open(mklens(load3(si,False)).out_examples.syspath, 'rb') as ef: _next=fd2examples(ef) e=_next() print(type(e)) s=pickle_dumps(e, protocol=HIGHEST_PROTOCOL) print(s) # while True: # acc.append(_next()) except KeyboardInterrupt: raise except LookupError: pass return acc[:10]
def test_fetchlocal2(): with setup_storage('test_fetclocal') as tmp: mockdata = join(tmp, 'mockdata') with open(mockdata, 'w') as f: f.write('dogfood') wanted_sha256 = pipe_stdout([SHA256SUM, "mockdata"], cwd=tmp).split()[0] rref = realize( instantiate(fetchlocal, path=mockdata, sha256=wanted_sha256, mode='as-is')) assert isrref(rref) assert isfile(join(store_rref2path(rref), 'mockdata')) assert isfile(mklens(rref).out_path.syspath)
def test_lens(): with setup_storage('test_lens'): def _setting(m:Manager)->DRef: n1=mkstage(m, {'name':'1', 'promise':[promise,'artifact']}) n2=mkstage(m, {'name':'2', 'promise':[promise,'artifact'], 'dict':{'d1':1} }) def _realize(b:Build): o=build_outpath(b) c=build_cattrs(b) assert isrefpath(mklens(b).maman.promise.refpath) assert isfile(mklens(b).papa.promise.syspath) assert o in mklens(b).promise.syspath assert o == mklens(b).syspath assert mklens(b).papa.name.val == '2' assert mklens(b).papa.dref == c.papa with open(mklens(b).promise.syspath,'w') as f: f.write('chickenpoop') return mkdrv(m, mkconfig({'name':'3', 'maman':n1, 'papa':n2, 'promise':[promise,'artifact'], }), matcher=match_some(), realizer=build_wrapper(_realize)) clo=instantiate(_setting) assert isrefpath(mklens(clo.dref).maman.promise.refpath) assert isdir(mklens(clo.dref).syspath) rref=realize(clo) assert_valid_rref(rref) assert isrefpath(mklens(rref).maman.promise.refpath) assert isfile(mklens(rref).maman.promise.syspath) assert mklens(rref).rref == rref assert isrefpath(mklens(rref).papa.promise.refpath) assert mklens(rref).papa.dict.d1.val == 1 assert mklens(rref).dref == clo.dref assert isdir(mklens(rref).syspath) try: print(mklens(clo.dref).maman.promise.syspath) raise ShouldHaveFailed() except AssertionError: pass try: print(mklens(rref).papa.dict.d1.get('xxx')) raise ShouldHaveFailed() except AssertionError: pass try: print(mklens(rref).papa.dict.d1.syspath) raise ShouldHaveFailed() except AssertionError: pass try: mklens(rref).papa.dict.d1.val=42 # can't mutate rref raise ShouldHaveFailed() except AssertionError: pass d={'foo':'foo','bar':'bar'} mklens(d).foo.val='zzz' assert d['foo']=='zzz' try: mklens(d).x.val=42 # can't set new values raise ShouldHaveFailed() except AssertionError: pass mklens(d).bar.val+='33' assert d['bar']=='bar33'
def _make(b: Build): build_setoutpaths(b, 1) df = examples_dataframe(mklens(b).examples.syspath) df.to_csv(mklens(b).out_df.syspath)
def _setup_outer(m): nonlocal rref_inner rref_inner = realize(instantiate(_setup_inner, S=S)) return mkstage(m, {'baz': mklens(rref_inner, S=S).foo.val})
def hello_config() -> Config: name = 'hello-bin' src = mklens(hello_src).src.refpath return mkconfig(locals())
def load(what: int) -> DataFrame: rref = run(what, interactive=False) df = pd.read_csv(mklens(rref).df.syspath) return df
def _config(): nonlocal ref_data name = mklens(ref_data).name.val + '-df' examples = mklens(ref_data).out_examples.refpath out_df = [promise, 'df.csv'] return locals()
def _nc(c): mklens(c).bar.val = 42