def test_realize_nondetermenistic() -> None: with setup_storage2('test_realize_nondetermenistic') as (T, S): DATA: int n1: DRef n2: DRef n3: DRef def _gen(i, tag) -> int: nonlocal DATA return DATA def _setup(m): nonlocal n1, n2, n3 n1 = mkstage(m, {'a': '1'}) n2 = mkstage(m, {'maman': n1}, _gen) n3 = mkstage(m, {'papa': n2}) return n3 DATA = 1 rref1 = realize(instantiate(_setup, S=S)) DATA = 2 rref2 = realize(instantiate(_setup, S=S), force_rebuild=[n2]) assert len(list(drefrrefs(n1, S))) == 1 assert len(list(drefrrefs(n2, S))) == 2 assert len(list(drefrrefs(n3, S))) == 2 assert rref1 != rref2 n2_gr1 = store_deref(rref1, n2, S) n2_gr2 = store_deref(rref2, n2, S) assert n2_gr1 != n2_gr2
def test_realize_readonly() -> None: with setup_storage2('test_realize_readonly') as (T, S): rref1 = realize(instantiate(mkstage, {'a': '1'}, S=S)) try: with open(join(store_rref2path(rref1, S), 'newfile'), 'w') as f: f.write('foo') raise ShouldHaveFailed('No write-protection??') except OSError: pass try: rmtree(store_rref2path(rref1, S)) raise ShouldHaveFailed('No remove-protection??') except OSError: pass def _realize(b: Build): with open(join(build_outpath(b), 'exe'), 'w') as f: f.write('#!/bin/sh\necho "Fooo"') chmod(join(build_outpath(b), 'exe'), S_IWRITE | S_IREAD | S_IEXEC) rref2 = realize( instantiate(mkdrv, Config({}), match_some(), build_wrapper(_realize), S=S)) assert pipe_stdout([join(store_rref2path(rref2,S),'exe')])=='Fooo\n', \ "Did we lost exec permission?"
def test_mksymlink() -> None: with setup_storage2('test_mksymlink') as (T, S): tp = T def _setting1(m: Manager) -> DRef: return mkstage(m, {'a': '1'}, lambda i, tag: 33, buildtime=False) def _setting2(m: Manager) -> DRef: return mkstage(m, {'a': '1'}, lambda i, tag: 42, buildtime=True) clo = instantiate(_setting1, S=S) rref = realize(clo) clo2 = instantiate(_setting2, S=S) rref2 = realize(clo2, force_rebuild=True) assert clo.dref == clo2.dref assert rref2 != rref s = mksymlink(rref, tgtpath=tp, name='thelink', S=S) assert islink(s) assert not isfile(join(s, '__buildtime__.txt')) assert tp in s s2 = mksymlink(rref2, tgtpath=tp, name='thelink', S=S) assert islink(s2) assert isfile(join(s2, '__buildtime__.txt')) assert tp in s assert s2 != s, "s2 should have timestamp" s3 = mksymlink(rref2, tgtpath=tp, name='thelink', withtime=False, S=S) assert s3 == s
def test_repeated_realize() -> None: with setup_storage2('test_repeated_realize') as (T, S): def _setting(m: Manager) -> DRef: return mkstage(m, {'a': '1'}) clo = instantiate(_setting, S=S) rref1 = realize(clo) rref2 = realize(clo) rref3 = realize(clo, force_rebuild=[clo.dref]) assert rref1 == rref2 and rref2 == rref3 try: rref3 = realize(clo, force_rebuild='Foobar') # type:ignore raise ShouldHaveFailed except AssertionError: pass
def test_realize_dependencies() -> None: with setup_storage2('test_realize_dependencies') as (T, S): n1 = None n2 = None n3 = None toplevel = None def _setup(m): nonlocal n1, n2, n3, toplevel n1 = mkstage(m, {'a': 1}) n2 = mkstage(m, {'parent1': n1}) n3 = mkstage(m, {'parent2': n1}) toplevel = mkstage(m, {'n2': n2, 'n3': n3}) return toplevel rref = realize(instantiate(_setup, S=S)) assert_valid_rref(rref) assert n1 is not None assert n2 is not None assert n3 is not None assert toplevel is not None c = store_config(rref, S) assert_valid_config(c) cro = store_cattrs(rref, S) assert_valid_dref(getattr(cro, 'n2')) assert_valid_dref(getattr(cro, 'n3')) assert set(store_deps([n1], S)) == set([]) assert set(store_deps([n2, n3], S)) == set([n1]) assert set(store_deps([toplevel], S)) == set([n2, n3]) assert store_deepdeps([n1], S) == set([]) assert store_deepdeps([n2, n3], S) == set([n1]) assert store_deepdeps([toplevel], S) == set([n1, n2, n3])
def run(mode: int = 2, interactive: bool = True): maxitems = 5000 Wdef = 1 gather_depth = None num_inputs = 2 if mode == 1: batch_size = 5 elif mode == 2: batch_size = 1 else: assert False, 'Invalid mode' def _stage(m: Manager): inp = stage_inputs(m, num_inputs=num_inputs, batch_size=batch_size) ds = stage_dataset2(m, inp, Wdef=Wdef, maxitems=maxitems, gather_depth=gather_depth) df = stage_df(m, ds) dsvis = stage_vis(m, df) return dsvis rref = realize(instantiate(_stage), force_rebuild=interactive) linkrref(rref, '_results') if interactive: system(f"feh {mklens(rref).out_plot.syspath} &") return rref
def test_fetchurl(): with setup_storage('test_fetchurl'): with TemporaryDirectory() as tmp: mockwget = join(tmp, 'mockwget') mockdata = join(tmp, 'mockdata') with open(mockdata, 'w') as f: f.write('blala') system(f'tar -zcvf {mockdata}.tar.gz {mockdata}') with open(mockwget, 'w') as f: f.write(f"#!/bin/sh\n") f.write(f"mv {mockdata}.tar.gz $3\n") chmod(mockwget, stat(mockwget).st_mode | S_IEXEC) wanted_sha256 = pipe_stdout([SHA256SUM, f"{mockdata}.tar.gz"]).split()[0] import pylightnix.stages.fetch oldwget = pylightnix.stages.fetch.WGET try: pylightnix.stages.fetch.WGET = lambda: mockwget clo = instantiate(fetchurl, url='mockwget://result.tar.gz', filename='validname.tar.gz', sha256=wanted_sha256) rref = realize(clo) finally: pylightnix.stages.fetch.WGET = oldwget
def test_diff(): with setup_storage('test_find') as s: s1=partial(mkstage, config={'name':'1'}, nondet=lambda:42) s2=partial(mkstage, config={'name':'2'}, nondet=lambda:33) dref1=instantiate(s1).dref rref2=realize(instantiate(s2)) diff(dref1, rref2) diff(dref1, s2)
def test_minimal_closure() -> None: with setup_storage2('test_minimal_closure') as (T, S): def _somenode(m): return mkstage(m, {'a': 0}) def _anothernode(m): n1 = mkstage(m, {'a': 1}) n2 = _somenode(m) n3 = mkstage(m, {'maman': n1, 'papa': n2}) return n3 rref1 = realize(instantiate(_somenode, S=S)) rref = realize(instantiate(_anothernode, S=S)) rref2 = store_deref(rref, store_cattrs(rref, S=S).papa, S=S)[Tag('out')] assert rref1 == rref2, '''
def test_no_dref_deps_without_realizers() -> None: with setup_storage2('test_no_dref_deps_without_realizers') as (T, S): try: clo = instantiate(mkstage, {'a': 1}, S=S) _ = realize(instantiate(mkstage, {'maman': clo.dref}, S=S)) raise ShouldHaveFailed("We shouldn't share DRefs across managers") except AssertionError: pass
def test_mkfile() -> None: with setup_storage('test_mkfile'): def _setting(m: Manager, nm) -> DRef: return mkfile(m, Name('foo'), bytes((nm or 'bar').encode('utf-8')), nm) rref1 = realize(instantiate(_setting, None)) with open(join(store_rref2path(rref1), 'foo'), 'r') as f: bar = f.read() assert bar == 'bar' rref2 = realize(instantiate(_setting, 'baz')) with open(join(store_rref2path(rref2), 'baz'), 'r') as f: baz = f.read() assert baz == 'baz' assert rref1 != rref2
def test_find(): with setup_storage('test_find') as s: s1=partial(mkstage, config={'name':'1'}, nondet=lambda:42) s2=partial(mkstage, config={'name':'2'}, nondet=lambda:33) rref1=realize(instantiate(s1)) sleep(0.1) now=parsetime(timestring()) rref2=realize(instantiate(s2)) rrefs=find() assert set(rrefs)==set([rref1,rref2]) rrefs=find(name='1') assert rrefs==[rref1] rrefs=find(name=s2) assert rrefs==[rref2] rrefs=find(newer=-10) assert len(rrefs)==2 rrefs=find(newer=now) assert rrefs==[rref2]
def test_gc() -> None: with setup_storage2('test_gc') as (T, S): def _node1(m: Manager) -> DRef: return mkstage(m, {'name': '1'}) def _node2(m: Manager) -> DRef: return mkstage(m, {'name': '2', 'maman': _node1(m)}) def _node3(m: Manager) -> DRef: return mkstage(m, {'name': '3', 'maman': _node1(m)}) r1 = realize(instantiate(_node1, S=S)) r2 = realize(instantiate(_node2, S=S)) r3 = realize(instantiate(_node3, S=S)) rm_drefs, rm_rrefs = store_gc([], [r2], S) assert rm_drefs == {rref2dref(r) for r in [r3]} assert rm_rrefs == {x for x in [r3]}
def test_du(): with setup_storage('test_du') as s: usage=du() assert usage=={} clo=instantiate(mkstage, {'name':'1'}, lambda:42) usage=du() assert clo.dref in usage assert usage[clo.dref][0]>0 assert usage[clo.dref][1]=={} rref=realize(clo) usage=du() assert rref in usage[clo.dref][1] assert usage[clo.dref][1][rref]>0
def test_mknode_with_artifacts(d, a) -> None: with setup_storage('test_mknode_with_artifacts'): def _setting(m: Manager) -> DRef: return mknode(m, config_dict=d, artifacts=a) cl = instantiate(_setting) assert len(cl.derivations) == 1 rref = realize(instantiate(_setting)) for nm, val in a.items(): assert isfile(join(store_rref2path(rref),nm)), \ f"RRef {rref} doesn't contain artifact {nm}"
def test_realized() -> None: with setup_storage('test_realized'): def _setting(m: Manager, assume_realized: bool) -> DRef: def _realize(b: Build): if assume_realized: raise ShouldHaveFailed('Should not call the real realizer') return build_outpath(b) return mkdrv(m, mkconfig({'name': '1'}), match_some(), build_wrapper(_realize)) dref = instantiate(realized(_setting), assume_realized=True) try: rref = realize(dref) raise ShouldHaveFailed('Should not be realized') except AssertionError: pass rref = realize(instantiate(_setting, assume_realized=False)) rref2 = realize(instantiate(realized(_setting), assume_realized=True)) assert rref == rref2
def load3(index, allow_realize: bool = True): """ Loads the stabilized dataset. Raises LookupError if realization is forbidden and the data are absent. """ if allow_realize: stage = stage_datasetS else: def _err(*args, **kwargs): raise LookupError( "Attempting to realize dataset section with index {index}") stage = redefine(stage_datasetS, new_realizer=_err) return realize(instantiate(stage, index=index, Nvalid=N1, npoints=2000))
def test_lens_closures(): with setup_storage('test_lens_closures'): def _setting(m:Manager)->DRef: n1=mkstage(m, {'name':'1', 'x':33, 'promise':[promise,'artifact']}) n2=mkstage(m, {'name':'2', 'papa':n1, 'dict':{'d1':1} }) n3=mkstage(m, {'name':'3', 'maman':n2 }) return n3 clo=instantiate(_setting) assert isclosure(clo) rref=realize(mklens(clo).maman.papa.closure) assert mklens(rref).x.val==33 assert open(mklens(rref).promise.syspath).read()=='0'
def test_linkrrefs() -> None: with setup_storage2('test_linkrefs') as (T, S): s1 = partial(mkstage, config={ 'name': '1', 'promise': [promise, 'artifact'] }) rref1 = realize(instantiate(s1, S=S)) l = linkrrefs([rref1, rref1], destdir=S, S=S) assert len(l) == 2 assert str(l[0]) == join(S, 'result-1') assert islink(join(S, 'result-1')) l = linkrrefs([rref1], destdir=S, withtime=True, S=S) assert S in l[0]
def test_promise() -> None: with setup_storage2('test_promise') as (T, S): def _setting(m: Manager, fullfill: bool) -> DRef: n1 = mkstage(m, {'name': '1', 'promise': [promise, 'artifact']}) def _realize(b: Build): o = build_outpath(b) c = build_cattrs(b) assert b.dref in c.promise assert n1 in store_cattrs(c.maman, S).promise assert build_path(b, c.promise) == join(o, 'uber-artifact') assert build_path(b, store_cattrs(c.maman, S).promise) == build_path( b, c.maman_promise) if fullfill: with open(build_path(b, c.promise), 'w') as f: f.write('chickenpoop') return mkdrv(m, mkconfig({ 'name': '2', 'maman': n1, 'promise': [promise, 'uber-artifact'], 'maman_promise': [n1, 'artifact'] }), matcher=match_some(), realizer=build_wrapper(_realize)) try: rref = realize(instantiate(_setting, False, S=S)) raise ShouldHaveFailed('Promise trigger') except AssertionError: pass rref = realize(instantiate(_setting, True, S=S)) assert_valid_rref(rref)
def test_bashlike(): with setup_storage('test_bashlike'): clo=instantiate(mkstage, {'a':1}, lambda:42) rref1=realize(clo, force_rebuild=[clo.dref]) rref2=realize(clo, force_rebuild=[clo.dref]) assert 'artifact' in lsref(rref1) assert 'context.json' in lsref(rref1) assert '__buildtime__.txt' in lsref(rref1) h1,_,_=unrref(rref1) h2,_,_=unrref(rref2) assert len(lsref(clo.dref))==2 assert h1 in lsref(clo.dref) assert h2 in lsref(clo.dref) assert '42' in catref(rref1,['artifact']) try: lsref('foobar') # type:ignore raise ShouldHaveFailed('should reject garbage') except AssertionError: pass try: catref(clo.dref, ['artifact']) # type:ignore raise ShouldHaveFailed('notimpl') except AssertionError: pass
def test_rmdref(): with setup_storage('test_rmdref') as s: clo=instantiate(mkstage, {'a':1}, lambda:42) drefpath=store_dref2path(clo.dref) rref1=realize(clo, force_rebuild=[clo.dref]) rrefpath=store_rref2path(rref1) assert isdir(rrefpath) rmref(rref1) assert not isdir(rrefpath) assert isdir(drefpath) rmref(clo.dref) assert not isdir(drefpath) try: rmref('asdasd') # type:ignore raise ShouldHaveFailed('shoud reject garbage') except AssertionError: pass
def test_fetchlocal2(): with setup_storage('test_fetclocal') as tmp: mockdata = join(tmp, 'mockdata') with open(mockdata, 'w') as f: f.write('dogfood') wanted_sha256 = pipe_stdout([SHA256SUM, "mockdata"], cwd=tmp).split()[0] rref = realize( instantiate(fetchlocal, path=mockdata, sha256=wanted_sha256, mode='as-is')) assert isrref(rref) assert isfile(join(store_rref2path(rref), 'mockdata')) assert isfile(mklens(rref).out_path.syspath)
def test_fetchlocal(): with setup_storage('test_fetclocal') as tmp: mockdata = join(tmp, 'mockdata') with open(mockdata, 'w') as f: f.write('dogfood') system(f"tar -C '{tmp}' -zcvf {tmp}/mockdata.tar.gz mockdata") wanted_sha256 = pipe_stdout([SHA256SUM, "mockdata.tar.gz"], cwd=tmp).split()[0] rref = realize( instantiate(fetchlocal, path=mockdata + '.tar.gz', filename='validname.tar.gz', sha256=wanted_sha256)) assert isrref(rref) assert isfile(join(store_rref2path(rref), 'mockdata'))
def test_path2dref() -> None: with setup_storage2('test_path2dref') as (T, S): s1 = partial(mkstage, config={ 'name': '1', 'promise': [promise, 'artifact'] }) clo1 = instantiate(s1, S=S) dref1 = clo1.dref rref1 = realize(clo1) dref2 = path2dref(store_dref2path(rref2dref(rref1), S=S)) assert dref1 == dref2 l = linkdref(dref1, S, 'result_dref', S=S) assert path2dref(Path(l)) == dref1 dref3 = path2dref(Path("/foo/00000000000000000000000000000000-bar")) assert dref3 == 'dref:00000000000000000000000000000000-bar' for x in [path2dref(Path('')), path2dref(Path('foo'))]: assert x is None
def test_redefine() -> None: with setup_storage('test_redefine'): def _setting(m: Manager) -> DRef: return mknode(m, { 'name': 'foo', 'bar': 'baz', 'output': [promise, 'f'] }, {Name('f'): bytes(('umgh').encode('utf-8'))}) def _nc(c): mklens(c).bar.val = 42 _setting2 = redefine(_setting, new_config=_nc) rref = realize(instantiate(_setting2)) assert mklens(rref).bar.val == 42 assert tryread(mklens(rref).output.syspath) == 'umgh'
def test_repl_race(): with setup_storage('test_repl_recursion'): def _setting(m: Manager) -> DRef: n1 = mkstage(m, {'a': '1'}) n2 = mkstage(m, {'maman': n1}) return n2 clo = instantiate(_setting) rh = repl_realize(clo, force_interrupt=True) assert repl_rref(rh) is None assert rh.dref == clo.dref clo2 = instantiate(_setting) rref2 = realize(clo2) # Realize dref while repl_realizing same dref repl_cancel(rh) assert_valid_rref(rref2)
def test_repl_globalCancel(): with setup_storage('test_repl_globalCancel'): n1: DRef n2: DRef def _setting(m: Manager) -> DRef: nonlocal n1, n2 n1 = mkstage(m, {'a': '1'}) n2 = mkstage(m, {'maman': n1}) return n2 rh = repl_realize(instantiate(_setting), force_interrupt=True) assert repl_rref(rh) is None repl_cancel() assert rh.gen is None rref = realize(instantiate(_setting)) assert isrref(rref)
def test_recursive_realize_with_another_manager() -> None: with setup_storage2('test_recursive_realize_with_another_manager') as (T, S): rref_inner = None def _setup_inner(m): return mkstage(m, {'foo': 'bar'}) def _setup_outer(m): nonlocal rref_inner rref_inner = realize(instantiate(_setup_inner, S=S)) return mkstage(m, {'baz': mklens(rref_inner, S=S).foo.val}) rref = realize(instantiate(_setup_outer, S=S)) assert rref_inner is not None assert len(store_deepdeps([rref2dref(rref)], S=S)) == 0 assert len(store_deepdeps([rref2dref(rref_inner)], S=S)) == 0 assert mklens(rref_inner, S=S).foo.val == 'bar' assert mklens(rref, S=S).baz.val == 'bar'
def test_path2rref() -> None: with setup_storage2('test_path2rref') as (T, S): s1 = partial(mkstage, config={ 'name': '1', 'promise': [promise, 'artifact'] }) rref1 = realize(instantiate(s1, S=S)) rref2 = path2rref(store_rref2path(rref1, S)) assert rref1 == rref2 l = mksymlink(rref1, S, 'result', S=S) assert path2rref(Path(l)) == rref1 rref3 = path2rref( Path( "/foo/00000000000000000000000000000000-bar/11111111111111111111111111111111" )) assert rref3 == 'rref:11111111111111111111111111111111-00000000000000000000000000000000-bar' for x in [path2rref(Path('')), path2rref(Path('foo'))]: assert x is None