def test_realize_readonly() -> None: with setup_storage2('test_realize_readonly') as (T, S): rref1 = realize(instantiate(mkstage, {'a': '1'}, S=S)) try: with open(join(store_rref2path(rref1, S), 'newfile'), 'w') as f: f.write('foo') raise ShouldHaveFailed('No write-protection??') except OSError: pass try: rmtree(store_rref2path(rref1, S)) raise ShouldHaveFailed('No remove-protection??') except OSError: pass def _realize(b: Build): with open(join(build_outpath(b), 'exe'), 'w') as f: f.write('#!/bin/sh\necho "Fooo"') chmod(join(build_outpath(b), 'exe'), S_IWRITE | S_IREAD | S_IEXEC) rref2 = realize( instantiate(mkdrv, Config({}), match_some(), build_wrapper(_realize), S=S)) assert pipe_stdout([join(store_rref2path(rref2,S),'exe')])=='Fooo\n', \ "Did we lost exec permission?"
def test_mksymlink() -> None: with setup_storage2('test_mksymlink') as (T, S): tp = T def _setting1(m: Manager) -> DRef: return mkstage(m, {'a': '1'}, lambda i, tag: 33, buildtime=False) def _setting2(m: Manager) -> DRef: return mkstage(m, {'a': '1'}, lambda i, tag: 42, buildtime=True) clo = instantiate(_setting1, S=S) rref = realize(clo) clo2 = instantiate(_setting2, S=S) rref2 = realize(clo2, force_rebuild=True) assert clo.dref == clo2.dref assert rref2 != rref s = mksymlink(rref, tgtpath=tp, name='thelink', S=S) assert islink(s) assert not isfile(join(s, '__buildtime__.txt')) assert tp in s s2 = mksymlink(rref2, tgtpath=tp, name='thelink', S=S) assert islink(s2) assert isfile(join(s2, '__buildtime__.txt')) assert tp in s assert s2 != s, "s2 should have timestamp" s3 = mksymlink(rref2, tgtpath=tp, name='thelink', withtime=False, S=S) assert s3 == s
def test_realize_nondetermenistic() -> None: with setup_storage2('test_realize_nondetermenistic') as (T, S): DATA: int n1: DRef n2: DRef n3: DRef def _gen(i, tag) -> int: nonlocal DATA return DATA def _setup(m): nonlocal n1, n2, n3 n1 = mkstage(m, {'a': '1'}) n2 = mkstage(m, {'maman': n1}, _gen) n3 = mkstage(m, {'papa': n2}) return n3 DATA = 1 rref1 = realize(instantiate(_setup, S=S)) DATA = 2 rref2 = realize(instantiate(_setup, S=S), force_rebuild=[n2]) assert len(list(drefrrefs(n1, S))) == 1 assert len(list(drefrrefs(n2, S))) == 2 assert len(list(drefrrefs(n3, S))) == 2 assert rref1 != rref2 n2_gr1 = store_deref(rref1, n2, S) n2_gr2 = store_deref(rref2, n2, S) assert n2_gr1 != n2_gr2
def test_diff(): with setup_storage('test_find') as s: s1=partial(mkstage, config={'name':'1'}, nondet=lambda:42) s2=partial(mkstage, config={'name':'2'}, nondet=lambda:33) dref1=instantiate(s1).dref rref2=realize(instantiate(s2)) diff(dref1, rref2) diff(dref1, s2)
def test_no_dref_deps_without_realizers() -> None: with setup_storage2('test_no_dref_deps_without_realizers') as (T, S): try: clo = instantiate(mkstage, {'a': 1}, S=S) _ = realize(instantiate(mkstage, {'maman': clo.dref}, S=S)) raise ShouldHaveFailed("We shouldn't share DRefs across managers") except AssertionError: pass
def test_repeated_instantiate() -> None: with setup_storage2('test_repeated_instantiate') as (T, S): def _setting(m: Manager) -> DRef: return mkstage(m, {'a': '1'}) cl1 = instantiate(_setting, S=S) cl2 = instantiate(_setting, S=S) assert len(cl1.derivations) == 1 assert len(cl2.derivations) == 1 assert cl1.derivations[0].dref == cl2.derivations[0].dref assert cl1.dref == cl2.dref
def test_mknode(d) -> None: with setup_storage('test_mknode'): def _setting(m: Manager) -> DRef: return mknode(m, d) cl1 = instantiate(_setting) cl2 = instantiate(_setting) assert len(cl1.derivations) == 1 assert len(cl2.derivations) == 1 assert cl1.derivations[0].dref == cl2.derivations[0].dref assert cl1.dref == cl2.dref
def test_mknode_with_artifacts(d, a) -> None: with setup_storage('test_mknode_with_artifacts'): def _setting(m: Manager) -> DRef: return mknode(m, config_dict=d, artifacts=a) cl = instantiate(_setting) assert len(cl.derivations) == 1 rref = realize(instantiate(_setting)) for nm, val in a.items(): assert isfile(join(store_rref2path(rref),nm)), \ f"RRef {rref} doesn't contain artifact {nm}"
def run(mode: int = 2, interactive: bool = True): maxitems = 5000 Wdef = 1 gather_depth = None num_inputs = 2 if mode == 1: batch_size = 5 elif mode == 2: batch_size = 1 else: assert False, 'Invalid mode' def _stage(m: Manager): inp = stage_inputs(m, num_inputs=num_inputs, batch_size=batch_size) ds = stage_dataset2(m, inp, Wdef=Wdef, maxitems=maxitems, gather_depth=gather_depth) df = stage_df(m, ds) dsvis = stage_vis(m, df) return dsvis rref = realize(instantiate(_stage), force_rebuild=interactive) linkrref(rref, '_results') if interactive: system(f"feh {mklens(rref).out_plot.syspath} &") return rref
def test_repl_override(): with setup_storage('test_repl_override'): n1: DRef n2: DRef def _setting(m: Manager) -> DRef: nonlocal n1, n2 n1 = mkstage(m, {'a': '1'}, lambda i, tag: 33) n2 = mkstage(m, {'maman': n1}, lambda i, tag: 42) return n2 clo = instantiate(_setting) rh = repl_realize(clo, force_interrupt=[n1]) assert rh.dref == n1 b = repl_build(rh) with open(join(build_outpath(b), 'artifact'), 'w') as f: f.write('777') repl_continue(b.outgroups, rh=rh) rref = repl_rref(rh) assert rref is not None rrefn1 = store_deref(rref, n1)[Tag('out')] assert tryread(Path(join(store_rref2path(rrefn1), 'artifact'))) == '777'
def test_realize_dependencies() -> None: with setup_storage2('test_realize_dependencies') as (T, S): n1 = None n2 = None n3 = None toplevel = None def _setup(m): nonlocal n1, n2, n3, toplevel n1 = mkstage(m, {'a': 1}) n2 = mkstage(m, {'parent1': n1}) n3 = mkstage(m, {'parent2': n1}) toplevel = mkstage(m, {'n2': n2, 'n3': n3}) return toplevel rref = realize(instantiate(_setup, S=S)) assert_valid_rref(rref) assert n1 is not None assert n2 is not None assert n3 is not None assert toplevel is not None c = store_config(rref, S) assert_valid_config(c) cro = store_cattrs(rref, S) assert_valid_dref(getattr(cro, 'n2')) assert_valid_dref(getattr(cro, 'n3')) assert set(store_deps([n1], S)) == set([]) assert set(store_deps([n2, n3], S)) == set([n1]) assert set(store_deps([toplevel], S)) == set([n2, n3]) assert store_deepdeps([n1], S) == set([]) assert store_deepdeps([n2, n3], S) == set([n1]) assert store_deepdeps([toplevel], S) == set([n1, n2, n3])
def test_fetchurl(): with setup_storage('test_fetchurl'): with TemporaryDirectory() as tmp: mockwget = join(tmp, 'mockwget') mockdata = join(tmp, 'mockdata') with open(mockdata, 'w') as f: f.write('blala') system(f'tar -zcvf {mockdata}.tar.gz {mockdata}') with open(mockwget, 'w') as f: f.write(f"#!/bin/sh\n") f.write(f"mv {mockdata}.tar.gz $3\n") chmod(mockwget, stat(mockwget).st_mode | S_IEXEC) wanted_sha256 = pipe_stdout([SHA256SUM, f"{mockdata}.tar.gz"]).split()[0] import pylightnix.stages.fetch oldwget = pylightnix.stages.fetch.WGET try: pylightnix.stages.fetch.WGET = lambda: mockwget clo = instantiate(fetchurl, url='mockwget://result.tar.gz', filename='validname.tar.gz', sha256=wanted_sha256) rref = realize(clo) finally: pylightnix.stages.fetch.WGET = oldwget
def test_repl_realizeInval(): with setup_storage('test_repl_realizeInval'): try: repl_realize(instantiate(mkstage, {'a': 1}), force_interrupt=33) #type:ignore raise ShouldHaveFailed('wrong force_interrupt value') except AssertionError: pass
def test_minimal_closure() -> None: with setup_storage2('test_minimal_closure') as (T, S): def _somenode(m): return mkstage(m, {'a': 0}) def _anothernode(m): n1 = mkstage(m, {'a': 1}) n2 = _somenode(m) n3 = mkstage(m, {'maman': n1, 'papa': n2}) return n3 rref1 = realize(instantiate(_somenode, S=S)) rref = realize(instantiate(_anothernode, S=S)) rref2 = store_deref(rref, store_cattrs(rref, S=S).papa, S=S)[Tag('out')] assert rref1 == rref2, '''
def test_mkfile() -> None: with setup_storage('test_mkfile'): def _setting(m: Manager, nm) -> DRef: return mkfile(m, Name('foo'), bytes((nm or 'bar').encode('utf-8')), nm) rref1 = realize(instantiate(_setting, None)) with open(join(store_rref2path(rref1), 'foo'), 'r') as f: bar = f.read() assert bar == 'bar' rref2 = realize(instantiate(_setting, 'baz')) with open(join(store_rref2path(rref2), 'baz'), 'r') as f: baz = f.read() assert baz == 'baz' assert rref1 != rref2
def test_repl_globalCancel(): with setup_storage('test_repl_globalCancel'): n1: DRef n2: DRef def _setting(m: Manager) -> DRef: nonlocal n1, n2 n1 = mkstage(m, {'a': '1'}) n2 = mkstage(m, {'maman': n1}) return n2 rh = repl_realize(instantiate(_setting), force_interrupt=True) assert repl_rref(rh) is None repl_cancel() assert rh.gen is None rref = realize(instantiate(_setting)) assert isrref(rref)
def test_repl_race(): with setup_storage('test_repl_recursion'): def _setting(m: Manager) -> DRef: n1 = mkstage(m, {'a': '1'}) n2 = mkstage(m, {'maman': n1}) return n2 clo = instantiate(_setting) rh = repl_realize(clo, force_interrupt=True) assert repl_rref(rh) is None assert rh.dref == clo.dref clo2 = instantiate(_setting) rref2 = realize(clo2) # Realize dref while repl_realizing same dref repl_cancel(rh) assert_valid_rref(rref2)
def test_find(): with setup_storage('test_find') as s: s1=partial(mkstage, config={'name':'1'}, nondet=lambda:42) s2=partial(mkstage, config={'name':'2'}, nondet=lambda:33) rref1=realize(instantiate(s1)) sleep(0.1) now=parsetime(timestring()) rref2=realize(instantiate(s2)) rrefs=find() assert set(rrefs)==set([rref1,rref2]) rrefs=find(name='1') assert rrefs==[rref1] rrefs=find(name=s2) assert rrefs==[rref2] rrefs=find(newer=-10) assert len(rrefs)==2 rrefs=find(newer=now) assert rrefs==[rref2]
def test_gc() -> None: with setup_storage2('test_gc') as (T, S): def _node1(m: Manager) -> DRef: return mkstage(m, {'name': '1'}) def _node2(m: Manager) -> DRef: return mkstage(m, {'name': '2', 'maman': _node1(m)}) def _node3(m: Manager) -> DRef: return mkstage(m, {'name': '3', 'maman': _node1(m)}) r1 = realize(instantiate(_node1, S=S)) r2 = realize(instantiate(_node2, S=S)) r3 = realize(instantiate(_node3, S=S)) rm_drefs, rm_rrefs = store_gc([], [r2], S) assert rm_drefs == {rref2dref(r) for r in [r3]} assert rm_rrefs == {x for x in [r3]}
def test_no_foreign_dref_deps() -> None: with setup_storage2('test_no_foreign_dref_deps') as (T, S): def _setup(m): clo = instantiate(mkstage, {'foo': 'bar'}) return mkstage(m, {'bogus': clo.dref}) try: dref = instantiate(_setup, S=S) raise ShouldHaveFailed(f"Should fail, but got {dref}") except AssertionError: pass
def test_realized() -> None: with setup_storage('test_realized'): def _setting(m: Manager, assume_realized: bool) -> DRef: def _realize(b: Build): if assume_realized: raise ShouldHaveFailed('Should not call the real realizer') return build_outpath(b) return mkdrv(m, mkconfig({'name': '1'}), match_some(), build_wrapper(_realize)) dref = instantiate(realized(_setting), assume_realized=True) try: rref = realize(dref) raise ShouldHaveFailed('Should not be realized') except AssertionError: pass rref = realize(instantiate(_setting, assume_realized=False)) rref2 = realize(instantiate(realized(_setting), assume_realized=True)) assert rref == rref2
def test_no_rref_deps() -> None: with setup_storage2('test_no_rref_deps') as (T, S): def _setup(m): rref = realize(instantiate(mkstage, {'foo': 'bar'})) n2 = mkstage(m, {'bogus': rref}) return n2 try: dref = instantiate(_setup, S=S) raise ShouldHaveFailed(f"Should fail, but got {dref}") except AssertionError: pass
def test_du(): with setup_storage('test_du') as s: usage=du() assert usage=={} clo=instantiate(mkstage, {'name':'1'}, lambda:42) usage=du() assert clo.dref in usage assert usage[clo.dref][0]>0 assert usage[clo.dref][1]=={} rref=realize(clo) usage=du() assert rref in usage[clo.dref][1] assert usage[clo.dref][1][rref]>0
def load3(index, allow_realize: bool = True): """ Loads the stabilized dataset. Raises LookupError if realization is forbidden and the data are absent. """ if allow_realize: stage = stage_datasetS else: def _err(*args, **kwargs): raise LookupError( "Attempting to realize dataset section with index {index}") stage = redefine(stage_datasetS, new_realizer=_err) return realize(instantiate(stage, index=index, Nvalid=N1, npoints=2000))
def test_no_recursive_instantiate_with_same_manager() -> None: with setup_storage2('test_no_recursive_instantiate_with_same_manager') as ( T, S): def _setup(m): derivs = instantiate_(m, _setup) n2 = mkstage(m, {'bogus': derivs.dref}) return n2 try: dref = instantiate(_setup, S=S) raise ShouldHaveFailed(f"Should fail, but got {dref}") except AssertionError: pass
def test_linkrrefs() -> None: with setup_storage2('test_linkrefs') as (T, S): s1 = partial(mkstage, config={ 'name': '1', 'promise': [promise, 'artifact'] }) rref1 = realize(instantiate(s1, S=S)) l = linkrrefs([rref1, rref1], destdir=S, S=S) assert len(l) == 2 assert str(l[0]) == join(S, 'result-1') assert islink(join(S, 'result-1')) l = linkrrefs([rref1], destdir=S, withtime=True, S=S) assert S in l[0]
def test_lens_closures(): with setup_storage('test_lens_closures'): def _setting(m:Manager)->DRef: n1=mkstage(m, {'name':'1', 'x':33, 'promise':[promise,'artifact']}) n2=mkstage(m, {'name':'2', 'papa':n1, 'dict':{'d1':1} }) n3=mkstage(m, {'name':'3', 'maman':n2 }) return n3 clo=instantiate(_setting) assert isclosure(clo) rref=realize(mklens(clo).maman.papa.closure) assert mklens(rref).x.val==33 assert open(mklens(rref).promise.syspath).read()=='0'
def test_promise() -> None: with setup_storage2('test_promise') as (T, S): def _setting(m: Manager, fullfill: bool) -> DRef: n1 = mkstage(m, {'name': '1', 'promise': [promise, 'artifact']}) def _realize(b: Build): o = build_outpath(b) c = build_cattrs(b) assert b.dref in c.promise assert n1 in store_cattrs(c.maman, S).promise assert build_path(b, c.promise) == join(o, 'uber-artifact') assert build_path(b, store_cattrs(c.maman, S).promise) == build_path( b, c.maman_promise) if fullfill: with open(build_path(b, c.promise), 'w') as f: f.write('chickenpoop') return mkdrv(m, mkconfig({ 'name': '2', 'maman': n1, 'promise': [promise, 'uber-artifact'], 'maman_promise': [n1, 'artifact'] }), matcher=match_some(), realizer=build_wrapper(_realize)) try: rref = realize(instantiate(_setting, False, S=S)) raise ShouldHaveFailed('Promise trigger') except AssertionError: pass rref = realize(instantiate(_setting, True, S=S)) assert_valid_rref(rref)
def test_shellref(): with setup_storage('test_shellref') as s: with TemporaryDirectory() as tmp: mockshell=join(tmp,'mockshell') with open(mockshell,'w') as f: f.write(f"#!/bin/sh\n") f.write(f"pwd\n") chmod(mockshell, stat(mockshell).st_mode | S_IEXEC) environ['SHELL']=mockshell rref=realize(instantiate(mkstage, {'a':1})) shellref(rref) shellref(rref2dref(rref)) shellref() shell(store_rref2path(rref)) repl_realize(instantiate(mkstage, {'n':1}), force_interrupt=True) b=repl_build() o=build_outpath(b) shell(b) repl_cancelBuild(b) try: shellref('foo') # type:ignore raise ShouldHaveFailed('shellref should reject garbage') except AssertionError: pass
def test_repeated_realize() -> None: with setup_storage2('test_repeated_realize') as (T, S): def _setting(m: Manager) -> DRef: return mkstage(m, {'a': '1'}) clo = instantiate(_setting, S=S) rref1 = realize(clo) rref2 = realize(clo) rref3 = realize(clo, force_rebuild=[clo.dref]) assert rref1 == rref2 and rref2 == rref3 try: rref3 = realize(clo, force_rebuild='Foobar') # type:ignore raise ShouldHaveFailed except AssertionError: pass