def test_dirhash4(d) -> None: with setup_storage('dirhash4') as path: with open(join(path, 'a'), 'w') as f: f.write(str(d)) dh = dirhash(path) fh = filehash(Path(join(path, 'a'))) assert dh == fh, "Hash of a 1-file dir should match the hash of the file"
def test_mklogdir1() -> None: with setup_storage('mklogdir1') as path: logdir = mklogdir(tag='testtag', logrootdir=path) assert isdir(logdir) logdir = mklogdir(tag='testtag', logrootdir=path, subdirs=['a', 'b']) assert isdir(join(logdir, 'a')) assert isdir(join(logdir, 'b'))
def test_fetchurl(): with setup_storage('test_fetchurl'): with TemporaryDirectory() as tmp: mockwget = join(tmp, 'mockwget') mockdata = join(tmp, 'mockdata') with open(mockdata, 'w') as f: f.write('blala') system(f'tar -zcvf {mockdata}.tar.gz {mockdata}') with open(mockwget, 'w') as f: f.write(f"#!/bin/sh\n") f.write(f"mv {mockdata}.tar.gz $3\n") chmod(mockwget, stat(mockwget).st_mode | S_IEXEC) wanted_sha256 = pipe_stdout([SHA256SUM, f"{mockdata}.tar.gz"]).split()[0] import pylightnix.stages.fetch oldwget = pylightnix.stages.fetch.WGET try: pylightnix.stages.fetch.WGET = lambda: mockwget clo = instantiate(fetchurl, url='mockwget://result.tar.gz', filename='validname.tar.gz', sha256=wanted_sha256) rref = realize(clo) finally: pylightnix.stages.fetch.WGET = oldwget
def test_dirhash3(d) -> None: with setup_storage('dirhash3') as path: with open(join(path, 'a'), 'w') as f: f.write(str(d)) p = run([SHA256SUM, join(path, 'a')], stdout=-1, check=True, cwd=path) h = dirhash(path) assert (p.stdout[:len(h)].decode('utf-8')) == h
def test_repl_override(): with setup_storage('test_repl_override'): n1: DRef n2: DRef def _setting(m: Manager) -> DRef: nonlocal n1, n2 n1 = mkstage(m, {'a': '1'}, lambda i, tag: 33) n2 = mkstage(m, {'maman': n1}, lambda i, tag: 42) return n2 clo = instantiate(_setting) rh = repl_realize(clo, force_interrupt=[n1]) assert rh.dref == n1 b = repl_build(rh) with open(join(build_outpath(b), 'artifact'), 'w') as f: f.write('777') repl_continue(b.outgroups, rh=rh) rref = repl_rref(rh) assert rref is not None rrefn1 = store_deref(rref, n1)[Tag('out')] assert tryread(Path(join(store_rref2path(rrefn1), 'artifact'))) == '777'
def test_diff(): with setup_storage('test_find') as s: s1=partial(mkstage, config={'name':'1'}, nondet=lambda:42) s2=partial(mkstage, config={'name':'2'}, nondet=lambda:33) dref1=instantiate(s1).dref rref2=realize(instantiate(s2)) diff(dref1, rref2) diff(dref1, s2)
def test_repl_realizeInval(): with setup_storage('test_repl_realizeInval'): try: repl_realize(instantiate(mkstage, {'a': 1}), force_interrupt=33) #type:ignore raise ShouldHaveFailed('wrong force_interrupt value') except AssertionError: pass
def test_mknode(d) -> None: with setup_storage('test_mknode'): def _setting(m: Manager) -> DRef: return mknode(m, d) cl1 = instantiate(_setting) cl2 = instantiate(_setting) assert len(cl1.derivations) == 1 assert len(cl2.derivations) == 1 assert cl1.derivations[0].dref == cl2.derivations[0].dref assert cl1.dref == cl2.dref
def test_du(): with setup_storage('test_du') as s: usage=du() assert usage=={} clo=instantiate(mkstage, {'name':'1'}, lambda:42) usage=du() assert clo.dref in usage assert usage[clo.dref][0]>0 assert usage[clo.dref][1]=={} rref=realize(clo) usage=du() assert rref in usage[clo.dref][1] assert usage[clo.dref][1][rref]>0
def test_mknode_with_artifacts(d, a) -> None: with setup_storage('test_mknode_with_artifacts'): def _setting(m: Manager) -> DRef: return mknode(m, config_dict=d, artifacts=a) cl = instantiate(_setting) assert len(cl.derivations) == 1 rref = realize(instantiate(_setting)) for nm, val in a.items(): assert isfile(join(store_rref2path(rref),nm)), \ f"RRef {rref} doesn't contain artifact {nm}"
def test_dirhash() -> None: with setup_storage('dirhash') as path: h1 = dirhash(path) assert_valid_hash(h1) with open(join(path, '_a'), 'w') as f: f.write('1') h2 = dirhash(path) assert_valid_hash(h2) assert h1 == h2, "Test expected to ignore files starting with underscope" with open(join(path, 'a'), 'w') as f: f.write('1') h3 = dirhash(path) assert_valid_hash(h3) assert h3 != h2
def test_lens_closures(): with setup_storage('test_lens_closures'): def _setting(m:Manager)->DRef: n1=mkstage(m, {'name':'1', 'x':33, 'promise':[promise,'artifact']}) n2=mkstage(m, {'name':'2', 'papa':n1, 'dict':{'d1':1} }) n3=mkstage(m, {'name':'3', 'maman':n2 }) return n3 clo=instantiate(_setting) assert isclosure(clo) rref=realize(mklens(clo).maman.papa.closure) assert mklens(rref).x.val==33 assert open(mklens(rref).promise.syspath).read()=='0'
def test_store_initialize() -> None: with setup_storage('test_store_initialize') as p: import pylightnix.core try: pylightnix.core.PYLIGHTNIX_TMP = join(p, 'tmp') pylightnix.core.PYLIGHTNIX_STORE = join(p, 'store') store_initialize(custom_store=None, custom_tmp=None) assert isdir(join(p, 'tmp')) assert isdir(join(p, 'store')) store_initialize(custom_store=None, custom_tmp=None) assert isdir(join(p, 'tmp')) assert isdir(join(p, 'store')) finally: pylightnix.core.PYLIGHTNIX_TMP = None # type:ignore pylightnix.core.PYLIGHTNIX_STORE = None # type:ignore
def test_mklogdir2(strtag, timetag) -> None: with setup_storage('mklogdir2') as path: linkpath = join(path, f'_{strtag}_latest') logdir = mklogdir(tag=strtag, logrootdir=Path(path), timetag=timetag) open(join(logdir, 'a'), 'w').write('a') assert isdir(logdir) assert islink(linkpath) assert isfile(join(linkpath, 'a')) logdir2 = mklogdir(tag=strtag, logrootdir=Path(path), timetag=timetag + '2') open(join(logdir2, 'b'), 'w').write('b') assert isdir(logdir2) assert islink(linkpath) assert isfile(join(linkpath, 'b'))
def test_rmdref(): with setup_storage('test_rmdref') as s: clo=instantiate(mkstage, {'a':1}, lambda:42) drefpath=store_dref2path(clo.dref) rref1=realize(clo, force_rebuild=[clo.dref]) rrefpath=store_rref2path(rref1) assert isdir(rrefpath) rmref(rref1) assert not isdir(rrefpath) assert isdir(drefpath) rmref(clo.dref) assert not isdir(drefpath) try: rmref('asdasd') # type:ignore raise ShouldHaveFailed('shoud reject garbage') except AssertionError: pass
def test_mkfile() -> None: with setup_storage('test_mkfile'): def _setting(m: Manager, nm) -> DRef: return mkfile(m, Name('foo'), bytes((nm or 'bar').encode('utf-8')), nm) rref1 = realize(instantiate(_setting, None)) with open(join(store_rref2path(rref1), 'foo'), 'r') as f: bar = f.read() assert bar == 'bar' rref2 = realize(instantiate(_setting, 'baz')) with open(join(store_rref2path(rref2), 'baz'), 'r') as f: baz = f.read() assert baz == 'baz' assert rref1 != rref2
def test_fetchlocal2(): with setup_storage('test_fetclocal') as tmp: mockdata = join(tmp, 'mockdata') with open(mockdata, 'w') as f: f.write('dogfood') wanted_sha256 = pipe_stdout([SHA256SUM, "mockdata"], cwd=tmp).split()[0] rref = realize( instantiate(fetchlocal, path=mockdata, sha256=wanted_sha256, mode='as-is')) assert isrref(rref) assert isfile(join(store_rref2path(rref), 'mockdata')) assert isfile(mklens(rref).out_path.syspath)
def test_fetchlocal(): with setup_storage('test_fetclocal') as tmp: mockdata = join(tmp, 'mockdata') with open(mockdata, 'w') as f: f.write('dogfood') system(f"tar -C '{tmp}' -zcvf {tmp}/mockdata.tar.gz mockdata") wanted_sha256 = pipe_stdout([SHA256SUM, "mockdata.tar.gz"], cwd=tmp).split()[0] rref = realize( instantiate(fetchlocal, path=mockdata + '.tar.gz', filename='validname.tar.gz', sha256=wanted_sha256)) assert isrref(rref) assert isfile(join(store_rref2path(rref), 'mockdata'))
def test_redefine() -> None: with setup_storage('test_redefine'): def _setting(m: Manager) -> DRef: return mknode(m, { 'name': 'foo', 'bar': 'baz', 'output': [promise, 'f'] }, {Name('f'): bytes(('umgh').encode('utf-8'))}) def _nc(c): mklens(c).bar.val = 42 _setting2 = redefine(_setting, new_config=_nc) rref = realize(instantiate(_setting2)) assert mklens(rref).bar.val == 42 assert tryread(mklens(rref).output.syspath) == 'umgh'
def test_repl_globalCancel(): with setup_storage('test_repl_globalCancel'): n1: DRef n2: DRef def _setting(m: Manager) -> DRef: nonlocal n1, n2 n1 = mkstage(m, {'a': '1'}) n2 = mkstage(m, {'maman': n1}) return n2 rh = repl_realize(instantiate(_setting), force_interrupt=True) assert repl_rref(rh) is None repl_cancel() assert rh.gen is None rref = realize(instantiate(_setting)) assert isrref(rref)
def test_repl_race(): with setup_storage('test_repl_recursion'): def _setting(m: Manager) -> DRef: n1 = mkstage(m, {'a': '1'}) n2 = mkstage(m, {'maman': n1}) return n2 clo = instantiate(_setting) rh = repl_realize(clo, force_interrupt=True) assert repl_rref(rh) is None assert rh.dref == clo.dref clo2 = instantiate(_setting) rref2 = realize(clo2) # Realize dref while repl_realizing same dref repl_cancel(rh) assert_valid_rref(rref2)
def test_find(): with setup_storage('test_find') as s: s1=partial(mkstage, config={'name':'1'}, nondet=lambda:42) s2=partial(mkstage, config={'name':'2'}, nondet=lambda:33) rref1=realize(instantiate(s1)) sleep(0.1) now=parsetime(timestring()) rref2=realize(instantiate(s2)) rrefs=find() assert set(rrefs)==set([rref1,rref2]) rrefs=find(name='1') assert rrefs==[rref1] rrefs=find(name=s2) assert rrefs==[rref2] rrefs=find(newer=-10) assert len(rrefs)==2 rrefs=find(newer=now) assert rrefs==[rref2]
def test_inplace(): with setup_storage('test_inplace'): setup_inplace_reset() n1 = instantiate_inplace(mkstage, {'a': '1'}) n2 = instantiate_inplace(mkstage, {'b': '2'}) n3 = instantiate_inplace(mkstage, {'c': '3', 'maman': n1}) n4 = instantiate_inplace(mkstage, {'c': '4', 'papa': n3}) assert_valid_dref(n3) assert_valid_dref(n4) rref_n3 = realize_inplace(n3) assert_valid_rref(rref_n3) all_drefs = list(alldrefs()) assert len(all_drefs) == 4 assert len(list(store_rrefs_(n1))) == 1 assert len(list(store_rrefs_(n2))) == 0 assert len(list(store_rrefs_(n3))) == 1 assert len(list(store_rrefs_(n4))) == 0
def test_repl_globalHelper(): with setup_storage('test_repl_globalHelper'): n1: DRef n2: DRef def _setting(m: Manager) -> DRef: nonlocal n1, n2 n1 = mkstage(m, {'a': '1'}) n2 = mkstage(m, {'maman': n1}) return n2 rh = repl_realize(instantiate(_setting), force_interrupt=True) assert repl_rref(rh) is None b = repl_build() with open(join(build_outpath(b), 'artifact.txt'), 'w') as f: f.write("Fooo") repl_continueBuild(b) rref = repl_rref(rh) assert rref is not None assert isfile(join(store_rref2path(rref), 'artifact.txt'))
def test_realized() -> None: with setup_storage('test_realized'): def _setting(m: Manager, assume_realized: bool) -> DRef: def _realize(b: Build): if assume_realized: raise ShouldHaveFailed('Should not call the real realizer') return build_outpath(b) return mkdrv(m, mkconfig({'name': '1'}), match_some(), build_wrapper(_realize)) dref = instantiate(realized(_setting), assume_realized=True) try: rref = realize(dref) raise ShouldHaveFailed('Should not be realized') except AssertionError: pass rref = realize(instantiate(_setting, assume_realized=False)) rref2 = realize(instantiate(realized(_setting), assume_realized=True)) assert rref == rref2
def test_repl_basic(): with setup_storage('test_repl_default'): n1: DRef n2: DRef def _setting(m: Manager) -> DRef: nonlocal n1, n2 n1 = mkstage(m, {'a': '1'}) n2 = mkstage(m, {'maman': n1}) return n2 clo = instantiate(_setting) rh = repl_realize(clo, force_interrupt=False) assert repl_rref(rh) is not None rh = repl_realize(clo, force_interrupt=[n1, n2]) assert repl_rref(rh) is None assert rh.dref == n1 repl_continue(rh=rh) assert repl_rref(rh) is None assert rh.dref == n2 repl_continue(rh=rh) assert repl_rref(rh) is not None
def test_bashlike(): with setup_storage('test_bashlike'): clo=instantiate(mkstage, {'a':1}, lambda:42) rref1=realize(clo, force_rebuild=[clo.dref]) rref2=realize(clo, force_rebuild=[clo.dref]) assert 'artifact' in lsref(rref1) assert 'context.json' in lsref(rref1) assert '__buildtime__.txt' in lsref(rref1) h1,_,_=unrref(rref1) h2,_,_=unrref(rref2) assert len(lsref(clo.dref))==2 assert h1 in lsref(clo.dref) assert h2 in lsref(clo.dref) assert '42' in catref(rref1,['artifact']) try: lsref('foobar') # type:ignore raise ShouldHaveFailed('should reject garbage') except AssertionError: pass try: catref(clo.dref, ['artifact']) # type:ignore raise ShouldHaveFailed('notimpl') except AssertionError: pass
def test_shellref(): with setup_storage('test_shellref') as s: with TemporaryDirectory() as tmp: mockshell=join(tmp,'mockshell') with open(mockshell,'w') as f: f.write(f"#!/bin/sh\n") f.write(f"pwd\n") chmod(mockshell, stat(mockshell).st_mode | S_IEXEC) environ['SHELL']=mockshell rref=realize(instantiate(mkstage, {'a':1})) shellref(rref) shellref(rref2dref(rref)) shellref() shell(store_rref2path(rref)) repl_realize(instantiate(mkstage, {'n':1}), force_interrupt=True) b=repl_build() o=build_outpath(b) shell(b) repl_cancelBuild(b) try: shellref('foo') # type:ignore raise ShouldHaveFailed('shellref should reject garbage') except AssertionError: pass
def test_setup_storage() -> None: setup_storage('a') setup_storage('a')
def test_lens(): with setup_storage('test_lens'): def _setting(m:Manager)->DRef: n1=mkstage(m, {'name':'1', 'promise':[promise,'artifact']}) n2=mkstage(m, {'name':'2', 'promise':[promise,'artifact'], 'dict':{'d1':1} }) def _realize(b:Build): o=build_outpath(b) c=build_cattrs(b) assert isrefpath(mklens(b).maman.promise.refpath) assert isfile(mklens(b).papa.promise.syspath) assert o in mklens(b).promise.syspath assert o == mklens(b).syspath assert mklens(b).papa.name.val == '2' assert mklens(b).papa.dref == c.papa with open(mklens(b).promise.syspath,'w') as f: f.write('chickenpoop') return mkdrv(m, mkconfig({'name':'3', 'maman':n1, 'papa':n2, 'promise':[promise,'artifact'], }), matcher=match_some(), realizer=build_wrapper(_realize)) clo=instantiate(_setting) assert isrefpath(mklens(clo.dref).maman.promise.refpath) assert isdir(mklens(clo.dref).syspath) rref=realize(clo) assert_valid_rref(rref) assert isrefpath(mklens(rref).maman.promise.refpath) assert isfile(mklens(rref).maman.promise.syspath) assert mklens(rref).rref == rref assert isrefpath(mklens(rref).papa.promise.refpath) assert mklens(rref).papa.dict.d1.val == 1 assert mklens(rref).dref == clo.dref assert isdir(mklens(rref).syspath) try: print(mklens(clo.dref).maman.promise.syspath) raise ShouldHaveFailed() except AssertionError: pass try: print(mklens(rref).papa.dict.d1.get('xxx')) raise ShouldHaveFailed() except AssertionError: pass try: print(mklens(rref).papa.dict.d1.syspath) raise ShouldHaveFailed() except AssertionError: pass try: mklens(rref).papa.dict.d1.val=42 # can't mutate rref raise ShouldHaveFailed() except AssertionError: pass d={'foo':'foo','bar':'bar'} mklens(d).foo.val='zzz' assert d['foo']=='zzz' try: mklens(d).x.val=42 # can't set new values raise ShouldHaveFailed() except AssertionError: pass mklens(d).bar.val+='33' assert d['bar']=='bar33'