def test_dirhash4(d) -> None:
with setup_storage('dirhash4') as path:
with open(join(path, 'a'), 'w') as f:
f.write(str(d))
dh = dirhash(path)
fh = filehash(Path(join(path, 'a')))
assert dh == fh, "Hash of a 1-file dir should match the hash of the file"
def test_mklogdir1() -> None:
with setup_storage('mklogdir1') as path:
logdir = mklogdir(tag='testtag', logrootdir=path)
assert isdir(logdir)
logdir = mklogdir(tag='testtag', logrootdir=path, subdirs=['a', 'b'])
assert isdir(join(logdir, 'a'))
assert isdir(join(logdir, 'b'))
def test_fetchurl():
with setup_storage('test_fetchurl'):
with TemporaryDirectory() as tmp:
mockwget = join(tmp, 'mockwget')
mockdata = join(tmp, 'mockdata')
with open(mockdata, 'w') as f:
f.write('blala')
system(f'tar -zcvf {mockdata}.tar.gz {mockdata}')
with open(mockwget, 'w') as f:
f.write(f"#!/bin/sh\n")
f.write(f"mv {mockdata}.tar.gz $3\n")
chmod(mockwget, stat(mockwget).st_mode | S_IEXEC)
wanted_sha256 = pipe_stdout([SHA256SUM,
f"{mockdata}.tar.gz"]).split()[0]
import pylightnix.stages.fetch
oldwget = pylightnix.stages.fetch.WGET
try:
pylightnix.stages.fetch.WGET = lambda: mockwget
clo = instantiate(fetchurl,
url='mockwget://result.tar.gz',
filename='validname.tar.gz',
sha256=wanted_sha256)
rref = realize(clo)
finally:
pylightnix.stages.fetch.WGET = oldwget
def test_dirhash3(d) -> None:
with setup_storage('dirhash3') as path:
with open(join(path, 'a'), 'w') as f:
f.write(str(d))
p = run([SHA256SUM, join(path, 'a')], stdout=-1, check=True, cwd=path)
h = dirhash(path)
assert (p.stdout[:len(h)].decode('utf-8')) == h
def test_repl_override():
with setup_storage('test_repl_override'):
n1: DRef
n2: DRef
def _setting(m: Manager) -> DRef:
nonlocal n1, n2
n1 = mkstage(m, {'a': '1'}, lambda i, tag: 33)
n2 = mkstage(m, {'maman': n1}, lambda i, tag: 42)
return n2
clo = instantiate(_setting)
rh = repl_realize(clo, force_interrupt=[n1])
assert rh.dref == n1
b = repl_build(rh)
with open(join(build_outpath(b), 'artifact'), 'w') as f:
f.write('777')
repl_continue(b.outgroups, rh=rh)
rref = repl_rref(rh)
assert rref is not None
rrefn1 = store_deref(rref, n1)[Tag('out')]
assert tryread(Path(join(store_rref2path(rrefn1),
'artifact'))) == '777'
def test_diff():
with setup_storage('test_find') as s:
s1=partial(mkstage, config={'name':'1'}, nondet=lambda:42)
s2=partial(mkstage, config={'name':'2'}, nondet=lambda:33)
dref1=instantiate(s1).dref
rref2=realize(instantiate(s2))
diff(dref1, rref2)
diff(dref1, s2)
def test_repl_realizeInval():
with setup_storage('test_repl_realizeInval'):
try:
repl_realize(instantiate(mkstage, {'a': 1}),
force_interrupt=33) #type:ignore
raise ShouldHaveFailed('wrong force_interrupt value')
except AssertionError:
pass
def test_mknode(d) -> None:
with setup_storage('test_mknode'):
def _setting(m: Manager) -> DRef:
return mknode(m, d)
cl1 = instantiate(_setting)
cl2 = instantiate(_setting)
assert len(cl1.derivations) == 1
assert len(cl2.derivations) == 1
assert cl1.derivations[0].dref == cl2.derivations[0].dref
assert cl1.dref == cl2.dref
def test_du():
with setup_storage('test_du') as s:
usage=du()
assert usage=={}
clo=instantiate(mkstage, {'name':'1'}, lambda:42)
usage=du()
assert clo.dref in usage
assert usage[clo.dref][0]>0
assert usage[clo.dref][1]=={}
rref=realize(clo)
usage=du()
assert rref in usage[clo.dref][1]
assert usage[clo.dref][1][rref]>0
def test_mknode_with_artifacts(d, a) -> None:
with setup_storage('test_mknode_with_artifacts'):
def _setting(m: Manager) -> DRef:
return mknode(m, config_dict=d, artifacts=a)
cl = instantiate(_setting)
assert len(cl.derivations) == 1
rref = realize(instantiate(_setting))
for nm, val in a.items():
assert isfile(join(store_rref2path(rref),nm)), \
f"RRef {rref} doesn't contain artifact {nm}"
def test_dirhash() -> None:
with setup_storage('dirhash') as path:
h1 = dirhash(path)
assert_valid_hash(h1)
with open(join(path, '_a'), 'w') as f:
f.write('1')
h2 = dirhash(path)
assert_valid_hash(h2)
assert h1 == h2, "Test expected to ignore files starting with underscope"
with open(join(path, 'a'), 'w') as f:
f.write('1')
h3 = dirhash(path)
assert_valid_hash(h3)
assert h3 != h2
def test_lens_closures():
with setup_storage('test_lens_closures'):
def _setting(m:Manager)->DRef:
n1=mkstage(m, {'name':'1', 'x':33, 'promise':[promise,'artifact']})
n2=mkstage(m, {'name':'2', 'papa':n1, 'dict':{'d1':1} })
n3=mkstage(m, {'name':'3', 'maman':n2 })
return n3
clo=instantiate(_setting)
assert isclosure(clo)
rref=realize(mklens(clo).maman.papa.closure)
assert mklens(rref).x.val==33
assert open(mklens(rref).promise.syspath).read()=='0'
def test_store_initialize() -> None:
with setup_storage('test_store_initialize') as p:
import pylightnix.core
try:
pylightnix.core.PYLIGHTNIX_TMP = join(p, 'tmp')
pylightnix.core.PYLIGHTNIX_STORE = join(p, 'store')
store_initialize(custom_store=None, custom_tmp=None)
assert isdir(join(p, 'tmp'))
assert isdir(join(p, 'store'))
store_initialize(custom_store=None, custom_tmp=None)
assert isdir(join(p, 'tmp'))
assert isdir(join(p, 'store'))
finally:
pylightnix.core.PYLIGHTNIX_TMP = None # type:ignore
pylightnix.core.PYLIGHTNIX_STORE = None # type:ignore
def test_mklogdir2(strtag, timetag) -> None:
with setup_storage('mklogdir2') as path:
linkpath = join(path, f'_{strtag}_latest')
logdir = mklogdir(tag=strtag, logrootdir=Path(path), timetag=timetag)
open(join(logdir, 'a'), 'w').write('a')
assert isdir(logdir)
assert islink(linkpath)
assert isfile(join(linkpath, 'a'))
logdir2 = mklogdir(tag=strtag,
logrootdir=Path(path),
timetag=timetag + '2')
open(join(logdir2, 'b'), 'w').write('b')
assert isdir(logdir2)
assert islink(linkpath)
assert isfile(join(linkpath, 'b'))
def test_rmdref():
with setup_storage('test_rmdref') as s:
clo=instantiate(mkstage, {'a':1}, lambda:42)
drefpath=store_dref2path(clo.dref)
rref1=realize(clo, force_rebuild=[clo.dref])
rrefpath=store_rref2path(rref1)
assert isdir(rrefpath)
rmref(rref1)
assert not isdir(rrefpath)
assert isdir(drefpath)
rmref(clo.dref)
assert not isdir(drefpath)
try:
rmref('asdasd') # type:ignore
raise ShouldHaveFailed('shoud reject garbage')
except AssertionError:
pass
def test_mkfile() -> None:
with setup_storage('test_mkfile'):
def _setting(m: Manager, nm) -> DRef:
return mkfile(m, Name('foo'), bytes((nm or 'bar').encode('utf-8')),
nm)
rref1 = realize(instantiate(_setting, None))
with open(join(store_rref2path(rref1), 'foo'), 'r') as f:
bar = f.read()
assert bar == 'bar'
rref2 = realize(instantiate(_setting, 'baz'))
with open(join(store_rref2path(rref2), 'baz'), 'r') as f:
baz = f.read()
assert baz == 'baz'
assert rref1 != rref2
def test_fetchlocal2():
with setup_storage('test_fetclocal') as tmp:
mockdata = join(tmp, 'mockdata')
with open(mockdata, 'w') as f:
f.write('dogfood')
wanted_sha256 = pipe_stdout([SHA256SUM, "mockdata"],
cwd=tmp).split()[0]
rref = realize(
instantiate(fetchlocal,
path=mockdata,
sha256=wanted_sha256,
mode='as-is'))
assert isrref(rref)
assert isfile(join(store_rref2path(rref), 'mockdata'))
assert isfile(mklens(rref).out_path.syspath)
def test_fetchlocal():
with setup_storage('test_fetclocal') as tmp:
mockdata = join(tmp, 'mockdata')
with open(mockdata, 'w') as f:
f.write('dogfood')
system(f"tar -C '{tmp}' -zcvf {tmp}/mockdata.tar.gz mockdata")
wanted_sha256 = pipe_stdout([SHA256SUM, "mockdata.tar.gz"],
cwd=tmp).split()[0]
rref = realize(
instantiate(fetchlocal,
path=mockdata + '.tar.gz',
filename='validname.tar.gz',
sha256=wanted_sha256))
assert isrref(rref)
assert isfile(join(store_rref2path(rref), 'mockdata'))
def test_redefine() -> None:
with setup_storage('test_redefine'):
def _setting(m: Manager) -> DRef:
return mknode(m, {
'name': 'foo',
'bar': 'baz',
'output': [promise, 'f']
}, {Name('f'): bytes(('umgh').encode('utf-8'))})
def _nc(c):
mklens(c).bar.val = 42
_setting2 = redefine(_setting, new_config=_nc)
rref = realize(instantiate(_setting2))
assert mklens(rref).bar.val == 42
assert tryread(mklens(rref).output.syspath) == 'umgh'
def test_repl_globalCancel():
with setup_storage('test_repl_globalCancel'):
n1: DRef
n2: DRef
def _setting(m: Manager) -> DRef:
nonlocal n1, n2
n1 = mkstage(m, {'a': '1'})
n2 = mkstage(m, {'maman': n1})
return n2
rh = repl_realize(instantiate(_setting), force_interrupt=True)
assert repl_rref(rh) is None
repl_cancel()
assert rh.gen is None
rref = realize(instantiate(_setting))
assert isrref(rref)
def test_repl_race():
with setup_storage('test_repl_recursion'):
def _setting(m: Manager) -> DRef:
n1 = mkstage(m, {'a': '1'})
n2 = mkstage(m, {'maman': n1})
return n2
clo = instantiate(_setting)
rh = repl_realize(clo, force_interrupt=True)
assert repl_rref(rh) is None
assert rh.dref == clo.dref
clo2 = instantiate(_setting)
rref2 = realize(clo2) # Realize dref while repl_realizing same dref
repl_cancel(rh)
assert_valid_rref(rref2)
def test_find():
with setup_storage('test_find') as s:
s1=partial(mkstage, config={'name':'1'}, nondet=lambda:42)
s2=partial(mkstage, config={'name':'2'}, nondet=lambda:33)
rref1=realize(instantiate(s1))
sleep(0.1)
now=parsetime(timestring())
rref2=realize(instantiate(s2))
rrefs=find()
assert set(rrefs)==set([rref1,rref2])
rrefs=find(name='1')
assert rrefs==[rref1]
rrefs=find(name=s2)
assert rrefs==[rref2]
rrefs=find(newer=-10)
assert len(rrefs)==2
rrefs=find(newer=now)
assert rrefs==[rref2]
def test_inplace():
with setup_storage('test_inplace'):
setup_inplace_reset()
n1 = instantiate_inplace(mkstage, {'a': '1'})
n2 = instantiate_inplace(mkstage, {'b': '2'})
n3 = instantiate_inplace(mkstage, {'c': '3', 'maman': n1})
n4 = instantiate_inplace(mkstage, {'c': '4', 'papa': n3})
assert_valid_dref(n3)
assert_valid_dref(n4)
rref_n3 = realize_inplace(n3)
assert_valid_rref(rref_n3)
all_drefs = list(alldrefs())
assert len(all_drefs) == 4
assert len(list(store_rrefs_(n1))) == 1
assert len(list(store_rrefs_(n2))) == 0
assert len(list(store_rrefs_(n3))) == 1
assert len(list(store_rrefs_(n4))) == 0
def test_repl_globalHelper():
with setup_storage('test_repl_globalHelper'):
n1: DRef
n2: DRef
def _setting(m: Manager) -> DRef:
nonlocal n1, n2
n1 = mkstage(m, {'a': '1'})
n2 = mkstage(m, {'maman': n1})
return n2
rh = repl_realize(instantiate(_setting), force_interrupt=True)
assert repl_rref(rh) is None
b = repl_build()
with open(join(build_outpath(b), 'artifact.txt'), 'w') as f:
f.write("Fooo")
repl_continueBuild(b)
rref = repl_rref(rh)
assert rref is not None
assert isfile(join(store_rref2path(rref), 'artifact.txt'))
def test_realized() -> None:
with setup_storage('test_realized'):
def _setting(m: Manager, assume_realized: bool) -> DRef:
def _realize(b: Build):
if assume_realized:
raise ShouldHaveFailed('Should not call the real realizer')
return build_outpath(b)
return mkdrv(m, mkconfig({'name': '1'}), match_some(),
build_wrapper(_realize))
dref = instantiate(realized(_setting), assume_realized=True)
try:
rref = realize(dref)
raise ShouldHaveFailed('Should not be realized')
except AssertionError:
pass
rref = realize(instantiate(_setting, assume_realized=False))
rref2 = realize(instantiate(realized(_setting), assume_realized=True))
assert rref == rref2
def test_repl_basic():
with setup_storage('test_repl_default'):
n1: DRef
n2: DRef
def _setting(m: Manager) -> DRef:
nonlocal n1, n2
n1 = mkstage(m, {'a': '1'})
n2 = mkstage(m, {'maman': n1})
return n2
clo = instantiate(_setting)
rh = repl_realize(clo, force_interrupt=False)
assert repl_rref(rh) is not None
rh = repl_realize(clo, force_interrupt=[n1, n2])
assert repl_rref(rh) is None
assert rh.dref == n1
repl_continue(rh=rh)
assert repl_rref(rh) is None
assert rh.dref == n2
repl_continue(rh=rh)
assert repl_rref(rh) is not None
def test_bashlike():
with setup_storage('test_bashlike'):
clo=instantiate(mkstage, {'a':1}, lambda:42)
rref1=realize(clo, force_rebuild=[clo.dref])
rref2=realize(clo, force_rebuild=[clo.dref])
assert 'artifact' in lsref(rref1)
assert 'context.json' in lsref(rref1)
assert '__buildtime__.txt' in lsref(rref1)
h1,_,_=unrref(rref1)
h2,_,_=unrref(rref2)
assert len(lsref(clo.dref))==2
assert h1 in lsref(clo.dref)
assert h2 in lsref(clo.dref)
assert '42' in catref(rref1,['artifact'])
try:
lsref('foobar') # type:ignore
raise ShouldHaveFailed('should reject garbage')
except AssertionError:
pass
try:
catref(clo.dref, ['artifact']) # type:ignore
raise ShouldHaveFailed('notimpl')
except AssertionError:
pass
def test_shellref():
with setup_storage('test_shellref') as s:
with TemporaryDirectory() as tmp:
mockshell=join(tmp,'mockshell')
with open(mockshell,'w') as f:
f.write(f"#!/bin/sh\n")
f.write(f"pwd\n")
chmod(mockshell, stat(mockshell).st_mode | S_IEXEC)
environ['SHELL']=mockshell
rref=realize(instantiate(mkstage, {'a':1}))
shellref(rref)
shellref(rref2dref(rref))
shellref()
shell(store_rref2path(rref))
repl_realize(instantiate(mkstage, {'n':1}), force_interrupt=True)
b=repl_build()
o=build_outpath(b)
shell(b)
repl_cancelBuild(b)
try:
shellref('foo') # type:ignore
raise ShouldHaveFailed('shellref should reject garbage')
except AssertionError:
pass
def test_setup_storage() -> None:
setup_storage('a')
setup_storage('a')
def test_lens():
with setup_storage('test_lens'):
def _setting(m:Manager)->DRef:
n1=mkstage(m, {'name':'1', 'promise':[promise,'artifact']})
n2=mkstage(m, {'name':'2', 'promise':[promise,'artifact'],
'dict':{'d1':1} })
def _realize(b:Build):
o=build_outpath(b)
c=build_cattrs(b)
assert isrefpath(mklens(b).maman.promise.refpath)
assert isfile(mklens(b).papa.promise.syspath)
assert o in mklens(b).promise.syspath
assert o == mklens(b).syspath
assert mklens(b).papa.name.val == '2'
assert mklens(b).papa.dref == c.papa
with open(mklens(b).promise.syspath,'w') as f:
f.write('chickenpoop')
return mkdrv(m,
mkconfig({'name':'3', 'maman':n1, 'papa':n2,
'promise':[promise,'artifact'],
}),
matcher=match_some(),
realizer=build_wrapper(_realize))
clo=instantiate(_setting)
assert isrefpath(mklens(clo.dref).maman.promise.refpath)
assert isdir(mklens(clo.dref).syspath)
rref=realize(clo)
assert_valid_rref(rref)
assert isrefpath(mklens(rref).maman.promise.refpath)
assert isfile(mklens(rref).maman.promise.syspath)
assert mklens(rref).rref == rref
assert isrefpath(mklens(rref).papa.promise.refpath)
assert mklens(rref).papa.dict.d1.val == 1
assert mklens(rref).dref == clo.dref
assert isdir(mklens(rref).syspath)
try:
print(mklens(clo.dref).maman.promise.syspath)
raise ShouldHaveFailed()
except AssertionError:
pass
try:
print(mklens(rref).papa.dict.d1.get('xxx'))
raise ShouldHaveFailed()
except AssertionError:
pass
try:
print(mklens(rref).papa.dict.d1.syspath)
raise ShouldHaveFailed()
except AssertionError:
pass
try:
mklens(rref).papa.dict.d1.val=42 # can't mutate rref
raise ShouldHaveFailed()
except AssertionError:
pass
d={'foo':'foo','bar':'bar'}
mklens(d).foo.val='zzz'
assert d['foo']=='zzz'
try:
mklens(d).x.val=42 # can't set new values
raise ShouldHaveFailed()
except AssertionError:
pass
mklens(d).bar.val+='33'
assert d['bar']=='bar33'
