def test_realize_readonly() -> None:
with setup_storage2('test_realize_readonly') as (T, S):
rref1 = realize(instantiate(mkstage, {'a': '1'}, S=S))
try:
with open(join(store_rref2path(rref1, S), 'newfile'), 'w') as f:
f.write('foo')
raise ShouldHaveFailed('No write-protection??')
except OSError:
pass
try:
rmtree(store_rref2path(rref1, S))
raise ShouldHaveFailed('No remove-protection??')
except OSError:
pass
def _realize(b: Build):
with open(join(build_outpath(b), 'exe'), 'w') as f:
f.write('#!/bin/sh\necho "Fooo"')
chmod(join(build_outpath(b), 'exe'), S_IWRITE | S_IREAD | S_IEXEC)
rref2 = realize(
instantiate(mkdrv,
Config({}),
match_some(),
build_wrapper(_realize),
S=S))
assert pipe_stdout([join(store_rref2path(rref2,S),'exe')])=='Fooo\n', \
"Did we lost exec permission?"
def test_mksymlink() -> None:
with setup_storage2('test_mksymlink') as (T, S):
tp = T
def _setting1(m: Manager) -> DRef:
return mkstage(m, {'a': '1'}, lambda i, tag: 33, buildtime=False)
def _setting2(m: Manager) -> DRef:
return mkstage(m, {'a': '1'}, lambda i, tag: 42, buildtime=True)
clo = instantiate(_setting1, S=S)
rref = realize(clo)
clo2 = instantiate(_setting2, S=S)
rref2 = realize(clo2, force_rebuild=True)
assert clo.dref == clo2.dref
assert rref2 != rref
s = mksymlink(rref, tgtpath=tp, name='thelink', S=S)
assert islink(s)
assert not isfile(join(s, '__buildtime__.txt'))
assert tp in s
s2 = mksymlink(rref2, tgtpath=tp, name='thelink', S=S)
assert islink(s2)
assert isfile(join(s2, '__buildtime__.txt'))
assert tp in s
assert s2 != s, "s2 should have timestamp"
s3 = mksymlink(rref2, tgtpath=tp, name='thelink', withtime=False, S=S)
assert s3 == s
def test_realize_nondetermenistic() -> None:
with setup_storage2('test_realize_nondetermenistic') as (T, S):
DATA: int
n1: DRef
n2: DRef
n3: DRef
def _gen(i, tag) -> int:
nonlocal DATA
return DATA
def _setup(m):
nonlocal n1, n2, n3
n1 = mkstage(m, {'a': '1'})
n2 = mkstage(m, {'maman': n1}, _gen)
n3 = mkstage(m, {'papa': n2})
return n3
DATA = 1
rref1 = realize(instantiate(_setup, S=S))
DATA = 2
rref2 = realize(instantiate(_setup, S=S), force_rebuild=[n2])
assert len(list(drefrrefs(n1, S))) == 1
assert len(list(drefrrefs(n2, S))) == 2
assert len(list(drefrrefs(n3, S))) == 2
assert rref1 != rref2
n2_gr1 = store_deref(rref1, n2, S)
n2_gr2 = store_deref(rref2, n2, S)
assert n2_gr1 != n2_gr2
def test_diff():
with setup_storage('test_find') as s:
s1=partial(mkstage, config={'name':'1'}, nondet=lambda:42)
s2=partial(mkstage, config={'name':'2'}, nondet=lambda:33)
dref1=instantiate(s1).dref
rref2=realize(instantiate(s2))
diff(dref1, rref2)
diff(dref1, s2)
def test_no_dref_deps_without_realizers() -> None:
with setup_storage2('test_no_dref_deps_without_realizers') as (T, S):
try:
clo = instantiate(mkstage, {'a': 1}, S=S)
_ = realize(instantiate(mkstage, {'maman': clo.dref}, S=S))
raise ShouldHaveFailed("We shouldn't share DRefs across managers")
except AssertionError:
pass
def test_repeated_instantiate() -> None:
with setup_storage2('test_repeated_instantiate') as (T, S):
def _setting(m: Manager) -> DRef:
return mkstage(m, {'a': '1'})
cl1 = instantiate(_setting, S=S)
cl2 = instantiate(_setting, S=S)
assert len(cl1.derivations) == 1
assert len(cl2.derivations) == 1
assert cl1.derivations[0].dref == cl2.derivations[0].dref
assert cl1.dref == cl2.dref
def test_mknode(d) -> None:
with setup_storage('test_mknode'):
def _setting(m: Manager) -> DRef:
return mknode(m, d)
cl1 = instantiate(_setting)
cl2 = instantiate(_setting)
assert len(cl1.derivations) == 1
assert len(cl2.derivations) == 1
assert cl1.derivations[0].dref == cl2.derivations[0].dref
assert cl1.dref == cl2.dref
def test_mknode_with_artifacts(d, a) -> None:
with setup_storage('test_mknode_with_artifacts'):
def _setting(m: Manager) -> DRef:
return mknode(m, config_dict=d, artifacts=a)
cl = instantiate(_setting)
assert len(cl.derivations) == 1
rref = realize(instantiate(_setting))
for nm, val in a.items():
assert isfile(join(store_rref2path(rref),nm)), \
f"RRef {rref} doesn't contain artifact {nm}"
def run(mode: int = 2, interactive: bool = True):
maxitems = 5000
Wdef = 1
gather_depth = None
num_inputs = 2
if mode == 1:
batch_size = 5
elif mode == 2:
batch_size = 1
else:
assert False, 'Invalid mode'
def _stage(m: Manager):
inp = stage_inputs(m, num_inputs=num_inputs, batch_size=batch_size)
ds = stage_dataset2(m,
inp,
Wdef=Wdef,
maxitems=maxitems,
gather_depth=gather_depth)
df = stage_df(m, ds)
dsvis = stage_vis(m, df)
return dsvis
rref = realize(instantiate(_stage), force_rebuild=interactive)
linkrref(rref, '_results')
if interactive:
system(f"feh {mklens(rref).out_plot.syspath} &")
return rref
def test_repl_override():
with setup_storage('test_repl_override'):
n1: DRef
n2: DRef
def _setting(m: Manager) -> DRef:
nonlocal n1, n2
n1 = mkstage(m, {'a': '1'}, lambda i, tag: 33)
n2 = mkstage(m, {'maman': n1}, lambda i, tag: 42)
return n2
clo = instantiate(_setting)
rh = repl_realize(clo, force_interrupt=[n1])
assert rh.dref == n1
b = repl_build(rh)
with open(join(build_outpath(b), 'artifact'), 'w') as f:
f.write('777')
repl_continue(b.outgroups, rh=rh)
rref = repl_rref(rh)
assert rref is not None
rrefn1 = store_deref(rref, n1)[Tag('out')]
assert tryread(Path(join(store_rref2path(rrefn1),
'artifact'))) == '777'
def test_realize_dependencies() -> None:
with setup_storage2('test_realize_dependencies') as (T, S):
n1 = None
n2 = None
n3 = None
toplevel = None
def _setup(m):
nonlocal n1, n2, n3, toplevel
n1 = mkstage(m, {'a': 1})
n2 = mkstage(m, {'parent1': n1})
n3 = mkstage(m, {'parent2': n1})
toplevel = mkstage(m, {'n2': n2, 'n3': n3})
return toplevel
rref = realize(instantiate(_setup, S=S))
assert_valid_rref(rref)
assert n1 is not None
assert n2 is not None
assert n3 is not None
assert toplevel is not None
c = store_config(rref, S)
assert_valid_config(c)
cro = store_cattrs(rref, S)
assert_valid_dref(getattr(cro, 'n2'))
assert_valid_dref(getattr(cro, 'n3'))
assert set(store_deps([n1], S)) == set([])
assert set(store_deps([n2, n3], S)) == set([n1])
assert set(store_deps([toplevel], S)) == set([n2, n3])
assert store_deepdeps([n1], S) == set([])
assert store_deepdeps([n2, n3], S) == set([n1])
assert store_deepdeps([toplevel], S) == set([n1, n2, n3])
def test_fetchurl():
with setup_storage('test_fetchurl'):
with TemporaryDirectory() as tmp:
mockwget = join(tmp, 'mockwget')
mockdata = join(tmp, 'mockdata')
with open(mockdata, 'w') as f:
f.write('blala')
system(f'tar -zcvf {mockdata}.tar.gz {mockdata}')
with open(mockwget, 'w') as f:
f.write(f"#!/bin/sh\n")
f.write(f"mv {mockdata}.tar.gz $3\n")
chmod(mockwget, stat(mockwget).st_mode | S_IEXEC)
wanted_sha256 = pipe_stdout([SHA256SUM,
f"{mockdata}.tar.gz"]).split()[0]
import pylightnix.stages.fetch
oldwget = pylightnix.stages.fetch.WGET
try:
pylightnix.stages.fetch.WGET = lambda: mockwget
clo = instantiate(fetchurl,
url='mockwget://result.tar.gz',
filename='validname.tar.gz',
sha256=wanted_sha256)
rref = realize(clo)
finally:
pylightnix.stages.fetch.WGET = oldwget
def test_repl_realizeInval():
with setup_storage('test_repl_realizeInval'):
try:
repl_realize(instantiate(mkstage, {'a': 1}),
force_interrupt=33) #type:ignore
raise ShouldHaveFailed('wrong force_interrupt value')
except AssertionError:
pass
def test_minimal_closure() -> None:
with setup_storage2('test_minimal_closure') as (T, S):
def _somenode(m):
return mkstage(m, {'a': 0})
def _anothernode(m):
n1 = mkstage(m, {'a': 1})
n2 = _somenode(m)
n3 = mkstage(m, {'maman': n1, 'papa': n2})
return n3
rref1 = realize(instantiate(_somenode, S=S))
rref = realize(instantiate(_anothernode, S=S))
rref2 = store_deref(rref, store_cattrs(rref, S=S).papa,
S=S)[Tag('out')]
assert rref1 == rref2, '''
def test_mkfile() -> None:
with setup_storage('test_mkfile'):
def _setting(m: Manager, nm) -> DRef:
return mkfile(m, Name('foo'), bytes((nm or 'bar').encode('utf-8')),
nm)
rref1 = realize(instantiate(_setting, None))
with open(join(store_rref2path(rref1), 'foo'), 'r') as f:
bar = f.read()
assert bar == 'bar'
rref2 = realize(instantiate(_setting, 'baz'))
with open(join(store_rref2path(rref2), 'baz'), 'r') as f:
baz = f.read()
assert baz == 'baz'
assert rref1 != rref2
def test_repl_globalCancel():
with setup_storage('test_repl_globalCancel'):
n1: DRef
n2: DRef
def _setting(m: Manager) -> DRef:
nonlocal n1, n2
n1 = mkstage(m, {'a': '1'})
n2 = mkstage(m, {'maman': n1})
return n2
rh = repl_realize(instantiate(_setting), force_interrupt=True)
assert repl_rref(rh) is None
repl_cancel()
assert rh.gen is None
rref = realize(instantiate(_setting))
assert isrref(rref)
def test_repl_race():
with setup_storage('test_repl_recursion'):
def _setting(m: Manager) -> DRef:
n1 = mkstage(m, {'a': '1'})
n2 = mkstage(m, {'maman': n1})
return n2
clo = instantiate(_setting)
rh = repl_realize(clo, force_interrupt=True)
assert repl_rref(rh) is None
assert rh.dref == clo.dref
clo2 = instantiate(_setting)
rref2 = realize(clo2) # Realize dref while repl_realizing same dref
repl_cancel(rh)
assert_valid_rref(rref2)
def test_find():
with setup_storage('test_find') as s:
s1=partial(mkstage, config={'name':'1'}, nondet=lambda:42)
s2=partial(mkstage, config={'name':'2'}, nondet=lambda:33)
rref1=realize(instantiate(s1))
sleep(0.1)
now=parsetime(timestring())
rref2=realize(instantiate(s2))
rrefs=find()
assert set(rrefs)==set([rref1,rref2])
rrefs=find(name='1')
assert rrefs==[rref1]
rrefs=find(name=s2)
assert rrefs==[rref2]
rrefs=find(newer=-10)
assert len(rrefs)==2
rrefs=find(newer=now)
assert rrefs==[rref2]
def test_gc() -> None:
with setup_storage2('test_gc') as (T, S):
def _node1(m: Manager) -> DRef:
return mkstage(m, {'name': '1'})
def _node2(m: Manager) -> DRef:
return mkstage(m, {'name': '2', 'maman': _node1(m)})
def _node3(m: Manager) -> DRef:
return mkstage(m, {'name': '3', 'maman': _node1(m)})
r1 = realize(instantiate(_node1, S=S))
r2 = realize(instantiate(_node2, S=S))
r3 = realize(instantiate(_node3, S=S))
rm_drefs, rm_rrefs = store_gc([], [r2], S)
assert rm_drefs == {rref2dref(r) for r in [r3]}
assert rm_rrefs == {x for x in [r3]}
def test_no_foreign_dref_deps() -> None:
with setup_storage2('test_no_foreign_dref_deps') as (T, S):
def _setup(m):
clo = instantiate(mkstage, {'foo': 'bar'})
return mkstage(m, {'bogus': clo.dref})
try:
dref = instantiate(_setup, S=S)
raise ShouldHaveFailed(f"Should fail, but got {dref}")
except AssertionError:
pass
def test_realized() -> None:
with setup_storage('test_realized'):
def _setting(m: Manager, assume_realized: bool) -> DRef:
def _realize(b: Build):
if assume_realized:
raise ShouldHaveFailed('Should not call the real realizer')
return build_outpath(b)
return mkdrv(m, mkconfig({'name': '1'}), match_some(),
build_wrapper(_realize))
dref = instantiate(realized(_setting), assume_realized=True)
try:
rref = realize(dref)
raise ShouldHaveFailed('Should not be realized')
except AssertionError:
pass
rref = realize(instantiate(_setting, assume_realized=False))
rref2 = realize(instantiate(realized(_setting), assume_realized=True))
assert rref == rref2
def test_no_rref_deps() -> None:
with setup_storage2('test_no_rref_deps') as (T, S):
def _setup(m):
rref = realize(instantiate(mkstage, {'foo': 'bar'}))
n2 = mkstage(m, {'bogus': rref})
return n2
try:
dref = instantiate(_setup, S=S)
raise ShouldHaveFailed(f"Should fail, but got {dref}")
except AssertionError:
pass
def test_du():
with setup_storage('test_du') as s:
usage=du()
assert usage=={}
clo=instantiate(mkstage, {'name':'1'}, lambda:42)
usage=du()
assert clo.dref in usage
assert usage[clo.dref][0]>0
assert usage[clo.dref][1]=={}
rref=realize(clo)
usage=du()
assert rref in usage[clo.dref][1]
assert usage[clo.dref][1][rref]>0
def load3(index, allow_realize: bool = True):
""" Loads the stabilized dataset. Raises LookupError if realization is
forbidden and the data are absent. """
if allow_realize:
stage = stage_datasetS
else:
def _err(*args, **kwargs):
raise LookupError(
"Attempting to realize dataset section with index {index}")
stage = redefine(stage_datasetS, new_realizer=_err)
return realize(instantiate(stage, index=index, Nvalid=N1, npoints=2000))
def test_no_recursive_instantiate_with_same_manager() -> None:
with setup_storage2('test_no_recursive_instantiate_with_same_manager') as (
T, S):
def _setup(m):
derivs = instantiate_(m, _setup)
n2 = mkstage(m, {'bogus': derivs.dref})
return n2
try:
dref = instantiate(_setup, S=S)
raise ShouldHaveFailed(f"Should fail, but got {dref}")
except AssertionError:
pass
def test_linkrrefs() -> None:
with setup_storage2('test_linkrefs') as (T, S):
s1 = partial(mkstage,
config={
'name': '1',
'promise': [promise, 'artifact']
})
rref1 = realize(instantiate(s1, S=S))
l = linkrrefs([rref1, rref1], destdir=S, S=S)
assert len(l) == 2
assert str(l[0]) == join(S, 'result-1')
assert islink(join(S, 'result-1'))
l = linkrrefs([rref1], destdir=S, withtime=True, S=S)
assert S in l[0]
def test_lens_closures():
with setup_storage('test_lens_closures'):
def _setting(m:Manager)->DRef:
n1=mkstage(m, {'name':'1', 'x':33, 'promise':[promise,'artifact']})
n2=mkstage(m, {'name':'2', 'papa':n1, 'dict':{'d1':1} })
n3=mkstage(m, {'name':'3', 'maman':n2 })
return n3
clo=instantiate(_setting)
assert isclosure(clo)
rref=realize(mklens(clo).maman.papa.closure)
assert mklens(rref).x.val==33
assert open(mklens(rref).promise.syspath).read()=='0'
def test_promise() -> None:
with setup_storage2('test_promise') as (T, S):
def _setting(m: Manager, fullfill: bool) -> DRef:
n1 = mkstage(m, {'name': '1', 'promise': [promise, 'artifact']})
def _realize(b: Build):
o = build_outpath(b)
c = build_cattrs(b)
assert b.dref in c.promise
assert n1 in store_cattrs(c.maman, S).promise
assert build_path(b, c.promise) == join(o, 'uber-artifact')
assert build_path(b,
store_cattrs(c.maman,
S).promise) == build_path(
b, c.maman_promise)
if fullfill:
with open(build_path(b, c.promise), 'w') as f:
f.write('chickenpoop')
return mkdrv(m,
mkconfig({
'name': '2',
'maman': n1,
'promise': [promise, 'uber-artifact'],
'maman_promise': [n1, 'artifact']
}),
matcher=match_some(),
realizer=build_wrapper(_realize))
try:
rref = realize(instantiate(_setting, False, S=S))
raise ShouldHaveFailed('Promise trigger')
except AssertionError:
pass
rref = realize(instantiate(_setting, True, S=S))
assert_valid_rref(rref)
def test_shellref():
with setup_storage('test_shellref') as s:
with TemporaryDirectory() as tmp:
mockshell=join(tmp,'mockshell')
with open(mockshell,'w') as f:
f.write(f"#!/bin/sh\n")
f.write(f"pwd\n")
chmod(mockshell, stat(mockshell).st_mode | S_IEXEC)
environ['SHELL']=mockshell
rref=realize(instantiate(mkstage, {'a':1}))
shellref(rref)
shellref(rref2dref(rref))
shellref()
shell(store_rref2path(rref))
repl_realize(instantiate(mkstage, {'n':1}), force_interrupt=True)
b=repl_build()
o=build_outpath(b)
shell(b)
repl_cancelBuild(b)
try:
shellref('foo') # type:ignore
raise ShouldHaveFailed('shellref should reject garbage')
except AssertionError:
pass
def test_repeated_realize() -> None:
with setup_storage2('test_repeated_realize') as (T, S):
def _setting(m: Manager) -> DRef:
return mkstage(m, {'a': '1'})
clo = instantiate(_setting, S=S)
rref1 = realize(clo)
rref2 = realize(clo)
rref3 = realize(clo, force_rebuild=[clo.dref])
assert rref1 == rref2 and rref2 == rref3
try:
rref3 = realize(clo, force_rebuild='Foobar') # type:ignore
raise ShouldHaveFailed
except AssertionError:
pass
