def test_config_ro():
d = {'a': 1, 'b': 33}
c = mkconfig(d)
cro = config_cattrs(RConfig(
config_dict(c))) # We sure we don't have promises here
for k in d.keys():
assert getattr(cro, k) == d[k]
def _setting(m: Manager, fullfill: bool) -> DRef:
n1 = mkstage(m, {'name': '1', 'promise': [promise, 'artifact']})
def _realize(b: Build):
o = build_outpath(b)
c = build_cattrs(b)
assert b.dref in c.promise
assert n1 in store_cattrs(c.maman, S).promise
assert build_path(b, c.promise) == join(o, 'uber-artifact')
assert build_path(b,
store_cattrs(c.maman,
S).promise) == build_path(
b, c.maman_promise)
if fullfill:
with open(build_path(b, c.promise), 'w') as f:
f.write('chickenpoop')
return mkdrv(m,
mkconfig({
'name': '2',
'maman': n1,
'promise': [promise, 'uber-artifact'],
'maman_promise': [n1, 'artifact']
}),
matcher=match_some(),
realizer=build_wrapper(_realize))
def mnist_config(mnist:DRef)->Config: name = 'convnn_mnist' dataset:RefPath = [mnist, 'mnist.npz'] learning_rate = 1e-3 num_epoches = 1 version = 7 return mkconfig(locals())
def _setting(m: Manager, assume_realized: bool) -> DRef:
def _realize(b: Build):
if assume_realized:
raise ShouldHaveFailed('Should not call the real realizer')
return build_outpath(b)
return mkdrv(m, mkconfig({'name': '1'}), match_some(),
build_wrapper(_realize))
def evolution_config()->Config: name = 'ultimatum' nepoch = 30000 n = 300 nrounds = 10*30 cutoff = 0.1 version = 6 nrunners = 10 return mkconfig(locals())
def stage_datasetS(m: Manager,
index: int = 0,
Nvalid: int = 5000,
npoints: int = 100) -> DRef:
""" Creates stabilized dataset """
def _config():
name = 'dataset3'
nonlocal index, npoints
N = int(5 * Nvalid / 3)
min_allow_size = Nvalid
out_examples = [promise, 'examples.bin']
out_barplots = [promise, 'barplots']
out_traceplot = [promise, 'traceplot.png']
out_inputs = [promise, 'inputs.json']
version = 2
return locals()
def _make(b: Build):
build_setoutpaths(b, 1)
rref = run_dataset2(what=2,
maxitems=mklens(b).N.val,
index=index,
interactive=False)
assert isfile(mklens(rref).ref_df.examples.syspath)
system(
f"cp {mklens(rref).ref_df.ref_data.inputs.syspath} {mklens(b).out_inputs.syspath}"
)
df = pd.read_csv(mklens(rref).df.syspath)
makedirs(mklens(b).out_barplots.syspath)
allowed = stabilize(
df,
npoints=mklens(b).npoints.val,
path_barplot=lambda i: join(
mklens(b).out_barplots.syspath, f'_plot_{i:03d}.png'),
path_traceplot=mklens(b).out_traceplot.syspath,
min_allow_size=mklens(b).min_allow_size.val)
with open(mklens(rref).ref_df.examples.syspath, 'rb') as f:
with open(mklens(b).out_examples.syspath, 'wb') as f_o:
_read = fd2examples(f)
_write = examples2fd(f_o)
try:
idx = 0
while True:
example = _read()
if idx in allowed:
_write(example)
idx += 1
except KeyboardInterrupt:
raise
except Exception as e:
print(e)
pass
return mkdrv(m, mkconfig(_config()), match_only(), build_wrapper(_make))
def stage_df(m: Manager, ref_data: DRef):
def _config():
nonlocal ref_data
name = mklens(ref_data).name.val + '-df'
examples = mklens(ref_data).out_examples.refpath
out_df = [promise, 'df.csv']
return locals()
def _make(b: Build):
build_setoutpaths(b, 1)
df = examples_dataframe(mklens(b).examples.syspath)
df.to_csv(mklens(b).out_df.syspath)
return mkdrv(m, mkconfig(_config()), match_only(), build_wrapper(_make))
def stage_vis(m: Manager, ref_df: DRef):
def _config():
name = mklens(ref_df).name.val + '-vis'
df = mklens(ref_df).out_df.refpath
out_plot = [promise, 'plot.png']
title = mklens(ref_df).ref_data.name.val
version = 3
return locals()
def _make(b: Build):
build_setoutpaths(b, 1)
df = pd.read_csv(mklens(b).df.syspath)
vis_bars(df,
plot_fpath=mklens(b).out_plot.syspath,
async_plot=None,
plot_title=mklens(b).title.val)
return mkdrv(m, mkconfig(_config()), match_latest(), build_wrapper(_make))
def stage_inputs(m:Manager, num_inputs:int=4, batch_size:int=5, index:int=1)->DRef:
range_min = -100
range_max = 100
def _config():
name = 'dataset-inputs'
nonlocal num_inputs, batch_size, range_min, range_max, index
out_inputs = [promise, 'inputs.json']
version = 4
return locals()
def _make(b:Build):
build_setoutpaths(b, 1)
builtin_inputs:dict={
# Ref("i0"):IVal(0),
# Ref("i1"):IVal(1)
}
IMEMs:List[IMem] = [mkmap(_union(builtin_inputs,
{Ref(f"i{i+len(builtin_inputs.keys())}"): IVal(randint(range_min,range_max))
for i in range(num_inputs)}))
for _ in range(batch_size)]
writejson(mklens(b).out_inputs.syspath, [imem2json(M) for M in IMEMs])
return mkdrv(m, mkconfig(_config()), match_only(), build_wrapper(_make))
def stage_hello(m: Manager) -> DRef:
def _config() -> Config:
name: str = 'hello-bin'
src: RefPath = [hello_src, f'hello-{hello_version}']
return locals()
def _make(b: Build) -> None:
o: Path = build_outpath(b)
with TemporaryDirectory() as tmp:
copytree(mklens(b).src.syspath, join(tmp, 'src'))
dirrw(Path(join(tmp, 'src')))
cwd = getcwd()
try:
chdir(join(tmp, 'src'))
system('./configure --prefix=/usr')
system('make')
system(f'make install DESTDIR={o}')
finally:
chdir(cwd)
return mkdrv(m, mkconfig(_config()), match_latest(), build_wrapper(_make))
def stage_dataset1(m:Manager, ref_inputs:DRef)->DRef:
Wdef = 5
lib_impl = lib_arith
lib_methods = [mn.val for mn in lib_impl.keys()]
time2run_sec = int(0.5*60)
def _config():
nonlocal Wdef,lib_methods,time2run_sec
num_inputs = mklens(ref_inputs).num_inputs.val
batch_size = mklens(ref_inputs).batch_size.val
inputs = mklens(ref_inputs).out_inputs.refpath
out_examples = [promise, 'examples.bin']
version = ['0']
return locals()
def _make(b:Build):
build_setoutpaths(b, 1)
WLIB = mkwlib(lib_impl, Wdef)
IMEMs = [json2imem(j) for j in readjson(mklens(b).inputs.syspath)]
print(f"Inputs: {IMEMs}")
i = 0
acc:List[Expr] = []
g = genexpr(WLIB, IMEMs)
written_bytes = 0
time_start = time()
with open(mklens(b).out_examples.syspath,'wb') as f:
_add=examples2fd(f)
while time()<time_start+mklens(b).time2run_sec.val:
r,mem,vals,w = next(g)
ival = vals[0]
assert isinstance(ival[r], IVal)
expr = gather(r,mem)
acc.append(expr)
i += 1
for j in range(len(IMEMs)):
written_bytes+=_add(Example(IMEMs[j],expr,vals[j][r]))
if i%300 == 0:
print(f".. i {i} W {w} LAST_REF {r} WRBYTES {written_bytes}.. ")
return mkdrv(m, mkconfig(_config()), match_only(), build_wrapper(_make))
def _setting(m:Manager)->DRef:
n1=mkstage(m, {'name':'1', 'promise':[promise,'artifact']})
n2=mkstage(m, {'name':'2', 'promise':[promise,'artifact'],
'dict':{'d1':1} })
def _realize(b:Build):
o=build_outpath(b)
c=build_cattrs(b)
assert isrefpath(mklens(b).maman.promise.refpath)
assert isfile(mklens(b).papa.promise.syspath)
assert o in mklens(b).promise.syspath
assert o == mklens(b).syspath
assert mklens(b).papa.name.val == '2'
assert mklens(b).papa.dref == c.papa
with open(mklens(b).promise.syspath,'w') as f:
f.write('chickenpoop')
return mkdrv(m,
mkconfig({'name':'3', 'maman':n1, 'papa':n2,
'promise':[promise,'artifact'],
}),
matcher=match_some(),
realizer=build_wrapper(_realize))
def mnist_config()->Config: learning_rate = 1e-3 num_epoches = 1 dataset:RefPath = [mnist_dataset, 'mnist.npz'] accuracy:PromisePath = [promise, 'accuracy.txt'] return mkconfig(locals())
def test_config2() -> None:
try:
c = mkconfig({'a': (3, 1)})
raise ShouldHaveFailed(f'RConfig {c} is surprizingly valid')
except AssertionError:
pass
def hello_config() -> Config:
name = 'hello-bin'
src = mklens(hello_src).src.refpath
return mkconfig(locals())
def stage_dataset2(m:Manager, ref_inputs:DRef,
gather_depth:Optional[int]=999,
Wdef:int=5,
maxtime_sec:Optional[int]=None,
maxitems:Optional[int]=None)->DRef:
lib_impl = lib_arith
lib_methods = [mn.val for mn in lib_impl.keys()]
assert (maxtime_sec is not None) or (maxitems is not None), \
"At least one stop criteria should be defined"
time2run_sec:int = maxsize if maxtime_sec is None else maxtime_sec
maxitems:int = maxsize if maxitems is None else maxitems
def _config():
nonlocal Wdef,lib_methods,time2run_sec,maxitems,gather_depth
num_inputs = mklens(ref_inputs).num_inputs.val
batch_size = mklens(ref_inputs).batch_size.val
name=f'dataset2-ni{num_inputs}-bs{batch_size}-Wd{Wdef}-mi{maxitems}'
inputs = mklens(ref_inputs).out_inputs.refpath
out_examples = [promise, 'examples.bin']
version = ['21']
return locals()
def _make(b:Build):
build_setoutpaths(b, 1)
WLIB = mkwlib(lib_impl, Wdef)
IMEMs = [json2imem(j) for j in readjson(mklens(b).inputs.syspath)]
print(f"Inputs: {IMEMs}")
i = 0
# acc:List[Expr] = []
g = genexpr(WLIB, IMEMs)
written_bytes = 0
written_items = 0
time_start = time()
acci=set()
hb=time()
with open(mklens(b).out_examples.syspath,'wb') as f:
_add=examples2fd(f)
while time()<time_start+mklens(b).time2run_sec.val and \
written_items<mklens(b).maxitems.val:
# gt0=time()
r,mem,imems,exprw = next(g)
# gt1=time()
# print('gen time', gt1-gt0)
assert isinstance(imems[0][r], IVal), f"{imems[0][r]}"
i += 1
gi = 0
gexpr = gengather(r,mem)
# gg0=time()
exprs=[]
for expr in gexpr:
exprs.append(expr)
if gather_depth is not None:
exprs=exprs[-gather_depth:]
else:
exprs=[exprs[randint(0,len(exprs)-1)]]
for expr in exprs:
er=extrefs(expr)
ds=decls(expr)
# acc.append(expr)
for j in range(len(IMEMs)):
if len(ds)>0:
# print(gi, j, list(inps.keys()), print_expr(expr))
inps:IMem = TMap({k:imems[j][k] for k in er})
acci |= set(inps.values())
written_bytes+=_add(Example(inps,expr,imems[j][r]))
written_items+=1
hb2=time()
if written_items%100 == 0:
print(f".. NW {written_items} W {exprw[r]} DEP {depth(expr)} "
f"LAST_REF {r} WRBYTES {written_bytes // (1024) }K "
f"INPSZ {len(acci)} TIME {hb2-hb} "
f"VM {virtual_memory().used // 1024 // 1024}M")
hb=hb2
gi+=1
# gg1=time()
# print('gather time', gg1-gg0)
return mkdrv(m, mkconfig(_config()), match_only(), build_wrapper(_make))
def summary_config(evolution:DRef)->Config: name = 'summary' history_refpath = [evolution, 'history.json'] return mkconfig(locals())
def _config() -> Config:
c = deepcopy(config)
c['artifact'] = [promise_strength, 'artifact']
return mkconfig(c)