def mnist_build(b:Build)->None:
o = build_outpath(b)
c = build_cattrs(b)
with load(build_path(b, c.dataset), allow_pickle=True) as f:
x_train, y_train = f['x_train'], f['y_train']
x_test, y_test = f['x_test'], f['y_test']
x_train = x_train.reshape(x_train.shape[0], 28, 28, 1).astype('float32') / 255
y_train = to_categorical(y_train, 10)
x_test = x_test.reshape(x_test.shape[0], 28, 28, 1).astype('float32') / 255
y_test = to_categorical(y_test, 10)
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
model = Sequential()
model.add(Conv2D(32, kernel_size=(3, 3), activation = 'relu', input_shape = (28,28,1)))
model.add(Conv2D(64, (3, 3), activation = 'relu'))
model.add(MaxPool2D(pool_size = (2,2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128, activation = 'relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation = 'softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics = ['accuracy'])
model.fit(x_train, y_train, batch_size = 32, epochs = c.num_epoches, verbose = 0)
accuracy = model.evaluate(x_test, y_test, verbose = 0)[-1]
model.save_weights(join(o, 'weights.h5'), save_format='h5')
with open(build_path(b,c.accuracy),'w') as f:
f.write(str(accuracy))
def test_repl_override():
with setup_storage('test_repl_override'):
n1: DRef
n2: DRef
def _setting(m: Manager) -> DRef:
nonlocal n1, n2
n1 = mkstage(m, {'a': '1'}, lambda i, tag: 33)
n2 = mkstage(m, {'maman': n1}, lambda i, tag: 42)
return n2
clo = instantiate(_setting)
rh = repl_realize(clo, force_interrupt=[n1])
assert rh.dref == n1
b = repl_build(rh)
with open(join(build_outpath(b), 'artifact'), 'w') as f:
f.write('777')
repl_continue(b.outgroups, rh=rh)
rref = repl_rref(rh)
assert rref is not None
rrefn1 = store_deref(rref, n1)[Tag('out')]
assert tryread(Path(join(store_rref2path(rrefn1),
'artifact'))) == '777'
def summary_realize(b:Build)->None:
cwd = getcwd()
try:
chdir(build_outpath(b))
c = build_cattrs(b)
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
ax.set_ylabel('mean strategies')
ax.grid()
for nhist,histpath in enumerate(build_paths(b, c.history_refpath)):
epoches:List[float]=[]; pmeans:List[float]=[]; rmeans:List[float]=[]
with open(histpath,'r') as f:
epoches,pmeans,rmeans=json_loads(f.read())
if nhist==0:
pargs = {'label':'Proposer mean'}
rargs = {'label':'Responder mean'}
else:
pargs = {}; rargs = {}
ax.plot(epoches,pmeans,color='blue',**pargs)
ax.plot(epoches,rmeans,color='orange',**rargs)
plt.savefig('figure.png')
ax.legend(loc='upper right')
finally:
chdir(cwd)
def mnist_eval(b:Model):
o = build_outpath(b)
b.model.load(join(o, "checkpoint.ckpt"))
accuracy = b.model.evaluate(b.x_test, b.y_test, verbose = 0)[-1]
print(accuracy)
with open(join(o,'accuracy.txt'),'w') as f:
f.write(str(accuracy))
def _realize(b:Build):
o=build_outpath(b)
c=build_cattrs(b)
assert isrefpath(mklens(b).maman.promise.refpath)
assert isfile(mklens(b).papa.promise.syspath)
assert o in mklens(b).promise.syspath
assert o == mklens(b).syspath
assert mklens(b).papa.name.val == '2'
assert mklens(b).papa.dref == c.papa
with open(mklens(b).promise.syspath,'w') as f:
f.write('chickenpoop')
def _realize(b: Build):
o = build_outpath(b)
c = build_cattrs(b)
assert b.dref in c.promise
assert n1 in store_cattrs(c.maman, S).promise
assert build_path(b, c.promise) == join(o, 'uber-artifact')
assert build_path(b,
store_cattrs(c.maman,
S).promise) == build_path(
b, c.maman_promise)
if fullfill:
with open(build_path(b, c.promise), 'w') as f:
f.write('chickenpoop')
def _make(b: Build) -> None:
o: Path = build_outpath(b)
with TemporaryDirectory() as tmp:
copytree(mklens(b).src.syspath, join(tmp, 'src'))
dirrw(Path(join(tmp, 'src')))
cwd = getcwd()
try:
chdir(join(tmp, 'src'))
system('./configure --prefix=/usr')
system('make')
system(f'make install DESTDIR={o}')
finally:
chdir(cwd)
def _build(b:Build)->None:
chdir(build_outpath(b))
c=build_cattrs(b)
fig=plt.figure()
ax=fig.add_subplot(1, 1, 1)
ax.set_ylabel('mean strategies')
ax.grid()
for nhist,histpath in enumerate(build_paths(b, c.history)):
epoches:List[float]=[]; pmeans:List[float]=[]; rmeans:List[float]=[]
with open(histpath,'r') as f:
epoches,pmeans,rmeans=json_loads(f.read())
ax.plot(epoches,pmeans,label=f'pmeans{nhist}',color='blue')
ax.plot(epoches,rmeans,label=f'rmeans{nhist}',color='orange')
plt.savefig('figure.png')
def mnist_train(b:Model)->None:
o = build_outpath(b)
c = build_cattrs(b)
with np_load(build_path(b, c.dataset), allow_pickle=True) as f:
b.x_train, b.y_train = f['x_train'], f['y_train']
b.x_test, b.y_test = f['x_test'], f['y_test']
b.x_train = b.x_train.reshape(b.x_train.shape[0], 28, 28, 1).astype('float32') / 255
b.y_train = to_categorical(b.y_train, 10)
b.x_test = b.x_test.reshape(b.x_test.shape[0], 28, 28, 1).astype('float32') / 255
b.y_test = to_categorical(b.y_test, 10)
print('x_train shape:', b.x_train.shape)
print(b.x_train.shape[0], 'train samples')
print(b.x_test.shape[0], 'test samples')
model = Sequential()
b.model = model
model.add(Conv2D(32, kernel_size=(3, 3), activation = 'relu', input_shape = (28,28,1)))
model.add(Conv2D(64, (3, 3), activation = 'relu'))
model.add(MaxPool2D(pool_size = (2,2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128, activation = 'relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation = 'softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
callbacks = [
ModelCheckpoint(
monitor='val_accuracy',
filepath=join(o, "checkpoint.ckpt"),
save_weights_only=True,
save_best_only=True,
verbose=True)]
model.fit(b.x_train, b.y_train,
batch_size=32,
epochs=c.num_epoches,
verbose=0,
callbacks=callbacks,
validation_split=0.2)
def test_repl_globalHelper():
with setup_storage('test_repl_globalHelper'):
n1: DRef
n2: DRef
def _setting(m: Manager) -> DRef:
nonlocal n1, n2
n1 = mkstage(m, {'a': '1'})
n2 = mkstage(m, {'maman': n1})
return n2
rh = repl_realize(instantiate(_setting), force_interrupt=True)
assert repl_rref(rh) is None
b = repl_build()
with open(join(build_outpath(b), 'artifact.txt'), 'w') as f:
f.write("Fooo")
repl_continueBuild(b)
rref = repl_rref(rh)
assert rref is not None
assert isfile(join(store_rref2path(rref), 'artifact.txt'))
def test_shellref():
with setup_storage('test_shellref') as s:
with TemporaryDirectory() as tmp:
mockshell=join(tmp,'mockshell')
with open(mockshell,'w') as f:
f.write(f"#!/bin/sh\n")
f.write(f"pwd\n")
chmod(mockshell, stat(mockshell).st_mode | S_IEXEC)
environ['SHELL']=mockshell
rref=realize(instantiate(mkstage, {'a':1}))
shellref(rref)
shellref(rref2dref(rref))
shellref()
shell(store_rref2path(rref))
repl_realize(instantiate(mkstage, {'n':1}), force_interrupt=True)
b=repl_build()
o=build_outpath(b)
shell(b)
repl_cancelBuild(b)
try:
shellref('foo') # type:ignore
raise ShouldHaveFailed('shellref should reject garbage')
except AssertionError:
pass
def _realize(b: Build):
with open(join(build_outpath(b), 'exe'), 'w') as f:
f.write('#!/bin/sh\necho "Fooo"')
chmod(join(build_outpath(b), 'exe'), S_IWRITE | S_IREAD | S_IEXEC)
def _realize(b: Build):
if assume_realized:
raise ShouldHaveFailed('Should not call the real realizer')
return build_outpath(b)