def time_linker(name, linker):
steps_a = 10
steps_b = 100
x = tensor.vector()
a = build_graph(x, steps_a)
b = build_graph(x, steps_b)
f_a = function([x], a, mode=Mode(optimizer=None, linker=linker()))
f_b = function([x], b, mode=Mode(optimizer=None, linker=linker()))
f_a([2.0])
t0 = time.time()
f_a([2.0])
t1 = time.time()
f_b([2.0])
t2 = time.time()
f_b([2.0])
t3 = time.time()
t_a = t1 - t0
t_b = t3 - t2
print("%s takes %f s/Kop" % (name, (1000 * (t_b - t_a) /
(steps_b - steps_a))))
def visualize_states(hidden_states, updates, train_stream, valid_stream, args):
# Get all the hidden_states
filter_states = VariableFilter(theano_name_regex="hidden_state_.*")
all_states = filter_states(hidden_states)
all_states = sorted(all_states, key=lambda var: var.name[-1])
# Get all the hidden_cells
filter_cells = VariableFilter(theano_name_regex="hidden_cells_.*")
all_cells = filter_cells(hidden_states)
all_cells = sorted(all_cells, key=lambda var: var.name[-1])
# Handle the theano shared variables that allow carrying the hidden state
givens, f_updates = carry_hidden_state(updates, 1,
not (has_indices(args.dataset)))
# Compile the function
logger.info("The compilation of the function has started")
if args.rnn_type == "lstm" and args.visualize_cells:
compiled = theano.function(inputs=ComputationGraph(all_cells).inputs,
outputs=all_cells,
givens=givens,
updates=f_updates,
mode=Mode(optimizer='fast_compile'))
else:
compiled = theano.function(inputs=ComputationGraph(all_states).inputs,
outputs=all_states,
givens=givens,
updates=f_updates,
mode=Mode(optimizer='fast_compile'))
# Plot the function
plot("hidden_state", train_stream, compiled, args)
def check_updates(linker_name):
x = tensor.lscalar('input')
y = theano.shared(np.asarray(1, 'int64'), name='global')
f = theano.function([x], [x, x + 34], updates=[(y, x + 1)], mode=Mode(
optimizer=None, linker=linker_name))
g = theano.function([x], [x - 6], updates=[(y, y + 3)], mode=Mode(
optimizer=None, linker=linker_name))
assert f(3, output_subset=[]) == []
assert y.get_value() == 4
assert g(30, output_subset=[0]) == [24]
assert g(40, output_subset=[]) == []
assert y.get_value() == 10
def test1(self):
# this is a quick test after the LazyLinker branch merge
# to check that all the current modes can still be used.
linker_classes_involved = []
predef_modes = ['FAST_COMPILE', 'FAST_RUN', 'DEBUG_MODE']
# Use a new instance of ProfileMode instead of 'PROFILE_MODE' to
# avoid printing a profile mode summary in nose output
predef_modes.append(ProfileMode())
# Linkers to use with regular Mode
linkers = ['c|py', 'c|py_nogc', 'vm', 'vm_nogc', 'cvm', 'cvm_nogc']
modes = predef_modes + [Mode(linker, 'fast_run') for linker in linkers]
for mode in modes:
x = T.matrix()
y = T.vector()
f = theano.function([x, y], x + y, mode=mode)
# test that it runs something
f([[1, 2], [3, 4]], [5, 6])
linker_classes_involved.append(f.maker.mode.linker.__class__)
print 'MODE:', mode, f.maker.mode.linker, 'stop'
# regression check:
# there should be
# - VM_Linker
# - OpWiseCLinker (FAST_RUN)
# - WrapLinker (PROFILE_MODE)
# - PerformLinker (FAST_COMPILE)
# - DebugMode's Linker (DEBUG_MODE)
assert 5 == len(set(linker_classes_involved))
def test_partial_function_output_keys():
x = tensor.scalar('input')
y = 3 * x
f = theano.function([x], {'a': y * 5, 'b': y - 7}, mode=Mode(
optimizer=None, linker=vm.VM_Linker(allow_partial_eval=True)))
assert f(5, output_subset=['a'])['a'] == f(5)['a']
def test_modes(self):
# this is a quick test after the LazyLinker branch merge
# to check that all the current modes can still be used.
linker_classes_involved = []
predef_modes = ["FAST_COMPILE", "FAST_RUN", "DEBUG_MODE"]
# Linkers to use with regular Mode
if theano.config.cxx:
linkers = [
"py", "c|py", "c|py_nogc", "vm", "vm_nogc", "cvm", "cvm_nogc"
]
else:
linkers = ["py", "c|py", "c|py_nogc", "vm", "vm_nogc"]
modes = predef_modes + [Mode(linker, "fast_run") for linker in linkers]
for mode in modes:
x = tt.matrix()
y = tt.vector()
f = theano.function([x, y], x + y, mode=mode)
# test that it runs something
f([[1, 2], [3, 4]], [5, 6])
linker_classes_involved.append(f.maker.mode.linker.__class__)
# print 'MODE:', mode, f.maker.mode.linker, 'stop'
# regression check:
# there should be
# - `VMLinker`
# - OpWiseCLinker (FAST_RUN)
# - PerformLinker (FAST_COMPILE)
# - DebugMode's Linker (DEBUG_MODE)
assert 4 == len(set(linker_classes_involved))
def check_partial_function_output_keys(linker_name):
x = tensor.scalar('input')
y = 3 * x
f = theano.function([x], {'a': y * 5, 'b': y - 7}, mode=Mode(
optimizer=None, linker=linker_name))
assert f(5, output_subset=['a'])['a'] == f(5)['a']
def test_c(self):
for dtype in self.dtypes + self.bin_dtypes:
for op in self.reds:
self.with_mode(Mode(linker='c',
optimizer=mode_with_gpu.optimizer),
op,
dtype=dtype,
pre_scalar_op=self.pre_scalar_op)
def check_partial_function_output_keys(linker_name):
x = tensor.scalar("input")
y = 3 * x
f = theano.function(
[x], {"a": y * 5, "b": y - 7}, mode=Mode(optimizer=None, linker=linker_name)
)
assert f(5, output_subset=["a"])["a"] == f(5)["a"]
def test_callback_with_ifelse(self):
a, b, c = tensor.scalars('abc')
f = function([a, b, c],
ifelse(a, 2 * b, 2 * c),
mode=Mode(optimizer=None,
linker=vm.VM_Linker(callback=self.callback)))
f(1, 2, 3)
assert self.n_callbacks['IfElse'] == 2
def test_VMLinker_make_vm_cvm():
# We don't want this at module level, since CXX might not be present
from theano.link.c.cvm import CVM
a = tensor.scalar()
linker = VMLinker(allow_gc=False, use_cloop=True)
f = function([a], a, mode=Mode(optimizer=None, linker=linker))
assert isinstance(f.fn, CVM)
def time_linker(name, linker):
steps_a = 10
x = tensor.dvector()
a = build_graph(x, steps_a)
f_a = function([x], a, mode=Mode(optimizer=None, linker=linker()))
inp = np.random.rand(1000000)
for i in range(500):
f_a(inp)
def time_linker(name, linker):
steps_a = 10
x = tensor.vector()
a = build_graph(x, steps_a)
f_a = function([x], a, mode=Mode(optimizer=None, linker=linker()))
for i in xrange(500000):
f_a([2.0])
def check_partial_function(linker_name):
x = tensor.scalar("input")
y = x**2
f = theano.function([x], [y + 7, y - 9, y / 14.0],
mode=Mode(optimizer=None, linker=linker_name))
assert f(3, output_subset=[0, 1, 2]) == f(3)
assert f(4, output_subset=[0, 2]) == [f(4)[0], f(4)[2]]
utt.assert_allclose(f(5), np.array([32.0, 16.0, 1.7857142857142858]))
def test_callback_with_ifelse(self):
a, b, c = tensor.scalars("abc")
f = function(
[a, b, c],
ifelse(a, 2 * b, 2 * c),
mode=Mode(optimizer=None, linker=VMLinker(callback=self.callback)),
)
f(1, 2, 3)
assert self.n_callbacks["IfElse"] == 2
def test_callback(self):
a, b, c = tensor.scalars('abc')
f = function([a, b, c], (a + b) + c,
mode=Mode(optimizer=None,
linker=vm.VM_Linker(callback=self.callback)))
f(1, 2, 3)
assert sum(self.n_callbacks.values()) == len(f.maker.fgraph.toposort())
f(1, 2, 3)
assert (sum(
self.n_callbacks.values()) == len(f.maker.fgraph.toposort()) * 2)
def test_partial_function():
import numpy as np
from theano.tests import unittest_tools as utt
x = tensor.scalar('input')
y = x ** 2
f = theano.function([x], [y + 7, y - 9, y / 14.], mode=Mode(
optimizer=None, linker=vm.VM_Linker(allow_partial_eval=True)))
assert f(3, output_subset=[0, 1, 2]) == f(3)
assert f(4, output_subset=[0, 2]) == [f(4)[0], f(4)[2]]
utt.assert_allclose(f(5), np.array([32., 16., 1.7857142857142858]))
def test_c_nan(self):
for dtype in self.dtypes:
if not dtype.startswith('float'):
continue
for op in self.reds:
self.with_mode(Mode(linker='c',
optimizer=mode_with_gpu.optimizer),
op,
dtype=dtype,
test_nan=True,
pre_scalar_op=self.pre_scalar_op)
def time_linker(name, linker):
steps_a = 5
steps_b = 100
x = tensor.vector()
a = build_graph(x, steps_a)
b = build_graph(x, steps_b)
f_a = function(
[x],
a,
mode=Mode(optimizer=None, linker=linker()),
#profile='f_a speed test %s'%name,
)
f_b = function(
[x],
b,
mode=Mode(optimizer=None, linker=linker()),
#profile='f_b speed test %s'%name,
)
print f_a([2.0, 3.0])
t0 = time.time()
print f_a([2.0, 3.0])
t1 = time.time()
print f_b([2.0, 3.0])
t2 = time.time()
print f_b([2.0, 3.0])
t3 = time.time()
t_a = t1 - t0
t_b = t3 - t2
print "%s takes %f s/Kop" % (name, (1000 * (t_b - t_a) /
(steps_b - steps_a)))
def visualize_gates_soft(gate_values, hidden_states, updates, train_stream,
valid_stream, args):
# Handle the theano shared variables that allow carrying the hidden state
givens, f_updates = carry_hidden_state(updates, 1,
not (has_indices(args.dataset)))
# Compile the function
compiled = theano.function(inputs=ComputationGraph(gate_values).inputs,
outputs=gate_values,
givens=givens,
updates=f_updates,
mode=Mode(optimizer='fast_compile'))
plot("gates_soft", train_stream, compiled, args)
def time_linker(name, linker):
steps_a = 10
x = tensor.dvector()
a = build_graph(x, steps_a)
f_a = function([x], a, mode=Mode(optimizer=None, linker=linker()))
inp = numpy.random.rand(1000000)
for i in xrange(100):
f_a(inp)
if 0: # this doesn't seem to work, prints 0 for everything
import resource
pre = resource.getrusage(resource.RUSAGE_SELF)
post = resource.getrusage(resource.RUSAGE_SELF)
print pre.ru_ixrss, post.ru_ixrss
print pre.ru_idrss, post.ru_idrss
print pre.ru_maxrss, post.ru_maxrss
def test_VMLinker_no_cxx():
from importlib import reload
from unittest.mock import patch
with config.change_flags(cxx=""):
with pytest.raises(MissingGXX):
import theano.link.c.cvm
reload(theano.link.c.cvm)
with patch.dict("sys.modules", {"theano.link.c.cvm": None}):
linker = VMLinker(allow_gc=False, use_cloop=True)
a = tensor.scalar()
with pytest.raises(ModuleNotFoundError):
_ = function([a], a, mode=Mode(optimizer=None, linker=linker))
def test_c_thunks():
a = tensor.scalars('a')
b, c = tensor.vectors('bc')
cases = [False]
if theano.config.cxx:
cases.append(True)
for c_thunks in cases:
f = function([a, b, c],
ifelse(a, a * b, b * c),
mode=Mode(optimizer=None,
linker=vm.VM_Linker(c_thunks=c_thunks,
use_cloop=False)))
f(1, [2], [3, 2])
from nose.tools import assert_raises
assert_raises(ValueError, f, 0, [2], [3, 4])
assert any([hasattr(t, 'cthunk') for t in f.fn.thunks]) == c_thunks
def test_c_thunks():
a = tensor.scalars("a")
b, c = tensor.vectors("bc")
cases = [False]
if theano.config.cxx:
cases.append(True)
for c_thunks in cases:
f = function(
[a, b, c],
ifelse(a, a * b, b * c),
mode=Mode(optimizer=None,
linker=vm.VM_Linker(c_thunks=c_thunks, use_cloop=False)),
)
f(1, [2], [3, 2])
with pytest.raises(ValueError):
f(0, [2], [3, 4])
assert any([hasattr(t, "cthunk") for t in f.fn.thunks]) == c_thunks
def test_no_leak_many_graphs():
# Verify no memory leaks when creating and deleting a lot of functions
# This isn't really a unit test, you have to run it and look at top to
# see if there's a leak
for i in range(10000):
x = tensor.vector()
z = x
for d in range(10):
z = tensor.sin(-z + 1)
f = function([x], z, mode=Mode(optimizer=None, linker="cvm"))
if not i % 100:
print(gc.collect())
sys.stdout.flush()
gc.collect()
if 1:
f([2.0])
f([3.0])
f([4.0])
f([5.0])
def test_VMLinker_make_vm_no_cvm():
from importlib import reload
from unittest.mock import patch
with config.change_flags(cxx=""):
# Make sure that GXX isn't present
with pytest.raises(MissingGXX):
import theano.link.c.cvm
reload(theano.link.c.cvm)
# Make sure that `cvm` module is missing
with patch.dict("sys.modules", {"theano.link.c.cvm": None}):
a = tensor.scalar()
linker = VMLinker(allow_gc=False, use_cloop=True)
with pytest.raises(ModuleNotFoundError):
import theano.link.c.cvm
f = function([a], a, mode=Mode(optimizer=None, linker=linker))
assert isinstance(f.fn, Loop)
def visualize_presoft(cost, hidden_states, updates, train_stream, valid_stream,
args):
filter_presoft = VariableFilter(theano_name="presoft")
presoft = filter_presoft(ComputationGraph(cost).variables)[0]
# Get all the hidden_states
filter_states = VariableFilter(theano_name_regex="hidden_state_.*")
all_states = filter_states(hidden_states)
all_states = sorted(all_states, key=lambda var: var.name[-1])
# Assertion part
assert len(all_states) == args.layers
logger.info("The computation of the gradients has started")
gradients = []
for i in range(args.visualize_length - args.context):
gradients.extend(
tensor.grad(tensor.mean(tensor.abs_(presoft[i, 0, :])),
all_states))
logger.info("The computation of the gradients is done")
# Handle the theano shared variables that allow carrying the hidden state
givens, f_updates = carry_hidden_state(updates, 1,
not (has_indices(args.dataset)))
# Compile the function
logger.info("The compilation of the function has started")
compiled = theano.function(inputs=ComputationGraph(presoft).inputs,
outputs=gradients,
givens=givens,
updates=f_updates,
mode=Mode(optimizer='fast_compile'))
logger.info("The function has been compiled")
# Generate
epoch_iterator = train_stream.get_epoch_iterator()
for num in range(10):
init_ = next(epoch_iterator)[0][0:args.visualize_length, 0:1]
hidden_state = compiled(init_)
value_of_layer = {}
for d in range(args.layers):
value_of_layer[d] = 0
for i in range(len(hidden_state) / args.layers):
for d in range(args.layers):
value_of_layer[d] += hidden_state[d + i * args.layers]
time = hidden_state[0].shape[0]
if has_indices(args.dataset):
ticks = tuple(conv_into_char(init_[:, 0], args.dataset))
else:
ticks = tuple(np.arange(time))
for d in range(args.layers):
plt.plot(np.arange(time),
np.mean(np.abs(value_of_layer[d][:, 0, :]), axis=1),
label="Layer " + str(d))
plt.xticks(range(args.visualize_length), ticks)
plt.grid(True)
plt.title("hidden_state_of_layer_" + str(d))
plt.legend()
plt.tight_layout()
if args.local:
plt.show()
else:
plt.savefig(args.save_path + "/visualize_presoft_" + str(num) +
".png")
logger.info("Figure \"visualize_presoft_" + str(num) +
".png\" saved at directory: " + args.save_path)
def visualize_jacobian(hidden_states, updates,
train_stream, valid_stream,
args):
# Get all the hidden_states
all_states = [
var for var in hidden_states if re.match("hidden_state_.*", var.name)]
all_states = sorted(all_states, key=lambda var: var.name[-1])
# Get all the hidden_cells
all_cells = [var for var in hidden_states if re.match(
"hidden_cell_.*", var.name)]
all_cells = sorted(all_cells, key=lambda var: var.name[-1])
# Get the variable on which we compute the gradients
variables = ComputationGraph(hidden_states).variables
wrt = [
var for var in variables if
(var.name is not None) and (re.match("pre_rnn.*", var.name))]
wrt = sorted(wrt, key=lambda var: var.name[-1])
len_wrt = len(wrt)
# We have wrt = [pre_rnn] or [pre_rnn_0, pre_rnn_1, ...]
# Assertion part
assert len(all_states) == args.layers
assert len(all_cells) == (args.layers * (args.rnn_type == "lstm"))
if args.skip_connections:
assert len_wrt == args.layers
else:
assert len_wrt == 1
# Comupute the gradients of states or cells
if args.rnn_type == "lstm" and args.visualize_cells:
states = all_cells
else:
states = all_states
logger.info("The computation of the gradients has started")
gradients = []
for i, state in enumerate(states):
gradients.append(
tensor.grad(tensor.mean(tensor.abs_(
state[-1])), state))
# -1 indicates that gradient is gradient of the last time-step.c
logger.info("The computation of the gradients is done")
# Handle the theano shared variables for the state
state_vars = [theano.shared(
v[0:1, :].zeros_like().eval(), v.name + '-gen')
for v, _ in updates]
givens = [(v, x) for (v, _), x in zip(updates, state_vars)]
f_updates = [(x, upd) for x, (_, upd) in zip(state_vars, updates)]
# Compile the function
logger.info("The compilation of the function has started")
compiled = theano.function(inputs=ComputationGraph(states).inputs,
outputs=gradients,
givens=givens, updates=f_updates,
mode=Mode(optimizer='fast_compile'))
logger.info("The function has been compiled")
import ipdb
ipdb.set_trace()
# Generate
epoch_iterator = train_stream.get_epoch_iterator()
for num in range(10):
init_ = next(epoch_iterator)[0][
0: args.visualize_length, 0:1]
# [layers * len_wrt] [Time, 1, Hidden_dim]
gradients = compiled(init_)
time = gradients[0].shape[0]
if has_indices(args.dataset):
ticks = tuple(conv_into_char(init_[:, 0], args.dataset))
else:
ticks = tuple(np.arange(time))
# One row subplot for each variable wrt which we are computing
# the gradients
for var in range(len_wrt):
plt.subplot(len_wrt, 1, var + 1)
for d in range(args.layers - var):
plt.plot(
np.arange(time),
np.mean(np.abs(gradients[d][:, 0, :]), axis=1),
label="layer " + str(d + var))
plt.xticks(range(args.visualize_length), ticks)
plt.grid(True)
plt.yscale('log')
axes = plt.gca()
axes.set_ylim([5e-20, 5e-1])
plt.title("gradients plotting w.r.t pre_rrn" + str(var))
plt.legend()
plt.tight_layout()
if args.local:
plt.show()
else:
plt.savefig(
args.save_path + "/visualize_jacobian_" + str(num) + ".png")
logger.info("Figure \"visualize_jacobian_" + str(num) +
".png\" saved at directory: " + args.save_path)
def visualize_gradients(hidden_states, updates, train_stream, valid_stream,
args):
# Get all the hidden_states
filter_states = VariableFilter(theano_name_regex="hidden_state_.*")
all_states = filter_states(hidden_states)
all_states = sorted(all_states, key=lambda var: var.name[-1])
# Get all the hidden_cells
filter_cells = VariableFilter(theano_name_regex="hidden_cell_.*")
all_cells = filter_cells(hidden_states)
all_cells = sorted(all_cells, key=lambda var: var.name[-1])
# Get the variable on which we compute the gradients
filter_pre_rnn = VariableFilter(theano_name_regex="pre_rnn.*")
wrt = filter_pre_rnn(ComputationGraph(hidden_states).variables)
wrt = sorted(wrt, key=lambda var: var.name[-1])
len_wrt = len(wrt)
# We have wrt = [pre_rnn] or [pre_rnn_0, pre_rnn_1, ...]
# Assertion part
assert len(all_states) == args.layers
assert len(all_cells) == (args.layers * (args.rnn_type == "lstm"))
if args.skip_connections:
assert len_wrt == args.layers
else:
assert len_wrt == 1
# Comupute the gradients of states or cells
if args.rnn_type == "lstm" and args.visualize_cells:
states = all_cells
else:
states = all_states
logger.info("The computation of the gradients has started")
gradients = []
for i, state in enumerate(states):
gradients.extend(
tensor.grad(tensor.mean(tensor.abs_(state[-1, 0, :])),
wrt[:i + 1]))
# -1 indicates that gradient is gradient of the last time-step.c
logger.info("The computation of the gradients is done")
# Handle the theano shared variables that allow carrying the hidden state
givens, f_updates = carry_hidden_state(
updates, 1, reset=not (has_indices(args.dataset)))
# Compile the function
logger.info("The compilation of the function has started")
compiled = theano.function(inputs=ComputationGraph(states).inputs,
outputs=gradients,
givens=givens,
updates=f_updates,
mode=Mode(optimizer='fast_compile'))
logger.info("The function has been compiled")
# Generate
epoch_iterator = train_stream.get_epoch_iterator()
for num in range(10):
init_ = next(epoch_iterator)[0][0:args.visualize_length, 0:1]
# [layers * len_wrt] [Time, 1, Hidden_dim]
gradients = compiled(init_)
if args.skip_connections:
assert len(gradients) == (args.layers * (args.layers + 1)) / 2
else:
assert len(gradients) == args.layers
time = gradients[0].shape[0]
if has_indices(args.dataset):
ticks = tuple(conv_into_char(init_[:, 0], args.dataset))
else:
ticks = tuple(np.arange(time))
# One row subplot for each variable wrt which we are computing
# the gradients
for var in range(len_wrt):
plt.subplot(len_wrt, 1, var + 1)
for d in range(args.layers - var):
plt.plot(np.arange(time),
np.mean(np.abs(gradients[d][:, 0, :]), axis=1),
label="layer " + str(d + var))
plt.xticks(range(args.visualize_length), ticks)
plt.grid(True)
plt.yscale('log')
axes = plt.gca()
axes.set_ylim([5e-20, 5e-1])
plt.title("gradients plotting w.r.t pre_rrn" + str(var))
plt.legend()
plt.tight_layout()
if args.local:
plt.show()
else:
plt.savefig(args.save_path + "/visualize_gradients_" + str(num) +
".png")
logger.info("Figure \"visualize_gradients_" + str(num) +
".png\" saved at directory: " + args.save_path)
self.err = 0.5 * T.sum((out - x)**2)
err = self.err
params = [self.w, self.a, self.b]
gparams = T.grad(err, params)
updates = [(p, p - 0.01 * gp) for p, gp in zip(params, gparams)]
self.step = module.Method([x], err, updates=dict(updates))
mod = M()
mode = 'FAST_RUN'
#mode = ProfileMode(optimizer='fast_run', linker=theano.gof.OpWiseCLinker())
mode = Mode(optimizer='fast_run', linker=theano.gof.OpWiseCLinker(nice_errors=True))
mode = Mode(optimizer='fast_run', linker='c')
mode = Mode(optimizer='fast_run', linker='c|py')
print mod.pretty(mode=mode)
m = mod.make(mode=mode)
neg, nout, nhid, niter = [int(a) for a in sys.argv[1:]]
rng = numpy.random.RandomState(342)
m.w = rng.rand(nout, nhid)
m.a = rng.randn(nhid) * 0.0
m.b = rng.randn(nout) * 0.0
x = (rng.rand(neg, nout)-0.5) * 1.5
t = time.time()
for i in xrange(niter):
self.err = 0.5 * T.sum((out - x)**2)
err = self.err
params = [self.w, self.a, self.b]
gparams = T.grad(err, params)
updates = [(p, p - 0.01 * gp) for p, gp in zip(params, gparams)]
self.step = module.Method([x], err, updates=dict(updates))
mod = M()
mode = 'FAST_RUN'
mode = Mode(optimizer='fast_run',
linker=theano.gof.OpWiseCLinker(nice_errors=True))
mode = Mode(optimizer='fast_run', linker='c')
mode = Mode(optimizer='fast_run', linker='c|py')
print(mod.pretty(mode=mode))
m = mod.make(mode=mode)
neg, nout, nhid, niter = [int(a) for a in sys.argv[1:]]
rng = numpy.random.RandomState(342)
m.w = rng.rand(nout, nhid)
m.a = rng.randn(nhid) * 0.0
m.b = rng.randn(nout) * 0.0
x = (rng.rand(neg, nout) - 0.5) * 1.5
t = time.time()
for i in xrange(niter):
