def test_shape_of_variables_simple():
x = theano.tensor.matrix('x')
y = x + x
fgraph = theano.FunctionGraph([x], [y])
assert shape_of_variables(fgraph, {x: (5, 5)}) == {x: (5, 5), y: (5, 5)}
x = theano.tensor.matrix('x')
y = theano.tensor.dot(x, x.T)
fgraph = theano.FunctionGraph([x], [y])
shapes = shape_of_variables(fgraph, {x: (5, 1)})
assert shapes[x] == (5, 1)
assert shapes[y] == (5, 5)
def test_simple(self):
x = theano.tensor.matrix("x")
y = x + x
fgraph = theano.FunctionGraph([x], [y], clone=False)
shapes = shape_of_variables(fgraph, {x: (5, 5)})
assert shapes == {x: (5, 5), y: (5, 5)}
x = theano.tensor.matrix("x")
y = theano.tensor.dot(x, x.T)
fgraph = theano.FunctionGraph([x], [y], clone=False)
shapes = shape_of_variables(fgraph, {x: (5, 1)})
assert shapes[x] == (5, 1)
assert shapes[y] == (5, 5)
def cpu_to_gpu_graph(inputs, outputs):
""" Converts a cpu-only subgraph into a gpu-only subgraph
>>> x, y = theano.tensor.matrix('x'), theano.tensor.matrix('y')
>>> z = theano.tensor.dot(x, y)
>>> gpu_inputs, gpu_outputs = cpu_to_gpu_graph((x,y), (z,))
>>> f = theano.function(gpu_inputs, gpu_outputs)
>>> theano.printing.debugprint(f)
GpuDot22 [@A] '' 0
|gpu_x [@B]
|gpu_y [@C]
"""
math_opt = theano.compile.optdb.query('-inplace', '+fast_run', '-gpu')
gpu_opt = cuda.opt.gpu_optimizer.query('+gpu', '-inplace', '-async')
gpu_comm = cuda.opt.gpu_cut_copies.query('+gpu')
gpu_inputs, cpu_inputs = zip(*map(cpu_to_gpu_var, inputs))
outputs2 = theano.clone(outputs, replace=dict(zip(inputs, cpu_inputs)))
gpu_outputs = map(theano.sandbox.cuda.basic_ops.gpu_from_host, outputs2)
fgraph = theano.FunctionGraph(gpu_inputs, gpu_outputs)
math_opt.optimize(fgraph)
gpu_opt.optimize(fgraph)
gpu_comm.optimize(fgraph)
fgraph.disown()
for go, co in zip(gpu_outputs, outputs):
go.name = gpu_name(co.name)
return tuple(gpu_inputs), tuple(gpu_outputs)
def test_give_variables_names_small():
x = theano.tensor.matrix('x')
y = theano.tensor.dot(x, x)
fgraph = theano.FunctionGraph((x, ), (y, ))
give_variables_names(fgraph.variables)
assert all(var.name for var in fgraph.variables)
assert unique([var.name for var in fgraph.variables])
def test_shape_of_variables():
x = theano.tensor.matrix('x')
y = x[1:, 2:]
known_shapes = shape_of_variables((x, ), (y, ), {x: (10, 10)})
assert known_shapes[y] == (9, 8)
assert not isinstance(known_shapes[y][0], np.ndarray)
# just make sure that we can do this afterwards
fgraph = theano.FunctionGraph((x, ), (y, ))
def test_read_write_graph():
x = theano.tensor.matrix('x', dtype='float32')
y = theano.tensor.matrix('y', dtype='float32')
z = x + y
fname = testdir + "test_read_write_fgraph"
write_graph(((x, y), (z, )), fname)
fgraph = theano.FunctionGraph((x, y), (z, ))
fgraph2 = read_graph(fname)
assert str(fgraph) == str(fgraph2)
assert isinstance(fgraph2, theano.FunctionGraph)
def test_make_ilp():
x = theano.tensor.matrix('x')
y = theano.tensor.matrix('y')
z = theano.tensor.dot(x, x) + y[:, 0].sum() - x * y
env = theano.FunctionGraph([x, y], [z])
machine_ids = ["ankaa", "arroyitos"]
prob, X, S, Cmax = make_ilp(env, machine_ids, dummy_compute_cost,
dummy_comm_cost, dummy_ability, 100)
prob.solve()
assert Cmax.value() == 5
def test_compute_schedule():
x = theano.tensor.matrix('x')
y = theano.tensor.matrix('y')
z = theano.tensor.dot(x, x) + y[:, 0].sum() - x * y
env = theano.FunctionGraph([x, y], [z])
machine_ids = ["ankaa", "arroyitos"]
sched = compute_schedule(*make_ilp(env, machine_ids, dummy_compute_cost,
dummy_comm_cost, dummy_ability, 100))
# nodes are the jobs
assert env.apply_nodes == set([job for job, (time, id) in sched])
times = [time for job, (time, id) in sched]
# jobs are sorted by time
assert list(sorted(times)) == times
# the machine ids match what we put in
assert all(id in machine_ids for job, (time, id) in sched)
def unpack(source):
"""
Unpickle a packed string into a FunctionGraph
source can be a
string
file
if the source is a file then one can call pack and unpack many times on the
same file
See Also:
pack
"""
if isinstance(source, str):
ins, outs = cPickle.loads(source)
elif isinstance(source, file):
ins, outs = cPickle.load(source)
env = theano.FunctionGraph(ins, outs)
return env
def _env(self):
inputs = [inp.clone() for inp in self.apply.inputs]
output = self.apply.op(inputs)
env = theano.FunctionGraph(inputs, [output])
return env
def test_err(self):
x = theano.tensor.matrix('x')
subx = x[1:]
fgraph = theano.FunctionGraph([x], [subx])
self.assertRaises(ValueError, shape_of_variables, fgraph,
{x: (10, 10)})
def test_subtensor(self):
x = theano.tensor.matrix('x')
subx = x[1:]
fgraph = theano.FunctionGraph([x], [subx], clone=False)
shapes = shape_of_variables(fgraph, {x: (10, 10)})
assert shapes[subx] == (9, 10)
def write_rankfile(rankdict, filename):
file = open(filename, 'w')
for machine, rank in sorted(rankdict.items(), key=lambda (m, rank): rank):
file.write("rank %d=%s slot=0\n" % (rank, machine))
file.close()
def write_hostfile(machines, filename):
file = open(filename, 'w')
for machine in machines:
file.write("%s\n" % machine)
file.close()
def write_graph((inputs, outputs), filename):
fgraph = theano.FunctionGraph(*theano.gof.graph.clone(inputs, outputs))
file = open(filename, 'w')
pack(fgraph, file)
file.close()
def read_graph(filename):
file = open(filename, 'r')
fgraph = unpack(file)
file.close()
return fgraph
def write_sched(sched, filename):
file = open(filename, 'w')
for node in sched:
import theano
from ape.timings.communication.master import make_commtime_function, commtime_dict
from theano.tensor.utils import shape_of_variables
x = theano.tensor.matrix('x')
y = theano.tensor.matrix('y')
dotxx = theano.tensor.dot(x, x)
z = dotxx + y[:, 0].sum() - x * y
env = theano.FunctionGraph([x, y], [z])
dot = env.toposort()[2]
known_shapes = shape_of_variables(env, {x: (100, 100), y: (100, 100)})
def test_make_commtime_function():
data = {
('a', 'b'): {
'intercept': 1,
'slope': 1
},
('b', 'a'): {
'intercept': 0,
'slope': 10
}
}
commtime = make_commtime_function(data, known_shapes)
assert commtime(dotxx, 'a', 'b') == 4 * 100 * 100 * 1 + 1
assert commtime(dotxx, 'b', 'a') == 4 * 100 * 100 * 10 + 0
comms = timings.commtime_dict(network)
save_dict(rootdir + 'comps.dat', comps)
save_dict(rootdir + 'comms.dat', comms)
else:
comps = load_dict(rootdir + 'comps.dat')
comms = load_dict(rootdir + 'comms.dat')
known_shapes = shape_of_variables(inputs, outputs, input_shapes)
comptime = timings.make_runtime_function(comps)
commtime = timings.make_commtime_function(comms, known_shapes)
# Break up graph
graphs, scheds, rankfile, make = distribute(inputs, outputs, input_shapes,
machines, commtime, comptime)
# Write to disk
write(graphs, scheds, rankfile, rootdir, known_shapes)
# Print out fgraphs as pdfs
fgraphs = {
m: theano.FunctionGraph(*theano.gof.graph.clone(i, o))
for m, (i, o) in graphs.items()
}
for m, g in fgraphs.items():
theano.printing.pydotprint(g,
outfile="%s%s.pdf" % (rootdir, m),
format="pdf")
print "Makespan: %f" % make
print run_command(rankfile, rootdir)
def graph_iter(apply_nodes):
""" Returns iterator of atomic funciton graphs - really just apply nodes"""
for node in apply_nodes:
nn = node.clone_with_new_inputs([inp.clone() for inp in node.inputs])
yield theano.FunctionGraph(nn.inputs, nn.outputs)
def test_err(self):
x = theano.tensor.matrix("x")
subx = x[1:]
fgraph = theano.FunctionGraph([x], [subx])
with pytest.raises(ValueError):
shape_of_variables(fgraph, {x: (10, 10)})
dx = T.matrix('dx')
dz = T.Rop(z,x,dx)
f = theano.function([x], z)
fp = theano.function([x,dx], dz)
ffp = theano.function([x,dx], [z,dz])
As = [T.matrix('A%d'%i) for i in range(5)]
Bs = [mmul(A,A) for A in As]
C = sum(Bs)
g = theano.function(As, C)
someAs = [numpy.eye(3, dtype=numpy.float32) for i in range(5)]
myenv = theano.FunctionGraph(As, [C])
myoptimizer = f.maker.mode.optimizer
def show_optimization_path(env, filename='opt_path.txt', optimizer=myoptimizer):
file = open(filename, 'w')
for opt in myoptimizer:
file.write('\n\n')
file.write(str(opt)+'\n')
opt.optimize(env)
theano.printing.debugprint(env, file=file)
file.flush()
file.close()
# can now do
# someAs = someAs = [numpy.eye(3, dtype=numpy.float32) for i in range(5)]
""" Pack variables into numpy object array
Theano variables are iterable so asarray is confused when used normally
"""
result = np.empty(len(args), dtype=object)
for i, arg in enumerate(args):
result[i] = arg
return result
def lorenz_f(t, (x, y, z)):
""" Lorenz Attractor '67 """
return pack(sigma * (y - x), x * (rho - z) - y, -beta * z + x * y)
x, y, z = map(theano.tensor.scalar, 'xyz')
start_state = pack(x, y, z)
dt = theano.tensor.scalar('dt')
A, b = midpoint
xout, yout, zout = butcher_explicit(A, b, lorenz_f, dt, 0, start_state)
fgraph = theano.FunctionGraph([x, y, z, dt], [xout, yout, zout])
fgraph_simplified = theano_simplify(fgraph)
theano.printing.pydotprint(fgraph, outfile='lorenz.pdf', format='pdf')
theano.printing.pydotprint(fgraph_simplified,
outfile='lorenz-simplified.pdf',
format='pdf')
def alt_shape_of_variables(inputs, outputs, input_shapes):
# Thanks to P. Lamblin, F. Bastien for help to make this work
# mapping from initial to cloned var
equiv = theano.gof.graph.clone_get_equiv(inputs, outputs)
cloned_inputs = [equiv[inp] for inp in inputs]
cloned_outputs = [equiv[out] for out in outputs]
cloned_shapes = {equiv[k]: v for k, v in input_shapes.items()}
fgraph = theano.FunctionGraph(cloned_inputs, cloned_outputs, clone=False)
if not hasattr(fgraph, 'shape_feature'):
fgraph.attach_feature(tensor.opt.ShapeFeature())
kept_input = [
n for n, f in enumerate(fgraph.inputs)
if f in fgraph.shape_feature.shape_of.keys()
]
input_dims = [
dimension for kept_idx in kept_input
for dimension in fgraph.shape_feature.shape_of[fgraph.inputs[kept_idx]]
]
"""
# The old way
output_dims = [dimension for f, shape in
fgraph.shape_feature.shape_of.items()
for dimension in shape
if f in fgraph.outputs]
"""
output_dims = list(*infer_shape(cloned_outputs, cloned_inputs,
[input_shapes[k] for k in inputs]))
try:
compute_shapes = theano.function(input_dims,
output_dims,
mode=theano.Mode(optimizer=None),
on_unused_input="ignore")
numeric_input_dims = [
dim for kept_idx in kept_input
for dim in cloned_shapes[fgraph.inputs[kept_idx]]
]
numeric_output_dims = compute_shapes(*numeric_input_dims)
except MissingInputError:
# need to add fake datasets and masks to input args for ?? reasons
# unfortunate things might start happening if intermediate vars named
# example that activated this code path
# data -> linear -> tanh_rnn_layer
dataset_and_mask_indices = [
n for n, f in enumerate(fgraph.inputs) if f.name is not None
]
compute_shapes = theano.function(
[fgraph.inputs[i] for i in dataset_and_mask_indices] + input_dims,
output_dims,
mode=theano.Mode(optimizer=None),
on_unused_input="ignore")
numeric_input_dims = [
dim for kept_idx in kept_input
for dim in cloned_shapes[fgraph.inputs[kept_idx]]
]
fake_numeric_data = [
np.ones(cloned_shapes[fgraph.inputs[i]]).astype(
fgraph.inputs[i].dtype) for i in dataset_and_mask_indices
]
numeric_inputs = fake_numeric_data + numeric_input_dims
numeric_output_dims = compute_shapes(*numeric_inputs)
final_shapes = {}
# This assumes only 1 OUTPUT!!!
for var in fgraph.outputs:
final_shapes[var] = np.array(numeric_output_dims).ravel()
shapes = dict((outputs[n], tuple(np.array(final_shapes[co]).ravel()))
for n, co in enumerate(cloned_outputs))
return shapes
def shape_of_variables(i, o, input_shapes):
fgraph = theano.FunctionGraph(i, o)
known_shapes = dearrayify(
theano.tensor.utils.shape_of_variables(fgraph, input_shapes))
fgraph.disown()
return known_shapes
