def test_shape_of_variables_simple(): x = theano.tensor.matrix('x') y = x + x fgraph = theano.FunctionGraph([x], [y]) assert shape_of_variables(fgraph, {x: (5, 5)}) == {x: (5, 5), y: (5, 5)} x = theano.tensor.matrix('x') y = theano.tensor.dot(x, x.T) fgraph = theano.FunctionGraph([x], [y]) shapes = shape_of_variables(fgraph, {x: (5, 1)}) assert shapes[x] == (5, 1) assert shapes[y] == (5, 5)
def test_simple(self): x = theano.tensor.matrix("x") y = x + x fgraph = theano.FunctionGraph([x], [y], clone=False) shapes = shape_of_variables(fgraph, {x: (5, 5)}) assert shapes == {x: (5, 5), y: (5, 5)} x = theano.tensor.matrix("x") y = theano.tensor.dot(x, x.T) fgraph = theano.FunctionGraph([x], [y], clone=False) shapes = shape_of_variables(fgraph, {x: (5, 1)}) assert shapes[x] == (5, 1) assert shapes[y] == (5, 5)
def cpu_to_gpu_graph(inputs, outputs): """ Converts a cpu-only subgraph into a gpu-only subgraph >>> x, y = theano.tensor.matrix('x'), theano.tensor.matrix('y') >>> z = theano.tensor.dot(x, y) >>> gpu_inputs, gpu_outputs = cpu_to_gpu_graph((x,y), (z,)) >>> f = theano.function(gpu_inputs, gpu_outputs) >>> theano.printing.debugprint(f) GpuDot22 [@A] '' 0 |gpu_x [@B] |gpu_y [@C] """ math_opt = theano.compile.optdb.query('-inplace', '+fast_run', '-gpu') gpu_opt = cuda.opt.gpu_optimizer.query('+gpu', '-inplace', '-async') gpu_comm = cuda.opt.gpu_cut_copies.query('+gpu') gpu_inputs, cpu_inputs = zip(*map(cpu_to_gpu_var, inputs)) outputs2 = theano.clone(outputs, replace=dict(zip(inputs, cpu_inputs))) gpu_outputs = map(theano.sandbox.cuda.basic_ops.gpu_from_host, outputs2) fgraph = theano.FunctionGraph(gpu_inputs, gpu_outputs) math_opt.optimize(fgraph) gpu_opt.optimize(fgraph) gpu_comm.optimize(fgraph) fgraph.disown() for go, co in zip(gpu_outputs, outputs): go.name = gpu_name(co.name) return tuple(gpu_inputs), tuple(gpu_outputs)
def test_give_variables_names_small(): x = theano.tensor.matrix('x') y = theano.tensor.dot(x, x) fgraph = theano.FunctionGraph((x, ), (y, )) give_variables_names(fgraph.variables) assert all(var.name for var in fgraph.variables) assert unique([var.name for var in fgraph.variables])
def test_shape_of_variables(): x = theano.tensor.matrix('x') y = x[1:, 2:] known_shapes = shape_of_variables((x, ), (y, ), {x: (10, 10)}) assert known_shapes[y] == (9, 8) assert not isinstance(known_shapes[y][0], np.ndarray) # just make sure that we can do this afterwards fgraph = theano.FunctionGraph((x, ), (y, ))
def test_read_write_graph(): x = theano.tensor.matrix('x', dtype='float32') y = theano.tensor.matrix('y', dtype='float32') z = x + y fname = testdir + "test_read_write_fgraph" write_graph(((x, y), (z, )), fname) fgraph = theano.FunctionGraph((x, y), (z, )) fgraph2 = read_graph(fname) assert str(fgraph) == str(fgraph2) assert isinstance(fgraph2, theano.FunctionGraph)
def test_make_ilp(): x = theano.tensor.matrix('x') y = theano.tensor.matrix('y') z = theano.tensor.dot(x, x) + y[:, 0].sum() - x * y env = theano.FunctionGraph([x, y], [z]) machine_ids = ["ankaa", "arroyitos"] prob, X, S, Cmax = make_ilp(env, machine_ids, dummy_compute_cost, dummy_comm_cost, dummy_ability, 100) prob.solve() assert Cmax.value() == 5
def test_compute_schedule(): x = theano.tensor.matrix('x') y = theano.tensor.matrix('y') z = theano.tensor.dot(x, x) + y[:, 0].sum() - x * y env = theano.FunctionGraph([x, y], [z]) machine_ids = ["ankaa", "arroyitos"] sched = compute_schedule(*make_ilp(env, machine_ids, dummy_compute_cost, dummy_comm_cost, dummy_ability, 100)) # nodes are the jobs assert env.apply_nodes == set([job for job, (time, id) in sched]) times = [time for job, (time, id) in sched] # jobs are sorted by time assert list(sorted(times)) == times # the machine ids match what we put in assert all(id in machine_ids for job, (time, id) in sched)
def unpack(source): """ Unpickle a packed string into a FunctionGraph source can be a string file if the source is a file then one can call pack and unpack many times on the same file See Also: pack """ if isinstance(source, str): ins, outs = cPickle.loads(source) elif isinstance(source, file): ins, outs = cPickle.load(source) env = theano.FunctionGraph(ins, outs) return env
def _env(self): inputs = [inp.clone() for inp in self.apply.inputs] output = self.apply.op(inputs) env = theano.FunctionGraph(inputs, [output]) return env
def test_err(self): x = theano.tensor.matrix('x') subx = x[1:] fgraph = theano.FunctionGraph([x], [subx]) self.assertRaises(ValueError, shape_of_variables, fgraph, {x: (10, 10)})
def test_subtensor(self): x = theano.tensor.matrix('x') subx = x[1:] fgraph = theano.FunctionGraph([x], [subx], clone=False) shapes = shape_of_variables(fgraph, {x: (10, 10)}) assert shapes[subx] == (9, 10)
def write_rankfile(rankdict, filename): file = open(filename, 'w') for machine, rank in sorted(rankdict.items(), key=lambda (m, rank): rank): file.write("rank %d=%s slot=0\n" % (rank, machine)) file.close() def write_hostfile(machines, filename): file = open(filename, 'w') for machine in machines: file.write("%s\n" % machine) file.close() def write_graph((inputs, outputs), filename): fgraph = theano.FunctionGraph(*theano.gof.graph.clone(inputs, outputs)) file = open(filename, 'w') pack(fgraph, file) file.close() def read_graph(filename): file = open(filename, 'r') fgraph = unpack(file) file.close() return fgraph def write_sched(sched, filename): file = open(filename, 'w') for node in sched:
import theano from ape.timings.communication.master import make_commtime_function, commtime_dict from theano.tensor.utils import shape_of_variables x = theano.tensor.matrix('x') y = theano.tensor.matrix('y') dotxx = theano.tensor.dot(x, x) z = dotxx + y[:, 0].sum() - x * y env = theano.FunctionGraph([x, y], [z]) dot = env.toposort()[2] known_shapes = shape_of_variables(env, {x: (100, 100), y: (100, 100)}) def test_make_commtime_function(): data = { ('a', 'b'): { 'intercept': 1, 'slope': 1 }, ('b', 'a'): { 'intercept': 0, 'slope': 10 } } commtime = make_commtime_function(data, known_shapes) assert commtime(dotxx, 'a', 'b') == 4 * 100 * 100 * 1 + 1 assert commtime(dotxx, 'b', 'a') == 4 * 100 * 100 * 10 + 0
comms = timings.commtime_dict(network) save_dict(rootdir + 'comps.dat', comps) save_dict(rootdir + 'comms.dat', comms) else: comps = load_dict(rootdir + 'comps.dat') comms = load_dict(rootdir + 'comms.dat') known_shapes = shape_of_variables(inputs, outputs, input_shapes) comptime = timings.make_runtime_function(comps) commtime = timings.make_commtime_function(comms, known_shapes) # Break up graph graphs, scheds, rankfile, make = distribute(inputs, outputs, input_shapes, machines, commtime, comptime) # Write to disk write(graphs, scheds, rankfile, rootdir, known_shapes) # Print out fgraphs as pdfs fgraphs = { m: theano.FunctionGraph(*theano.gof.graph.clone(i, o)) for m, (i, o) in graphs.items() } for m, g in fgraphs.items(): theano.printing.pydotprint(g, outfile="%s%s.pdf" % (rootdir, m), format="pdf") print "Makespan: %f" % make print run_command(rankfile, rootdir)
def graph_iter(apply_nodes): """ Returns iterator of atomic funciton graphs - really just apply nodes""" for node in apply_nodes: nn = node.clone_with_new_inputs([inp.clone() for inp in node.inputs]) yield theano.FunctionGraph(nn.inputs, nn.outputs)
def test_err(self): x = theano.tensor.matrix("x") subx = x[1:] fgraph = theano.FunctionGraph([x], [subx]) with pytest.raises(ValueError): shape_of_variables(fgraph, {x: (10, 10)})
dx = T.matrix('dx') dz = T.Rop(z,x,dx) f = theano.function([x], z) fp = theano.function([x,dx], dz) ffp = theano.function([x,dx], [z,dz]) As = [T.matrix('A%d'%i) for i in range(5)] Bs = [mmul(A,A) for A in As] C = sum(Bs) g = theano.function(As, C) someAs = [numpy.eye(3, dtype=numpy.float32) for i in range(5)] myenv = theano.FunctionGraph(As, [C]) myoptimizer = f.maker.mode.optimizer def show_optimization_path(env, filename='opt_path.txt', optimizer=myoptimizer): file = open(filename, 'w') for opt in myoptimizer: file.write('\n\n') file.write(str(opt)+'\n') opt.optimize(env) theano.printing.debugprint(env, file=file) file.flush() file.close() # can now do # someAs = someAs = [numpy.eye(3, dtype=numpy.float32) for i in range(5)]
""" Pack variables into numpy object array Theano variables are iterable so asarray is confused when used normally """ result = np.empty(len(args), dtype=object) for i, arg in enumerate(args): result[i] = arg return result def lorenz_f(t, (x, y, z)): """ Lorenz Attractor '67 """ return pack(sigma * (y - x), x * (rho - z) - y, -beta * z + x * y) x, y, z = map(theano.tensor.scalar, 'xyz') start_state = pack(x, y, z) dt = theano.tensor.scalar('dt') A, b = midpoint xout, yout, zout = butcher_explicit(A, b, lorenz_f, dt, 0, start_state) fgraph = theano.FunctionGraph([x, y, z, dt], [xout, yout, zout]) fgraph_simplified = theano_simplify(fgraph) theano.printing.pydotprint(fgraph, outfile='lorenz.pdf', format='pdf') theano.printing.pydotprint(fgraph_simplified, outfile='lorenz-simplified.pdf', format='pdf')
def alt_shape_of_variables(inputs, outputs, input_shapes): # Thanks to P. Lamblin, F. Bastien for help to make this work # mapping from initial to cloned var equiv = theano.gof.graph.clone_get_equiv(inputs, outputs) cloned_inputs = [equiv[inp] for inp in inputs] cloned_outputs = [equiv[out] for out in outputs] cloned_shapes = {equiv[k]: v for k, v in input_shapes.items()} fgraph = theano.FunctionGraph(cloned_inputs, cloned_outputs, clone=False) if not hasattr(fgraph, 'shape_feature'): fgraph.attach_feature(tensor.opt.ShapeFeature()) kept_input = [ n for n, f in enumerate(fgraph.inputs) if f in fgraph.shape_feature.shape_of.keys() ] input_dims = [ dimension for kept_idx in kept_input for dimension in fgraph.shape_feature.shape_of[fgraph.inputs[kept_idx]] ] """ # The old way output_dims = [dimension for f, shape in fgraph.shape_feature.shape_of.items() for dimension in shape if f in fgraph.outputs] """ output_dims = list(*infer_shape(cloned_outputs, cloned_inputs, [input_shapes[k] for k in inputs])) try: compute_shapes = theano.function(input_dims, output_dims, mode=theano.Mode(optimizer=None), on_unused_input="ignore") numeric_input_dims = [ dim for kept_idx in kept_input for dim in cloned_shapes[fgraph.inputs[kept_idx]] ] numeric_output_dims = compute_shapes(*numeric_input_dims) except MissingInputError: # need to add fake datasets and masks to input args for ?? reasons # unfortunate things might start happening if intermediate vars named # example that activated this code path # data -> linear -> tanh_rnn_layer dataset_and_mask_indices = [ n for n, f in enumerate(fgraph.inputs) if f.name is not None ] compute_shapes = theano.function( [fgraph.inputs[i] for i in dataset_and_mask_indices] + input_dims, output_dims, mode=theano.Mode(optimizer=None), on_unused_input="ignore") numeric_input_dims = [ dim for kept_idx in kept_input for dim in cloned_shapes[fgraph.inputs[kept_idx]] ] fake_numeric_data = [ np.ones(cloned_shapes[fgraph.inputs[i]]).astype( fgraph.inputs[i].dtype) for i in dataset_and_mask_indices ] numeric_inputs = fake_numeric_data + numeric_input_dims numeric_output_dims = compute_shapes(*numeric_inputs) final_shapes = {} # This assumes only 1 OUTPUT!!! for var in fgraph.outputs: final_shapes[var] = np.array(numeric_output_dims).ravel() shapes = dict((outputs[n], tuple(np.array(final_shapes[co]).ravel())) for n, co in enumerate(cloned_outputs)) return shapes
def shape_of_variables(i, o, input_shapes): fgraph = theano.FunctionGraph(i, o) known_shapes = dearrayify( theano.tensor.utils.shape_of_variables(fgraph, input_shapes)) fgraph.disown() return known_shapes