map(clean_variable, fgraph.variables) fgraph2 = math_optimize(fgraph) fgraph2_var_dict = {str(var): var for var in fgraph.variables} input_shapes2 = { fgraph2_var_dict[str(var)]: input_shapes[var] for var in input_shapes } os.system('mkdir -p tmp') all_shapes = shape_of_variables(fgraph, input_shapes) # Compute Cost compute_times = comptime_dict(fgraph, input_shapes, 10, machines, machine_groups) save_dict('tmp/compute_times.dat', compute_times) # compute_times = load_dict('tmp/compute_times.dat') compute_cost = make_runtime_function(compute_times) # Communication Cost comm_dict = commtime_dict(network, [10, 100, 1000, 2000] * 5) save_dict('tmp/comm_times.dat', comm_dict) # comm_dict = load_dict('tmp/comm_times.dat') comm_cost = make_commtime_function(comm_dict, all_shapes) code = compile(fgraph, machines, compute_cost, comm_cost, dummy_ability, input_shapes, 100) f = open('tmp/results.py', 'w') f.write(code)
def test_save_dict(): data = {('A', 'B'): {'gemm': 1, 'sum': 2}, ('C', ): {'gemm': 3, 'sum': 1}} save_dict('_temp.tmp', data) data2 = load_dict('_temp.tmp') assert data == data2
fgraph = FunctionGraph(inputs, outputs) theano.gof.utils.give_variables_names(fgraph.variables) map(clean_variable, fgraph.variables) fgraph2 = math_optimize(fgraph) fgraph2_var_dict = {str(var): var for var in fgraph.variables} input_shapes2 = {fgraph2_var_dict[str(var)]: input_shapes[var] for var in input_shapes} os.system("mkdir -p tmp") all_shapes = shape_of_variables(fgraph, input_shapes) # Compute Cost compute_times = comptime_dict(fgraph, input_shapes, 10, machines, machine_groups) save_dict("tmp/compute_times.dat", compute_times) # compute_times = load_dict('tmp/compute_times.dat') compute_cost = make_runtime_function(compute_times) # Communication Cost comm_dict = commtime_dict(network, [10, 100, 1000, 2000] * 5) save_dict("tmp/comm_times.dat", comm_dict) # comm_dict = load_dict('tmp/comm_times.dat') comm_cost = make_commtime_function(comm_dict, all_shapes) code = compile(fgraph, machines, compute_cost, comm_cost, dummy_ability, input_shapes, 100) f = open("tmp/results.py", "w") f.write(code)
if __name__ == '__main__': from ape.examples.kalman import inputs, outputs, input_shapes from ape.examples.triple import machines, machine_groups, network rootdir = 'tmp/' os.system('mkdir -p %s'%rootdir) # sanitize sanitize(inputs, outputs) # do timings if necessary recompute = False if recompute: comps = timings.comptime_dict(inputs, outputs, input_shapes, 5, machines, machine_groups) comms = timings.commtime_dict(network) save_dict(rootdir+'comps.dat', comps) save_dict(rootdir+'comms.dat', comms) else: comps = load_dict(rootdir+'comps.dat') comms = load_dict(rootdir+'comms.dat') known_shapes = shape_of_variables(inputs, outputs, input_shapes) comptime = timings.make_runtime_function(comps) commtime = timings.make_commtime_function(comms, known_shapes) # Break up graph graphs, scheds, rankfile, make = distribute(inputs, outputs, input_shapes, machines, commtime, comptime) # Write to disk write(graphs, scheds, rankfile, rootdir, known_shapes)
if __name__ == '__main__': from ape.examples.kalman import inputs, outputs, input_shapes from ape.examples.triple import machines, machine_groups, network rootdir = 'tmp/' os.system('mkdir -p %s' % rootdir) # sanitize sanitize(inputs, outputs) # do timings if necessary recompute = False if recompute: comps = timings.comptime_dict(inputs, outputs, input_shapes, 5, machines, machine_groups) comms = timings.commtime_dict(network) save_dict(rootdir + 'comps.dat', comps) save_dict(rootdir + 'comms.dat', comms) else: comps = load_dict(rootdir + 'comps.dat') comms = load_dict(rootdir + 'comms.dat') known_shapes = shape_of_variables(inputs, outputs, input_shapes) comptime = timings.make_runtime_function(comps) commtime = timings.make_commtime_function(comms, known_shapes) # Break up graph graphs, scheds, rankfile, make = distribute(inputs, outputs, input_shapes, machines, commtime, comptime) # Write to disk write(graphs, scheds, rankfile, rootdir, known_shapes)
def test_save_dict(): data = {('A', 'B'): {'gemm':1, 'sum':2}, ('C',) : {'gemm':3, 'sum':1}} save_dict('_temp.tmp', data) data2 = load_dict('_temp.tmp') assert data == data2