def g_tf(args, reuse=False): eps = 1e-3 """Tensorflow map from image to pi parameters""" shapes = [info[port]['shape'] for port in inv_arrow.param_ports()] inp = args[0] width, height = getn(options, 'width', 'height') inp = tf.reshape(inp, (-1, width, height)) inp = tf.expand_dims(inp, axis=3) from tflearn.layers import conv_2d, fully_connected tf.summary.image("g_tf_output", inp) # Do convolutional layers nlayers = 2 for i in range(nlayers): inp = conv_2d(inp, nb_filter=4, filter_size=1, activation="elu") inp = conv_2d(inp, nb_filter=options['nsteps'], filter_size=1, activation="sigmoid") out = [] for i, shape in enumerate(shapes): this_inp = inp[:, :, :, i:i + 1] if shape[1] == width * height: this_inp = tf.reshape(this_inp, (options['batch_size'], -1)) + eps out.append(this_inp) else: r_length = int(np.ceil(np.sqrt(shape[1]))) rs = tf.image.resize_images(this_inp, (r_length, r_length)) rs = tf.reshape(rs, (options['batch_size'], -1)) + eps out.append(rs[:, 0:shape[1]]) return out
def fwd_arrow(batch_size): model = mixing_model(3, batch_size=batch_size, source_len=sound_len) sources, source_pos_dist = getn(model, 'sources', 'source_pos_dist') inputs = sources + source_pos_dist outputs = [model['mix_signal']] arrow = graph_to_arrow(outputs, input_tensors=inputs, name="source_separation") return arrow
def gen_rand_data(batch_size, model_tensorflow, options): """Generate data for training""" graph = tf.Graph() n_links, n_angles, n_lengths = getn(options, 'n_links', 'n_angles', 'n_lengths') final_out_data = [] final_in_data = [] data_size = options["data_size"] # assert data_size % batch_size == 0 or batch_size, "Dataset size must be multipel of batch_size" nruns = data_size // batch_size # FIXME This hack if nruns == 0: nruns = 1 with graph.as_default(): sess = tf.Session() for i in range(nruns): inputs, outputs = getn(model_tensorflow(batch_size, n_links), 'inputs', 'outputs') input_data = rand_input(batch_size, n_angles, n_lengths) output_data = sess.run(outputs, feed_dict=dict(zip(inputs, input_data))) final_out_data.append(output_data) final_in_data.append(input_data) sess.close() noutputs = len(final_out_data[0]) all_all_out_data = [] for j in range(noutputs): all_data = [final_out_data[i][j] for i in range(nruns)] res = np.concatenate(all_data) all_all_out_data.append(res) ninputs = len(final_in_data[0]) all_all_in_data = [] for j in range(ninputs): all_data = [final_in_data[i][j] for i in range(nruns)] res = np.concatenate(all_data) all_all_in_data.append(res) return {'inputs': all_all_in_data, 'outputs': all_all_out_data}
def pi_supervised(options): """Neural network enhanced Parametric inverse! to do supervised learning""" tens = render_gen_graph(options) voxels, gdotl_cube, out_img = getn(tens, 'voxels', 'gdotl_cube', 'out_img') # Do the inversion data_right_inv = tensor_to_sup_right_inv([out_img], options) # data_right_inv = right_inv_nnet([out_img], options) callbacks = [] tf.reset_default_graph() grabs = ({ 'input': lambda p: is_in_port(p) and not is_param_port(p) and not has_port_label(p, 'train_output'), 'supervised_error': lambda p: has_port_label(p, 'supervised_error'), 'sub_arrow_error': lambda p: has_port_label(p, 'sub_arrow_error'), 'inv_fwd_error': lambda p: has_port_label(p, 'inv_fwd_error') }) # Not all arrows will have these ports optional = ['sub_arrow_error', 'inv_fwd_error'] tensors = extract_tensors(data_right_inv, grabs=grabs, optional=optional) train_voxel_data, test_voxel_data = train_test_model_net_40() batch_size = options['batch_size'] train_generators = infinite_batches(train_voxel_data, batch_size=batch_size) test_generators = infinite_batches(test_voxel_data, batch_size=batch_size) def gen_gen(gen): while True: data = next(gen) data = np.reshape(data, (batch_size, -1)) yield {tensors['input'][0]: data} sess = tf.Session() num_params = get_tf_num_params(data_right_inv) # to_min = ['sub_arrow_error', 'extra', 'supervised_error', 'input', 'inv_fwd_error to_min = ['supervised_error'] losses = {a_min: accum(tensors[a_min]) for a_min in to_min} fetch = {'losses': losses} train_supervised(sess, losses, [gen_gen(train_generators)], [gen_gen(test_generators)], callbacks, fetch, options) print("Number of params", num_params)
def example(model): sess = tf.InteractiveSession() sources, positions = getn(model, 'sources', 'positions') inputs1 = [[0.5, 0.5, 0.5]] inputs2 = [[1.0, 0.7, 0.3]] inputs3 = [[0.4, -1.0, 2.0]] inputs = [inputs1, inputs2, inputs3] source1 = np.expand_dims(sound_batch[0],axis=0) source2 = np.expand_dims(sound_batch[1],axis=0) source3 = np.expand_dims(sound_batch[2],axis=0) data_sources = [source1, source2, source3] pos_feed = {positions[i]: inputs[i] for i in range(len(positions))} source_feed = {sources[i]: data_sources[i] for i in range(len(sources))} feed_dict = {} feed_dict.update(pos_feed) feed_dict.update(source_feed) re = sess.run(pull(model, 'mix_signal', 'sources', 'weighted_signal'), feed_dict=feed_dict) sess.close() return re
def gen_sound_data(batch_size, model_tensorflow, options): """Generate data for training""" graph = tf.Graph() # n = fold1_dataset.shape[0] ns = batch_size fold1_a = fold1_dataset[0:ns] fold1_b = fold1_dataset[ns:ns+ns] fold1_c = fold1_dataset[ns+ns:ns+ns+ns] source_pos_dist_data = [np.random.rand(batch_size, 1) for i in range(3)] input_data = [fold1_a, fold1_b, fold1_c] + source_pos_dist_data with graph.as_default(): model = mixing_model(n_sources=3, batch_size=batch_size, source_len=sound_len) sources, source_pos_dist = getn(model, 'sources', 'source_pos_dist') inputs = sources + source_pos_dist outputs = [model['mix_signal']] sess = tf.Session() output_data = sess.run(outputs, feed_dict=dict(zip(inputs, input_data))) sess.close() return {'inputs': input_data, 'outputs': output_data}
def g_tf(args, reuse=False): eps = 1e-3 """Tensorflow map from image to pi parameters""" shapes = [info[port]['shape'] for port in fwd.in_ports()] inp = args[0] width, height = getn(options, 'width', 'height') inp = tf.reshape(inp, (-1, width, height)) inp = tf.expand_dims(inp, axis=3) from tflearn.layers import conv_2d, fully_connected tf.summary.image("g_tf_output", inp) # Do convolutional layers nlayers = 2 for i in range(nlayers): inp = conv_2d(inp, nb_filter=4, filter_size=1, activation="elu") ratio = width / options['res'] inp = conv_2d(inp, nb_filter=options['res'], filter_size=1, strides=int(ratio), activation="sigmoid") return [tf.reshape(inp, (options['batch_size'], -1))]
def render_rand_voxels(voxels_data, gdotl_cube_data, options): """Render `batch_size` randomly selected voxels for voxel_grids""" batch_size = options.get('batch_size') graph = tf.Graph() with graph.as_default(): voxels, gdotl_cube, out_img = getn(render_gen_graph(options), 'voxels', 'gdotl_cube', 'out_img') rand_id = np.random.randint(len(voxels_data), size=batch_size) input_voxels = [ voxels_data[rand_id[i]].reshape(voxels[i].get_shape()) for i in range(batch_size) ] sess = tf.Session() feed_dict = {voxels: input_voxels} if options['phong']: input_gdotl_cube = [ gdotl_cube_data[rand_id[i]].reshape(gdotl_cube[i].get_shape()) for i in range(batch_size) ] feed_dict[gdotl_cube] = input_gdotl_cube out_img_data = sess.run(out_img, feed_dict=feed_dict) sess.close() return {'input_voxels': input_voxels, 'out_img_data': out_img_data}
def mixing_model_tf(batch_size, **options): model = mixing_model(n_sources=3, batch_size=batch_size, source_len=sound_len) sources, source_pos_dist = getn(model, 'sources', 'source_pos_dist') inputs = sources + source_pos_dist outputs = [model['mix_signal']] return {'inputs': inputs, 'outputs': outputs}
def gen_arrow(batch_size, model_tensorflow, options): inputs, outputs = getn(model_tensorflow(**options), 'inputs', 'outputs') name = options['model_name'] arrow = graph_to_arrow(outputs, input_tensors=inputs, name=name) return arrow
def gen_img(voxels, gdotl_cube, rotation_matrix, options): """Renders `batch_size` voxel grids Args: voxels : (batch_size, res, res, res) rotation_matrix : (m, 4) width: width in pixels of rendered image height: height in pixels of rendered image nsteps: number of points along each ray to sample voxel grid res: voxel resolution 'voxels' should be res * res * res batch_size: number of voxels to render in batch gdot_cube: dot product of gradient and light, can be computed offline used only in phond shading phong: do phong shading Returns: (n, m, width, height) - from voxel data from functions in voxel_helpers """ width, height, nsteps, res, batch_size, phong, density = getn( options, 'width', 'height', 'nsteps', 'res', 'batch_size', 'phong', 'density') if phong: if gdotl_cube is None: raise (ValueError("Must provide gdotl_cube for phong rendering")) raster_space = gen_fragcoords(width, height) rd, ro = make_ro(rotation_matrix, raster_space, width, height) a = 0 - ro # c = 0 b = 1 - ro # c = 1 nmatrices = rotation_matrix.shape[0] tn = np.reshape(a, (nmatrices, 1, 1, 3)) / rd tff = np.reshape(b, (nmatrices, 1, 1, 3)) / rd tn_true = np.minimum(tn, tff) tff_true = np.maximum(tn, tff) # do X tn_x = tn_true[:, :, :, 0] tff_x = tff_true[:, :, :, 0] tmin = 0.0 tmax = 10.0 t0 = tmin t1 = tmax t02 = np.where(tn_x > t0, tn_x, t0) t12 = np.where(tff_x < t1, tff_x, t1) # y tn_x = tn_true[:, :, :, 1] tff_x = tff_true[:, :, :, 1] t03 = np.where(tn_x > t02, tn_x, t02) t13 = np.where(tff_x < t12, tff_x, t12) # z tn_x = tn_true[:, :, :, 2] tff_x = tff_true[:, :, :, 2] t04 = np.where(tn_x > t03, tn_x, t03) t14 = np.where(tff_x < t13, tff_x, t13) # Shift a little bit to avoid numerial inaccuracies t04 = t04 * 1.001 t14 = t14 * 0.999 left_over = np.ones(( batch_size, nmatrices * width * height, )) step_size = (t14 - t04) / nsteps orig = np.reshape( ro, (nmatrices, 1, 1, 3)) + rd * np.reshape(t04, (nmatrices, width, height, 1)) xres = yres = res orig = np.reshape(orig, (nmatrices * width * height, 3)) rd = np.reshape(rd, (nmatrices * width * height, 3)) step_sz = np.reshape(step_size, (nmatrices * width * height, 1)) # step_sz = np.exp(-step_sz) step_sz_flat = step_sz.reshape(nmatrices * width * height) # For batch rendering, we treat each voxel in each voxel independently, nrays = width * height x = np.arange(batch_size) x_tiled = np.repeat(x, nrays) # voxels = tf.exp(-voxels) # voxels = tf.Print(voxels, [tf.reduce_sum(voxels)], message="VOXEL SUM TF") # 998627.56 for i in range(nsteps): # Find the position (x,y,z) of ith step pos = orig + rd * step_sz * i # convert to indices for voxel cube voxel_indices = np.floor(pos * res) pruned = np.clip(voxel_indices, 0, res - 1) p_int = pruned.astype('int64') indices = np.reshape(p_int, (nmatrices * width * height, 3)) # convert to indices in flat list of voxels flat_indices = indices[:, 0] + res * (indices[:, 1] + res * indices[:, 2]) # tile the indices to repeat for all elements of batch tiled_indices = np.tile(flat_indices, batch_size) batched_indices = np.transpose([x_tiled, tiled_indices]) batched_indices = batched_indices.reshape(batch_size, len(flat_indices), 2) attenuation = tf.gather_nd(voxels, batched_indices) if phong: grad_samples = tf.gather_nd(gdotl_cube, batched_indices) attenuation = attenuation * grad_samples # left_over = left_over * -attenuation * density * step_sz_flat left_over = left_over * tf.exp(-attenuation * density * step_sz_flat) # left_over = left_over * attenuation img = left_over return img
def reparam_train(arrow: Arrow, extra_ports: Sequence[Port], train_data: List[Generator], test_data: List[Generator], options=None) -> CompositeArrow: options = {} if options is None else options grabs = ({'input': lambda p: is_in_port(p) and not is_param_port(p) and not has_port_label(p, 'train_output'), 'train_output': lambda p: has_port_label(p, 'train_output'), 'pure_output': lambda p: is_out_port(p) and not is_error_port(p), 'supervised_error': lambda p: has_port_label(p, 'supervised_error'), 'sub_arrow_error': lambda p: has_port_label(p, 'sub_arrow_error'), 'inv_fwd_error': lambda p: has_port_label(p, 'inv_fwd_error')}) # Not all arrows will have these ports optional = ['sub_arrow_error', 'inv_fwd_error', 'param', 'supervised_error', 'train_output'] tensors = extract_tensors(arrow, grabs=grabs, optional=optional) # Make parametric inputs train_gen_gens = [] test_gen_gens = [] param_feed_gens = [] for t in tensors['param']: shape = tuple(t.get_shape().as_list()) gen = infinite_samples(np.random.rand, options['batch_size'], shape) param_feed_gens.append(attach(t, gen)) train_gen_gens += param_feed_gens test_gen_gens += param_feed_gens n = len(tensors['input']) train_gen_gens += [attach(tensors['input'][i], train_data[i]) for i in range(n)] test_gen_gens += [attach(tensors['input'][i], test_data[i]) for i in range(n)] sound_loss = accumulate_losses(tensors['error']) # Generate permutation tensors with tf.name_scope("placeholder"): perm = tf.placeholder(shape=(None,), dtype='int32', name='perm') perm_idx = tf.placeholder(shape=(None,), dtype='int32', name='perm_idx') perm_feed_gen = [perm_gen(options['batch_size'], perm, perm_idx)] train_gen_gens += perm_feed_gen test_gen_gens += perm_feed_gen extra = tensors['output'] butim3d = tf.concat(tensors['pure_output'], axis=1) extra = [butim3d] euclids = [pairwise_dists(t, perm, perm_idx) for t in extra] min_gap_losses = [minimum_gap(euclid) for euclid in euclids] min_gap_loss = tf.reduce_sum(min_gap_losses) # mean_gap_losses = [mean_gap(euclid) for euclid in euclids] # mean_gap_loss = tf.reduce_mean(mean_gap_losses) lmbda = options['lambda'] min_gap_loss = min_gap_loss * lmbda losses = [sound_loss - min_gap_loss] loss_ratios = [1] loss_updates = [gen_update_step(loss, options['learning_rate']) for loss in losses] # options['debug'] = True # All losses loss_dict = {} for loss in ['error', 'sub_arrow_error', 'inv_fwd_error', 'supervised_error']: if loss in tensors: loss_dict[loss] = accumulate_losses(tensors[loss]) # loss_dict['mean_gap'] = mean_gap_loss loss_dict['min_gap_losses'] = min_gap_losses loss_dict['min_gap'] = min_gap_loss loss_dict['sound_loss'] = sound_loss loss_dict['general_loss'] = losses[0] sess = tf.Session() fetch = gen_fetch(sess, **options) fetch['input_tensors'] = tensors['input'] fetch['param_tensors'] = tensors['param'] fetch['output_tensors'] = tensors['output'] fetch['loss'] = loss_dict if inn(options, 'save', 'dirname', 'params_file', 'load'): ops = prep_save(sess, *getn(options, 'save', 'dirname', 'params_file', 'load')) options.update(ops) train_loop(sess, loss_updates, fetch, train_gen_gens, test_gen_gens, loss_ratios=loss_ratios, **options)