def replace_read_ops(loss_or_losses, var_list): """ Replaces read ops of each variable in `vars` with new read ops obtained from `read_value()`, thus forcing to read the most up-to-date values of the variables (which might incur copies across devices). The graph is seeded from the tensor(s) `loss_or_losses`. """ # ops between var ops and the loss ops = set(ge.get_walks_intersection_ops([var.op for var in var_list], loss_or_losses)) if not ops: # loss_or_losses doesn't depend on any var in var_list, so there is nothiing to replace return # filter out variables that are not involved in computing the loss var_list = [var for var in var_list if var.op in ops] # assume that for each variable, the only op required to compute the loss # is a read op, and there is exactly one per variable read_ops = [] for var in var_list: output, = var.op.outputs read_op, = set(output.consumers()) & ops read_ops.append(read_op) for var, read_op in zip(var_list, read_ops): with tf.name_scope('/'.join(read_op.name.split('/')[:-1])): with tf.device(read_op.device): read_t, = read_op.outputs consumer_ops = set(read_t.consumers()) & ops # consumer_sgv might have multiple inputs, but we only care # about replacing the input that is read_t consumer_sgv = ge.sgv(consumer_ops) consumer_sgv = consumer_sgv.remap_inputs([list(consumer_sgv.inputs).index(read_t)]) ge.connect(ge.sgv(var.read_value().op), consumer_sgv)
def tensor_swapin_and_out(g, origin_op, swapin_op): global added_control all_ops = g.get_operations() #find the origin_op's output tensor name origin_op_name = origin_op.values()[0].name added_control = False #search the to_swapin_op which use for op in all_ops: for i in range(len(op.inputs)): if ((op.inputs[i].name == origin_op_name) and ("_grad" in op.name)): print("gradient op.name:", op.name) """ ('op.name:', u'layer1/L1_SwapOut') ('op.name:', u'layer2/MatMul') ('op.name:', u'optimizer/gradients/layer1/Sigmoid_grad/SigmoidGrad') """ #Use connect and remap function to reconnect ge.connect(ge.sgv(swapin_op), ge.sgv(op).remap_inputs([i])) # FIXME: # obviously we cannot add more than 1 control dependency for swap_in op if added_control is False: print("Control Dependency==> swapin_op:", swapin_op, "op:", op) add_control_dependency(all_ops, swapin_op, op)
def _connect_ops(self, src_op, dest_op, remap_inputs=False, remap_outputs=False, idx=None, disconnect_first=False): """A wrapper of `tensorflow.contrib.graph_editor.connect`. This method does an in-place modification to the graph. Args: src_op: a `tf.Operation`. dest_op: a `tf.Operation`. remap_inputs: remap the input of `dest_op` or not. remap_outputs: remap the output of `src_op` or not. idx: index of input or output tensor. disconnect_first: True means the current outputs of sgv0 are disconnected. """ src_sgv = ge.sgv(src_op, graph=self._graph) dest_sgv = ge.sgv(dest_op, graph=self._graph) if remap_outputs: src_sgv = src_sgv.remap_outputs([idx]) if remap_inputs: dest_sgv = dest_sgv.remap_inputs([idx]) ge.connect(src_sgv, dest_sgv, disconnect_first)
def build_pb_fact(pb_location, main_location, breath, quant, lvl): graph = load_graph(pb_location) W1 = graph.get_tensor_by_name('prefix/w_in:0') matmul = graph.get_tensor_by_name('prefix/MatMul:0') bias = graph.get_tensor_by_name('prefix/b_in:0') add = graph.get_tensor_by_name('prefix/add:0') reshape = graph.get_tensor_by_name('prefix/Reshape:0') # #remove all conncetions from matmul ge.detach(ge.sgv(matmul.op)) with tf.Session(graph=graph) as sess: # os.system("mkdir " + main_location + breath + "/" + quant + "/fact_" + str(lvl)) # for op in sess.graph.get_operations(): # print(op.name) W = W1.eval() u, s, v, ss = svd_compress_gs(W, lvl) logEntry("structural_similarity == > " + str(ss)) u1 = tf.matmul(reshape, u, name="prefix/u1") s1 = tf.matmul(u1, s, name="prefix/s1") v1 = tf.matmul(s1, v, name="prefix/v1") ge.connect(ge.sgv(v1.op), ge.sgv(add.op).remap_inputs([0])) sess.run(tf.variables_initializer([tf.Variable(5, name="dummy" + str(lvl))])) saver = tf.train.Saver() # save log for tensorboad LOGDIR = main_location + '/LOG' train_writer = tf.summary.FileWriter(LOGDIR) train_writer.add_graph(sess.graph) train_writer.close() # save the freezed model os.system("mkdir " + main_location + "pb") tf.train.write_graph(sess.graph_def, main_location + 'pb/', "RNN_" + breath + "_" + quant + "_fact_" + str(lvl) + ".pbtxt") saver.save(sess, save_path=main_location + "model.ckpt") input_graph_path = main_location + '/pb/' + "RNN_" + breath + "_" + quant + "_fact_" + str(lvl) + ".pbtxt" checkpoint_path = main_location + "model.ckpt" restore_op_name = "save/restore_all" filename_tensor_name = "save/Const:0" output_frozen_graph_name = main_location + 'pb/' + "RNN_" + breath + "_" + quant + "_fact_" + str(lvl) + ".pb" logEntry("Start Freezing the graph") freeze_graph.freeze_graph(input_graph_path, input_saver="", input_binary=False, input_checkpoint=checkpoint_path, output_node_names="prefix/y_", restore_op_name="save/restore_all", filename_tensor_name="save/Const:0", output_graph=output_frozen_graph_name, clear_devices=True, initializer_nodes="") logEntry("End Freezing the graph") sess.close()
def _connect_ops(src_op, dest_op, remap_inputs=False, remap_outputs=True): src_sgv = ge.sgv(src_op, graph=tf.get_default_graph()) dest_sgv = ge.sgv(dest_op, graph=tf.get_default_graph()) if remap_outputs: src_sgv = src_sgv.remap_outputs([0]) if remap_inputs: dest_sgv = dest_sgv.remap_inputs([0]) ge.connect(src_sgv, dest_sgv)
def test_connect(self): """Test for ge.connect.""" with self.graph.as_default(): x = tf.constant([1., 1.], shape=[2], name="x") y = tf.constant([2., 2.], shape=[2], name="y") z = tf.add(x, y, name="z") sgv = ge.sgv(x.op, y.op, z.op) ge.connect(sgv, ge.sgv(self.e.op).remap_inputs([0])) self.assertTrue(ge.matcher("^foo/bar/e$").input_ops("^z$", "foo/d$") (self.e.op))
def remove_dropout_node(graph, output_op_name, input_op_name, dst_node_name_list): output_op = graph.get_operation_by_name(output_op_name) input_op = graph.get_operation_by_name(input_op_name) ge.connect(output_op, input_op) graph_def = graph.as_graph_def() # display_node(graph_def); new_graph_def = tf.graph_util.extract_sub_graph(graph_def, dst_node_name_list) return new_graph_def
def build_pb_fact(pb_location, main_location, breath, quant, lvl): graph = load_graph(pb_location) with tf.Session(graph=graph) as sess: count = 0 pre_layer = [ 'Placeholder', 'hidden_layer1', 'hidden_layer2', 'hidden_layer3', 'hidden_layer4', 'hidden_layer5', 'hidden_layer6' ] svd_num = [2048, 1024, 512, 256, 128, 64, 32] for layer in [ 'hidden_layer1', 'hidden_layer2', 'hidden_layer3', 'hidden_layer4', 'hidden_layer5', 'hidden_layer6', 'hidden_layer7' ]: if (count == 0): layer_input = graph.get_tensor_by_name('prefix/' + pre_layer[count] + ':0') else: layer_input = graph.get_tensor_by_name('prefix/' + pre_layer[count] + '/Tanh:0') W = graph.get_tensor_by_name('prefix/' + layer + '/kernel:0') matmul = graph.get_tensor_by_name('prefix/' + layer + '/MatMul:0') add = graph.get_tensor_by_name('prefix/' + layer + '/BiasAdd:0') ge.detach(ge.sgv(matmul.op)) W1 = W.eval() print(W1.shape) u_1, s_1, v_1, ss = svd_compress_gs( W1, math.ceil(svd_num[count] * lvl / 100)) logEntry( str(lvl) + " ==> layer ==> " + str(count) + " ssim ==> " + str(ss)) u1 = tf.matmul(layer_input, u_1, name="prefix/" + layer + "/u1") s1 = tf.matmul(u1, s_1, name="prefix/" + layer + "/s1") v1 = tf.matmul(s1, v_1, name="prefix/" + layer + "/v1") ge.connect(ge.sgv(v1.op), ge.sgv(add.op).remap_inputs([0])) count += 1 sess.run( tf.variables_initializer([tf.Variable(5, name="dummy" + str(lvl))])) saver = tf.train.Saver() # save log for tensorboad LOGDIR = main_location + '/LOG' train_writer = tf.summary.FileWriter(LOGDIR) train_writer.add_graph(sess.graph) train_writer.close() # save the freezed model os.system("mkdir " + main_location + "pb") tf.train.write_graph( sess.graph_def, main_location + 'pb/', "DNN_" + breath + "_" + quant + "_fact_" + str(lvl) + ".pbtxt") saver.save(sess, save_path=main_location + "model.ckpt") input_graph_path = main_location + '/pb/' + "DNN_" + breath + "_" + quant + "_fact_" + str( lvl) + ".pbtxt" checkpoint_path = main_location + "model.ckpt" restore_op_name = "save/restore_all" filename_tensor_name = "save/Const:0" output_frozen_graph_name = main_location + 'pb/' + "DNN_" + breath + "_" + quant + "_fact_" + str( lvl) + ".pb" logEntry("Start Freezing the graph") freeze_graph.freeze_graph(input_graph_path, input_saver="", input_binary=False, input_checkpoint=checkpoint_path, output_node_names="prefix/softmax", restore_op_name="save/restore_all", filename_tensor_name="save/Const:0", output_graph=output_frozen_graph_name, clear_devices=True, initializer_nodes="") logEntry("End Freezing the graph") sess.close()
def build_pb(file, lvl, svd_num): tf.reset_default_graph() graph = load_graph(file) logEntry(svd_num) with tf.Session(graph=graph) as sess: count = 0 pre_layer = [ 'Placeholder', 'hidden_layer1', 'hidden_layer2', 'hidden_layer3', 'hidden_layer4', 'hidden_layer5', 'hidden_layer6' ] # svd_num = [40, 8, 8, 8, 8, 8, 8] for layer in [ 'hidden_layer1', 'hidden_layer2', 'hidden_layer3', 'hidden_layer4', 'hidden_layer5', 'hidden_layer6', 'hidden_layer7' ]: if (count == 0): layer_input = graph.get_tensor_by_name('prefix/' + pre_layer[count] + ':0') else: layer_input = graph.get_tensor_by_name('prefix/' + pre_layer[count] + '/Tanh:0') W = graph.get_tensor_by_name('prefix/' + layer + '/kernel:0') matmul = graph.get_tensor_by_name('prefix/' + layer + '/MatMul:0') add = graph.get_tensor_by_name('prefix/' + layer + '/BiasAdd:0') W1 = W.eval() print(W1.shape) u_1, s_1, v_1, ss = svd_compress_gs(W1, svd_num[count]) logEntry( str(lvl) + " ==> layer ==> " + str(count) + " ssim ==> " + str(ss)) u1 = tf.matmul(layer_input, u_1, name="prefix/" + layer + "/u1") s1 = tf.matmul(u1, s_1, name="prefix/" + layer + "/s1") v1 = tf.matmul(s1, v_1, name="prefix/" + layer + "/v1") ge.connect(ge.sgv(v1.op), ge.sgv(add.op).remap_inputs([0])) count += 1 a = tf.Variable(5, name="dummy") sess.run(tf.variables_initializer([a])) saver = tf.train.Saver() os.system("mkdir /flush1/raj034/DNN/model/test_cases/" + breath + "/" + quant + "/fact_" + str(lvl)) os.system("mkdir /flush1/raj034/DNN/model/test_cases/" + breath + "/" + quant + "/fact_" + str(lvl) + "/pb") LOGDIR = "/flush1/raj034/DNN/model/test_cases/" + breath + "/" + quant + "/fact_" + str( lvl) + "/LOG" train_writer = tf.summary.FileWriter(LOGDIR) train_writer.add_graph(sess.graph) train_writer.close() tf.train.write_graph( sess.graph_def, "/flush1/raj034/DNN/model/test_cases/" + breath + "/" + quant + '/fact_' + str(lvl) + '/pb/', "DNN_" + breath + "_" + quant + "_fact_" + str(lvl) + ".pbtxt") saver.save(sess, save_path="/flush1/raj034/DNN/model/test_cases/" + breath + "/" + quant + '/fact_' + str(lvl) + "/model.ckpt") input_graph_path = "/flush1/raj034/DNN/model/test_cases/" + breath + "/" + quant + '/fact_' + \ str(lvl) + '/pb/' + "DNN_" + breath + "_" + quant + "_fact_" + str(lvl) + ".pbtxt" checkpoint_path = "/flush1/raj034/DNN/model/test_cases/" + breath + "/" + quant + '/fact_' + str( lvl) + "/model.ckpt" restore_op_name = "save/restore_all" filename_tensor_name = "save/Const:0" output_frozen_graph_name = "/flush1/raj034/DNN/model/test_cases/" + breath + "/" + quant + '/fact_' + \ str(lvl) + '/pb/' + "DNN_" + breath + "_" + quant + "_fact_" + str(lvl) + ".pb" logEntry("Start Freezing the graph") freeze_graph.freeze_graph(input_graph_path, input_saver="", input_binary=False, input_checkpoint=checkpoint_path, output_node_names="prefix/softmax", restore_op_name="save/restore_all", filename_tensor_name="save/Const:0", output_graph=output_frozen_graph_name, clear_devices=True, initializer_nodes="") logEntry("End Freezing the graph") sess.close()
# print(op.control_inputs, ge.get_ops_ios(op, False)) for i, relu_op in enumerate(relu_ops): relu_node = g.get_tensor_by_name(relu_op.name + ":0") new_name = '/'.join(relu_op.name.split('/')[:-1]) + "/wrapper_to_relu_" + str(i) wrapper = tf.identity(relu_node, name=new_name) # ge.detach_inputs(wrapper) # ge.swap_outputs(relu_node, wrapper) # ge.connect(relu_node, wrapper) print("here", relu_node, wrapper) post_relu_ops = ge.get_consuming_ops([relu_node]) print(post_relu_ops, ge.get_consuming_ops([wrapper])) wrapper_op = tf.get_default_graph().get_operation_by_name("tower0/linear3/output") print(wrapper_op, post_relu_ops[0]) ge.connect(wrapper_op, post_relu_ops[0]) print(post_relu_ops, ge.get_consuming_ops([wrapper])) print("after") for op in g.get_operations(): if 'wrapper' in op.name: print(op.name, op.type, op.values()) # node = g.get_tensor_by_name(op.name + ":0") # print(ge.sgv(node)) if 'Relu' in op.type and 'tower0' in op.name and 'gradients' not in op.name: print(op.name, op.type, op.values()) # node = g.get_tensor_by_name(op.name + ":0") # print(ge.sgv(node)) # saver = tf.train.Saver() a = tf.train.export_meta_graph('wrapperrelutest/model.meta', as_text=True)
def main(): with tf.name_scope('inputs'): x_update_ = tf.placeholder(dtype=tf.float32, shape=(None, None, 1)) x_meta_ = tf.placeholder(dtype=tf.float32, shape=(None, None, 1)) y_update_ = tf.placeholder(dtype=tf.float32, shape=(None, None, 1)) y_meta_ = tf.placeholder(dtype=tf.float32, shape=(None, None, 1)) updated_parameters = list() update_losses = list() # ______________________________________________________________________________________________ # trainable update learning rate lr = tf.Variable(1., dtype=tf.float32) lr = tf.nn.sigmoid(lr) * 1e-2 non_model_trainable_variables = tf.trainable_variables() model_variables = [] x_update_tasks = tf.unstack(x_update_, num=FLAGS.meta_batch_size) y_update_tasks = tf.unstack(y_update_, num=FLAGS.meta_batch_size) for j in range(FLAGS.meta_batch_size): updated_parameters.append([]) update_losses.append([]) for i in range(FLAGS.n_update_steps): with tf.name_scope('update_{}_task_{}'.format(i, j)): with tf.variable_scope('model', reuse=(i != 0 or j != 0)): y_update = model(x_update_tasks[j]) if i == 0 and j == 0: # __________________________________________________________________________________ # determine the models variables model_variables = \ list(item for item in tf.trainable_variables() if item not in non_model_trainable_variables) if i == 0: updated_parameters[j].append(model_variables) update_loss = tf.reduce_mean((y_update - y_update_tasks[j])**2) # __________________________________________________________________________________ # ATTENTION # need to differentiate wrt the original parameters because in the graph the updated steps # are until now only connected to the original parameters grads = tf.gradients(update_loss, model_variables) updated_parameters[j].append( list(v - lr * g for v, g in zip(updated_parameters[j][-1], grads))) update_losses[j].append(update_loss) logger.info('Graph built') y_task_meta = [] for j in range(FLAGS.meta_batch_size): with tf.name_scope('meta_task_{}'.format(j)): with tf.variable_scope('model', reuse=True): y_meta = model(x_meta_) y_task_meta.append(y_meta) with tf.name_scope('meta_loss'): meta_loss = tf.reduce_mean((tf.stack(y_task_meta) - y_meta_)**2) # ______________________________________________________________________________________________ # connect updated parameters to each update step for i in range(1, FLAGS.n_update_steps): for j in range(FLAGS.meta_batch_size): pv = ge.sgv(*updated_parameters[j][i]) s = ge.sgv_scope('update_{}_task_{}'.format(i, j), tf.get_default_graph()) ind = list( s.input_index(tf.convert_to_tensor(a)) for a in model_variables) s = s.remap_inputs(ind) ge.connect(pv, s) # ______________________________________________________________________________________________ # connect final updated parameters for j in range(FLAGS.meta_batch_size): pv = ge.sgv(*updated_parameters[j][-1]) s = ge.sgv_scope('meta_task_{}'.format(j), tf.get_default_graph()) ind = list( s.input_index(tf.convert_to_tensor(a)) for a in model_variables) s = s.remap_inputs(ind) ge.connect(pv, s) logger.info('Graph connected') with tf.name_scope('optimize'): optimizer = tf.train.AdamOptimizer(FLAGS.meta_lr) update_step = optimizer.minimize(meta_loss) def gen_data(phase): x_batch_update = np.random.uniform(-1, 1, size=( FLAGS.meta_batch_size, n_batch, )) x_batch_meta = np.random.uniform(-1, 1, size=( FLAGS.meta_batch_size, n_batch, )) y_batch_update = np.sin(x_batch_update + phase) y_batch_meta = np.sin(x_batch_meta + phase) return x_batch_update, y_batch_update, x_batch_meta, y_batch_meta board_writer = tf.summary.FileWriter('tensorboard_log', tf.get_default_graph()) n_batch = 10 print_every = 10 with tf.Session() as session: session.run(tf.global_variables_initializer()) for j in range(1000): phase = np.random.uniform(0, np.pi, size=FLAGS.meta_batch_size)[:, None] x_batch_update, y_batch_update, x_batch_meta, y_batch_meta = gen_data( phase) feed_dict = { x_update_: x_batch_update[:, :, None], x_meta_: x_batch_meta[:, :, None], y_update_: y_batch_update[:, :, None], y_meta_: y_batch_meta[:, :, None] } session.run(update_step, feed_dict=feed_dict) if j % print_every == 0: phase = np.random.uniform(0, np.pi, size=FLAGS.meta_batch_size)[:, None] x_batch_update, y_batch_update, x_batch_meta, y_batch_meta = gen_data( phase) feed_dict = { x_update_: x_batch_update[:, :, None], x_meta_: x_batch_meta[:, :, None], y_update_: y_batch_update[:, :, None], y_meta_: y_batch_meta[:, :, None] } first_update_losses = list(a[0] for a in update_losses) u, m = session.run([first_update_losses, meta_loss], feed_dict=feed_dict) print('{:.4g} | {:.4g}'.format(np.mean(u), np.mean(m)))
def __call__(self, model, update_loss_function, x_update_=None, y_update_=None, x_meta_=None): updated_parameters = list() update_losses = list() non_model_trainable_variables = tf.trainable_variables() model_variables = [] if x_meta_ is None: x_meta_ = tf.placeholder(dtype=tf.float32, shape=(None, x_update_.get_shape[1])) self.x_update_ = x_update_ self.y_update_ = y_update_ self.x_meta_ = x_meta_ with tf.variable_scope(self.name): for i in range(self.n_update_steps): with tf.name_scope('update_{}'.format(i)): with tf.variable_scope('model', reuse=(i != 0)): y_update = model(x_update_) if i == 0: # __________________________________________________________________________ # determine the models variables model_variables = list(item for item in tf.trainable_variables() if item not in non_model_trainable_variables) updated_parameters.append(model_variables) update_loss = update_loss_function(y_update, y_update_) update_losses.append(update_loss) # ______________________________________________________________________________ # ATTENTION: # need to differentiate wrt the original parameters, since the updated model # steps are still connected to the original parameters. grads = tf.gradients(update_loss, model_variables) if self.stop_gradients: grads = list(tf.stop_gradient(grad) for grad in grads) updated_parameters.append( list(v - self.update_lr * g for v, g in zip(updated_parameters[-1], grads))) with tf.name_scope('meta'): with tf.variable_scope('model', reuse=True): y_meta = model(x_meta_) # ______________________________________________________________________________________ # connect updated parameters to each update step for i in range(1, self.n_update_steps): pv = ge.sgv(*updated_parameters[i]) s = ge.sgv_scope('{}/update_{}'.format(self.name, i), tf.get_default_graph()) ind = list(s.input_index(tf.convert_to_tensor(a)) for a in model_variables) s = s.remap_inputs(ind) ge.connect(pv, s) # _____________________________________________________________________________________ # connect final updated parameters pv = ge.sgv(*updated_parameters[-1]) s = ge.sgv_scope('{}/meta'.format(self.name), tf.get_default_graph()) ind = list(s.input_index(tf.convert_to_tensor(a)) for a in model_variables) s = s.remap_inputs(ind) ge.connect(pv, s) # _________________________________________________________________________________________ # for debug and evaluation purposes self.update_losses = update_losses self.updated_parameters = updated_parameters return y_meta
def trainESL(checkpoint_path, checkpoint_file, checkpoint_epoch): global trX, trY, teX, teY if not os.path.exists(checkpoint_path): os.makedirs(checkpoint_path) ##### Modify the graph ##### sess = tf.Session() all_ops = sess.graph.get_operations() print("all_ops:", all_ops) #l1_swap_in = sess.graph.get_operation_by_name("layer1/L1_SwapIn") #print l1_swap_in.outputs[0] #matmul_1_grad = sess.graph.get_operation_by_name("optimizer/gradients/layer2/MatMul_grad/MatMul_1") #print matmul_1_grad.inputs[1] #ret = ge.connect(ge.sgv(l1_swap_in), ge.sgv(matmul_1_grad).remap_inputs([0])) ret = ge.connect( ge.sgv("layer1/L1_SwapIn", graph=tf.get_default_graph()), ge.sgv("optimizer/gradients/layer2/MatMul_grad/MatMul_1", graph=tf.get_default_graph()).remap_inputs([0])) sess.close() graph = tf.get_default_graph() writer = tf.summary.FileWriter("./simple_graph_events") writer.add_graph(graph=graph) ############################ # Launch the graph with tf.Session() as sess: merged = tf.summary.merge_all() # tf.train.SummaryWriter soon be deprecated, use following writer = tf.summary.FileWriter("logs/", sess.graph) # tf.initialize_all_variables() no long valid from # 2017-03-02 if using tensorflow >= 0.12 sess.run(tf.global_variables_initializer()) # restore all variables """ ckpt = tf.train.get_checkpoint_state(ckpt_dir) if ckpt and ckpt.model_checkpoint_path: print(ckpt.model_checkpoint_path) saver.restore(sess, ckpt.model_checkpoint_path) """ # Training cycle for epoch in range(checkpoint_epoch + 1): # Shuffle the data before each training iteration. p = np.random.permutation(range(len(trX))) trX, trY = trX[p], trY[p] # Train in batches of 4 inputs for start in range(0, n_samples, BATCH_SIZE): end = start + BATCH_SIZE sess.run(optimizer, feed_dict={ X: trX[start:end], Y: trY[start:end] }) # Record the summary of weights and biases if epoch % 50 == 0: rs = sess.run(merged, feed_dict={X: trX, Y: trY}) writer.add_summary(rs, epoch) # And print the current accuracy on the training data. if epoch % 100 == 0: saver.save(sess, os.path.join(checkpoint_path, checkpoint_file), global_step=epoch) # Loop over all batches # Display logs per epoch step print("Epoch:", '%04d' % (epoch + 1), "cost=", sess.run(cost, feed_dict={ X: trX, Y: trY })) print("Optimization Finished!") # USe testing samples to predict the result trY_pred = sess.run(predict_op, feed_dict={X: trX}) teY_pred = sess.run(predict_op, feed_dict={X: teX}) from sklearn.metrics import r2_score from sklearn.metrics import mean_squared_error print('MSE train: %.3f, test: %.3f' % (mean_squared_error( trY, trY_pred), mean_squared_error(teY, teY_pred))) print('R^2 train: %.3f, test: %.3f' % (r2_score(trY, trY_pred), r2_score(teY, teY_pred))) # Inverse_transforming for standardization teY_pred = scalerY.inverse_transform(teY_pred) teY = scalerY.inverse_transform(teY) # Plot the chart plt.figure() plt.subplot(211) plt.plot(teY[:, 0], 'bo', teY_pred[:, 0], 'k') plt.legend(['Original ESL1', 'Predicted ESL1'], loc='upper left') plt.subplot(212) plt.plot(teY[:, 1], 'ro', teY_pred[:, 1], 'k') plt.legend(['Original ESL2', 'Predicted ESL2'], loc='upper left') plt.show()
def convert(file_path, inputNodeName, outputNodeName, msuffix, binaryPB, readMode, folderPath, checkpointExt, checkpointPath, modelName, shapeArray, modifyshapeAttribue, fixBatchNormal=True): tf.reset_default_graph() os.environ['CUDA_VISIBLE_DEVICES'] = '' config = tf.ConfigProto(allow_soft_placement=True, device_count={ "GPU": 0, "CPU": 1 }) runIncommdLine = False if (os.path.isfile(file_path)): g_in, graph_def = loadGraph(file_path, binaryPB) else: raise ValueError('Can not find : %s ' % folderDir) fixTrainingBNAndDropout = False if (fixTrainingBNAndDropout): fcQuantWeightRemoved = False import tensorflow.contrib.graph_editor as ge maxModuleNumber = 6 for i in range(1, maxModuleNumber + 1): #if fire 6 is last fire layer set to 7 blockName = 'fire' + str(i) if (i == 1): bnNames = [ '/bn' ] # mobile net First layer Quant model all 7 fires are '/bn' else: bnNames = [ '/bn' ] # mobile net ['/dw_bn','/1x1_bn'] Squeeze Net ['/bn'] useConnectMethod = False for bnName in bnNames: if (useConnectMethod): nodeName = blockName + bnName + '/moments/Squeeze_1' oldInputName = blockName + bnName + '/moments/variance' newInputName = blockName + bnName + '/moving_variance' node = g_in.get_operation_by_name(nodeName) oldInputNode = g_in.get_operation_by_name(oldInputName) newInputNode = g_in.get_operation_by_name(newInputName) placeHolderNew = tf.identity(newInputNode.outputs[0]) expDim = tf.expand_dims( tf.expand_dims(tf.expand_dims(placeHolderNew, 0), 0), 0) ge.detach(ge.sgv(oldInputNode)) ge.connect(ge.sgv(expDim), ge.sgv(node)) nodeName = blockName + bnName + '/moments/Squeeze' oldInputName = blockName + bnName + '/moments/mean' newInputName = blockName + bnName + '/moving_mean' node = g_in.get_operation_by_name(nodeName) print( '%s before edit new node node.node_def .inputs[0]' % blockName, node.node_def, node.inputs[0]) oldInputNode = g_in.get_operation_by_name(oldInputName) newInputNode = g_in.get_operation_by_name(newInputName) placeHolderNew = tf.identity(newInputNode.outputs[0]) expDim = tf.expand_dims( tf.expand_dims(tf.expand_dims(placeHolderNew, 0), 0), 0) ge.detach(ge.sgv(oldInputNode)) ge.connect(ge.sgv(expDim), ge.sgv(node)) print( '%s after edit new node node.node_def .inputs[0]' % blockName, node.node_def, node.inputs[0]) else: oldInputName = blockName + bnName + '/moments/Squeeze_1' newInputName = blockName + bnName + '/moving_variance' oldInputNode = g_in.get_operation_by_name(oldInputName) newInputNode = g_in.get_operation_by_name(newInputName) placeHolderNew = tf.identity(newInputNode.outputs[0]) ge.swap_outputs(ge.sgv(placeHolderNew), ge.sgv(oldInputNode)) oldInputName = blockName + bnName + '/moments/Squeeze' newInputName = blockName + bnName + '/moving_mean' oldInputNode = g_in.get_operation_by_name(oldInputName) newInputNode = g_in.get_operation_by_name(newInputName) placeHolderNew = tf.identity(newInputNode.outputs[0]) ge.swap_outputs(ge.sgv(placeHolderNew), ge.sgv(oldInputNode)) removeWeightQuantize = False if (removeWeightQuantize): oldInputName = blockName + '/conv3x3/add' newInputName = blockName + '/conv3x3/kernels' #print('%s before edit new node node.node_def .inputs[0]'%blockName, node.node_def , node.inputs[0]) oldInputNode = g_in.get_operation_by_name(oldInputName) newInputNode = g_in.get_operation_by_name(newInputName) placeHolderNew = tf.identity(newInputNode.outputs[0]) # expDim=tf.expand_dims(tf.expand_dims(tf.expand_dims(placeHolderNew,0),0),0) # ge.detach (ge.sgv(oldInputNode)) ge.swap_outputs( ge.sgv(placeHolderNew), ge.sgv(oldInputNode)) #reroute_outputs get same results #Remove FC layer if (fcQuantWeightRemoved == False): oldInputName = 'logit/add' newInputName = 'logit/weights' #print('%s before edit new node node.node_def .inputs[0]'%blockName, node.node_def , node.inputs[0]) oldInputNode = g_in.get_operation_by_name(oldInputName) newInputNode = g_in.get_operation_by_name(newInputName) placeHolderNew = tf.identity(newInputNode.outputs[0]) # expDim=tf.expand_dims(tf.expand_dims(tf.expand_dims(placeHolderNew,0),0),0) # ge.detach (ge.sgv(oldInputNode)) ge.swap_outputs(ge.sgv(placeHolderNew), ge.sgv( oldInputNode)) #reroute_outputs get same results fcQuantWeightRemoved = True with tf.Session(config=config, graph=g_in) as sess: graph_def = sess.graph_def for node in graph_def.node: if 'dropout/mul' in node.name: deleteDropOut = True if (deleteDropOut): oldInputName = 'dropout/mul' newInputName = 'fire6/pool/MaxPool' oldInputNode = g_in.get_operation_by_name( oldInputName) newInputNode = g_in.get_operation_by_name( newInputName) placeHolderNew = tf.identity( newInputNode.outputs[0]) ge.swap_outputs( ge.sgv(placeHolderNew), ge.sgv(oldInputNode) ) #reroute_outputs get same results else: for node in graph_def.node: if node.name == 'dropout/keep_prob': #node.attr['value'].tensor.float_val=tf.convert_to_tensor (1,dtype=tf.float32) #node.attr['value'].tensor.float_val=1 node.attr['value'].tensor.CopyFrom( tensor_util.make_tensor_proto( 1.0, dtype=tf.float32)) #node.attr['value'].value =1.0 # fix batch normal node nodes https://github.com/tensorflow/tensorflow/issues/3628 if (fixBatchNormal): for node in graph_def.node: if node.op == 'RefSwitch': node.op = 'Switch' for index in range(len(node.input)): if 'moving_' in node.input[index]: node.input[index] = node.input[index] + '/read' elif node.op == 'AssignSub': node.op = 'Sub' if 'use_locking' in node.attr: del node.attr['use_locking'] elif node.op == 'AssignAdd': node.op = 'Add' if 'use_locking' in node.attr: del node.attr['use_locking'] if ('dilations') in node.attr: del node.attr['dilations'] node.device = "" #fixVariables not Working fixVariables = False if (fixVariables and node.op == 'VariableV2' and ('batchnorm/var' in node.name or 'batchnorm/mean' in node.name)): outputNodes = find_output_nodes(graph_def, node) for index in range(len(outputNodes)): if (outputNodes[index].op == 'Assign'): #node.output[index] = node.output[index] + '/read' #outputNodes[index].op ='Identity' outputNodes[index].name = outputNodes[index].name + '/read' print('Modified %s ' % outputNodes[index].name) #################### Step 1 Training to inference simplification , need checkpoint and .pbtxt files from training ###################################################### graphDef = optimize_for_inference_lib.optimize_for_inference( graph_def, [inputNodeName], # an array of the input node(s) [outputNodeName] if type(outputNodeName) is str else [item for item in outputNodeName], # an array of output nodes tf.float32.as_datatype_enum) if (modifyshapeAttribue): inputOpType = 'Placeholder' for n in graphDef.node: #print('node to modify',n.name) if (n.name == inputNodeName): print('node to modify', n) setNodeAttribute(n, 'shape', shapeArray) if (n.op != inputOpType): print( "--Node %s op %s set to op=%s" % (inputNodeName, n.op, inputOpType), shapeArray) n.op = inputOpType print("--Name of the node - %s shape set to " % n.name, shapeArray) print('node after modify', n) modifyClipValue = False if (modifyClipValue): for i in range(1, maxModuleNumber + 1): blockName = 'fire' + str(i) newClipValue = 127 clipVNodeName = blockName + '/conv3x3/Rint_1/x' #clipnode= g_in.get_operation_by_name(clipVNodeName) clipnode = find_node_by_name(graphDef, clipVNodeName) print('clipnode to modify', clipnode) setNodeConstValue(graph_def, clipnode, newClipValue) print("--Name of the node - %s shape set to %f" % (clipnode.name, newClipValue)) print('clipnode after modify', clipnode) modifyFCClip = True if (modifyFCClip and i == maxModuleNumber): clipVNodeName = 'conv12/Rint_1/x' clipnodeFC = find_node_by_name(graphDef, clipVNodeName) #clipnodeFC= g_in.get_operation_by_name(clipVNodeName) setNodeConstValue(graph_def, clipnodeFC, newClipValue) print('clipnode after modify', clipnodeFC) if (runIncommdLine): copyfile(file_path, file_path + trainModelSuffix) outputNameSuffix = '%s_frozenforInference.pb' % checkpointExt inferenceSuffix = '.Inference' tf.train.write_graph(graphDef, folderPath, checkpointPath + modelName + '.pb' + inferenceSuffix, as_text=False) tf.train.write_graph(graphDef, folderPath, checkpointPath + modelName + '.pbtxt' + inferenceSuffix, as_text=True) pbfileoutput_path = checkpointPath + modelName + outputNameSuffix checkpointfile_path = checkpointPath + modelName + checkpointExt pbfile_path = checkpointPath + modelName + msuffix + inferenceSuffix #################### Step 2 Frozen Inference mode ###################################################### freeze_graph.freeze_graph( input_graph=pbfile_path, input_saver='', input_binary=binaryPB, input_checkpoint=checkpointfile_path, # an array of the input node(s) output_node_names=outputNodeName if type(outputNodeName) is str else ",".join(outputNodeName), restore_op_name="save/restore_all", #Unused. filename_tensor_name="save/Const:0", # Unused. output_graph=pbfileoutput_path, # an array of output nodes clear_devices=True, initializer_nodes='') #################### Step 3 Save in tensor board ###################################################### saveTensorboardForVisualizatoin = False if (saveTensorboardForVisualizatoin): modelFullPath = checkpointPath + modelName + outputNameSuffix tensorboardPath = checkpointPath + '\\tensorboard' #if not os.path.exists(tensorboardPath): # os.mkdir(tensorboardPath) createTensorboard(modelFullPath, tensorboardPath)
biases['biases_h2'])) layer_final = tf.add(tf.matmul(layer_2, weights['weights_output']), biases['biases_output']) # Define loss and optimizer, minimize the squared error cost = tf.reduce_mean(tf.square(layer_final - Y)) optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01) train = optimizer.minimize(cost) # Modify the graph from tensorflow.contrib import graph_editor as ge sess = tf.Session() all_ops = sess.graph.get_operations() l1_swap_in = sess.graph.get_operation_by_name("L1_SwapIn") matmul_1_grad = sess.graph.get_operation_by_name("gradients/MatMul_1_grad/MatMul_1") print(ge.sgv(matmul_1_grad)) print(ge.sgv(l1_swap_in)) ret = ge.connect(ge.sgv(l1_swap_in), ge.sgv(matmul_1_grad).remap_inputs([0])) sess.close() # Generate graph visualization graph = tf.get_default_graph() writer = tf.summary.FileWriter("./simple_graph_events") writer.add_graph(graph=graph)
# #remove all conncetions from matmul ge.detach(ge.sgv(matmul.op)) with tf.Session(graph=graph) as sess: os.system("mkdir /flush1/raj034/RNN/model/test_cases/" + breath + "/" + quant + "/fact_" + str(lvl)) # for op in sess.graph.get_operations(): # print(op.name) W = W1.eval() u, s, v, ss = svd_compress_gs(W, lvl) logEntry("structural_similarity == > " + str(ss)) u1 = tf.matmul(reshape, u, name="prefix/u1") s1 = tf.matmul(u1, s, name="prefix/s1") v1 = tf.matmul(s1, v, name="prefix/v1") ge.connect(ge.sgv(v1.op), ge.sgv(add.op).remap_inputs([0])) sess.run(tf.variables_initializer([tf.Variable(5, name="dummy" + str(lvl))])) saver = tf.train.Saver() # save log for tensorboad LOGDIR = "/flush1/raj034/RNN/model/test_cases/" + breath + "/" + quant + '/fact_' + str(lvl) + '/LOG' train_writer = tf.summary.FileWriter(LOGDIR) train_writer.add_graph(sess.graph) train_writer.close() # save the freezed model os.system("mkdir /flush1/raj034/RNN/model/test_cases/" + breath + "/" + quant + "/fact_" + str(lvl) + "/pb") tf.train.write_graph(sess.graph_def, "/flush1/raj034/RNN/model/test_cases/" + breath + "/" + quant + '/fact_' + str(lvl) + '/pb/', "RNN_" + breath + "_" + quant + "_fact_" + str(lvl) + ".pbtxt") saver.save(sess, save_path="/flush1/raj034/RNN/model/test_cases/" + breath + "/" + quant + '/fact_' + str(lvl) + "/model.ckpt")
if next_op is None: raise ValueError('No suitable next op found to connect to. Try looking at the graph or full list of forward ops') # Add placeholder and variable add_op = tf.add(var, delta_placeholder) # TODO - might be neater if these were created in the same scope as the variable; also might solve issue with connecting add ops within while loop # Connect add_op output to next op input # Create subgraph 1 (outputs) sgv0 = ge.sgv(add_op.op) # Create subgraph 2 (inputs) sgv1 = ge.sgv(next_op).remap_inputs([1]) # Connect ge.connect(sgv0, sgv1) # TODO - sort out error with tf.while loops; may not be possible: try the assign_add method first # Define parameter update ops update = 0.01*delta_placeholder new_update_op = tf.assign_sub(var, update) if gamma_update_op is not None: gamma_update_op = tf.group(new_update_op, gamma_update_op) else: gamma_update_op = new_update_op merged = tf.summary.merge_all() sess = tf.Session() sess.run(tf.global_variables_initializer()) summary_writer = tf.summary.FileWriter('results\\add_noise_experiments\\', sess.graph)
l = tf.zeros_like(b) e1 = tf.identity(l) d = tf.placeholder(tf.bool) e = tf.cond(d, lambda: tf.identity(a), lambda: tf.identity(e1)) e2 = tf.identity(e) gr = tf.get_default_graph() pp = gr.get_operation_by_name('add') ss = ge.sgv(pp.outputs[0]) pp1 = gr.get_operation_by_name('Identity') ss1 = ge.sgv(pp1.outputs[0]) pp2 = gr.get_operation_by_name('Identity_1') ss2 = ge.sgv(pp2.inputs[0]) # ge.detach_inputs(c) # x = ge.graph_replace(x,{b:e}) # ge.detach_outputs(c) m1 = ge.connect(ss2, ss) m2 = ge.connect(ss, ss1) # m = ge.swap_inputs(ss,ss1) sess = tf.InteractiveSession() PoseTools.output_graph('ge') sess.close() ## with tf.Session() as sess: q = sess.run(x, {a: np.zeros([3, 3]), d: True}) print q q = sess.run(x, {a: np.zeros([3, 3]), d: False}) print q