def train(): """Train CIFAR-10 for a number of steps.""" g1 = tf.Graph() with g1.as_default(): global_step = tf.contrib.framework.get_or_create_global_step() # Get images and labels for CIFAR-10. images, labels = cifar10.distorted_inputs() # Build a Graph that computes the logits predictions from the # inference model. logits = cifar10.inference(images) # Calculate loss. loss = cifar10.loss(logits, labels) grads = cifar10.train_part1(loss, global_step) only_gradients = [g for g, _ in grads] only_vars = [v for _, v in grads] placeholder_gradients = [] #with tf.device("/gpu:0"): for grad_var in grads: placeholder_gradients.append( (tf.placeholder('float', shape=grad_var[0].get_shape()), grad_var[1])) feed_dict = {} for i, grad_var in enumerate(grads): feed_dict[placeholder_gradients[i][0]] = np.zeros( placeholder_gradients[i][0].shape) train_op = cifar10.train_part2(global_step, placeholder_gradients) sess = tf.Session() sess.run(tf.global_variables_initializer()) feeds = [] print("Reached here") for i, grad_var in enumerate(grads): feeds.append(placeholder_gradients[i][0]) # Partial Run print("Reached here", len(feeds)) for x in feeds: print(x, ) h = sess.partial_run_setup([only_gradients, train_op], feeds) print("Reached here") for i in xrange(10): res_grads = sess.partial_run(h, only_gradients, feed_dict=feed_dict) feed_dict = {} for i, grad_var in enumerate(res_grads): feed_dict[placeholder_gradients[i][0]] = res_grads[i] res_train_op = sess.partial_run(h, train_op, feed_dict=feed_dict)
def train(): """Train CIFAR-10 for a number of steps.""" g1 = tf.Graph() with g1.as_default(): #global_step = tf.contrib.framework.get_or_create_global_step() global_step = tf.Variable(-1, name='global_step', trainable=False, dtype=tf.int32) increment_global_step_op = tf.assign(global_step, global_step+1) # Get images and labels for CIFAR-10. images, labels = cifar10.distorted_inputs2() # Build a Graph that computes the logits predictions from the # inference model. logits = cifar10.inference(images) # Calculate loss. loss = cifar10.loss(logits, labels) grads = cifar10.train_part1(loss, global_step) only_gradients = [g for g,_ in grads] class _LoggerHook(tf.train.SessionRunHook): """Logs loss and runtime.""" def begin(self): self._step = -1 self._start_time = time.time() def before_run(self, run_context): self._step += 1 return tf.train.SessionRunArgs(loss) # Asks for loss value. def after_run(self, run_context, run_values): if self._step % FLAGS.log_frequency == 0: current_time = time.time() duration = current_time - self._start_time self._start_time = current_time loss_value = run_values.results examples_per_sec = FLAGS.log_frequency * FLAGS.batch_size / duration sec_per_batch = float(duration / FLAGS.log_frequency) format_str = ('%s: step %d, loss = %.2f (%.1f examples/sec; %.3f ' 'sec/batch)') print (format_str % (datetime.now(), self._step, loss_value, examples_per_sec, sec_per_batch)) with tf.train.MonitoredTrainingSession( checkpoint_dir=FLAGS.train_dir, hooks=[tf.train.StopAtStepHook(last_step=FLAGS.max_steps), tf.train.NanTensorHook(loss), _LoggerHook()], config=tf.ConfigProto( log_device_placement=FLAGS.log_device_placement, gpu_options=gpu_options)) as mon_sess: # Getting first set of variables s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((TCP_IP, port_main_1)) recv_size = safe_recv(8, s) recv_size = pickle.loads(recv_size) recv_data = safe_recv(recv_size, s) var_vals_1 = pickle.loads(recv_data) s.close() # Getting second set of variables s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((TCP_IP, port_main_2)) recv_size = safe_recv(8, s) recv_size = pickle.loads(recv_size) recv_data = safe_recv(recv_size, s) var_vals_2 = pickle.loads(recv_data) s.close() feed_dict = {} i=0 for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES): if(i < half_index): feed_dict[v] = var_vals_1[i] else: feed_dict[v] = var_vals_2[i-half_index] i=i+1 print("Received variable values from ps") s1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s1.connect((TCP_IP, port_ps1)) s2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s2.connect((TCP_IP, port_ps2)) print("Connected to both PSs") while not mon_sess.should_stop(): gradients, step_val = mon_sess.run([only_gradients,increment_global_step_op], feed_dict=feed_dict) #print("Sending grads port: ", port) # Opening the socket and connecting to server # sending the gradients grad_part1 = [] grad_part2 = [] i=0 for g in gradients: if(i < half_index): grad_part1.append(g) else: grad_part2.append(g) i=i+1 send_data_1 = pickle.dumps(grad_part1,pickle.HIGHEST_PROTOCOL) to_send_size_1 = len(send_data_1) send_size_1 = pickle.dumps(to_send_size_1, pickle.HIGHEST_PROTOCOL) s1.sendall(send_size_1) s1.sendall(send_data_1) send_data_2 = pickle.dumps(grad_part2,pickle.HIGHEST_PROTOCOL) to_send_size_2 = len(send_data_2) send_size_2 = pickle.dumps(to_send_size_2, pickle.HIGHEST_PROTOCOL) s2.sendall(send_size_2) s2.sendall(send_data_2) #print("sent grads") #receiving the variable values recv_size = safe_recv(8, s1) recv_size = pickle.loads(recv_size) recv_data = safe_recv(recv_size, s1) var_vals_1 = pickle.loads(recv_data) recv_size = safe_recv(8, s2) recv_size = pickle.loads(recv_size) recv_data = safe_recv(recv_size, s2) var_vals_2 = pickle.loads(recv_data) #print("recved grads") feed_dict = {} i=0 for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES): if(i < half_index): feed_dict[v] = var_vals_1[i] else: feed_dict[v] = var_vals_2[i-half_index] i=i+1 s1.close() s2.close()
def train(): """Train CIFAR-10 for a number of steps.""" g1 = tf.Graph() with g1.as_default(): global_step = tf.contrib.framework.get_or_create_global_step() # Get images and labels for CIFAR-10. images, labels = cifar10.distorted_inputs() # Build a Graph that computes the logits predictions from the # inference model. logits = cifar10.inference(images) # Calculate loss. loss = cifar10.loss(logits, labels) grads = cifar10.train_part1(loss, global_step) only_gradients = [g for g, _ in grads] only_vars = [v for _, v in grads] placeholder_gradients = [] #with tf.device("/gpu:0"): for grad_var in grads: placeholder_gradients.append( (tf.placeholder('float', shape=grad_var[0].get_shape()), grad_var[1])) feed_dict = {} for i, grad_var in enumerate(grads): feed_dict[placeholder_gradients[i][0]] = np.zeros( placeholder_gradients[i][0].shape) # Build a Graph that trains the model with one batch of examples and # updates the model parameters. train_op = cifar10.train_part2(global_step, placeholder_gradients) class _LoggerHook(tf.train.SessionRunHook): """Logs loss and runtime.""" def begin(self): self._step = -1 self._start_time = time.time() def before_run(self, run_context): self._step += 1 return tf.train.SessionRunArgs(loss) # Asks for loss value. def after_run(self, run_context, run_values): if self._step % FLAGS.log_frequency == 0: current_time = time.time() duration = current_time - self._start_time self._start_time = current_time loss_value = run_values.results examples_per_sec = FLAGS.log_frequency * FLAGS.batch_size / duration sec_per_batch = float(duration / FLAGS.log_frequency) format_str = ( '%s: step %d, loss = %.2f (%.1f examples/sec; %.3f ' 'sec/batch)') print(format_str % (datetime.now(), self._step, loss_value, examples_per_sec, sec_per_batch)) with tf.train.MonitoredTrainingSession( checkpoint_dir=FLAGS.train_dir, hooks=[ tf.train.StopAtStepHook(last_step=FLAGS.max_steps), tf.train.NanTensorHook(loss), _LoggerHook() ], config=tf.ConfigProto( log_device_placement=FLAGS.log_device_placement, gpu_options=gpu_options)) as mon_sess: global port while not mon_sess.should_stop(): gradients = mon_sess.run(only_gradients, feed_dict=feed_dict) # pickling the gradients send_data = pickle.dumps(gradients, pickle.HIGHEST_PROTOCOL) # finding size of pickled gradients to_send_size = len(send_data) # Sending the size of the gradients first send_size = pickle.dumps(to_send_size, pickle.HIGHEST_PROTOCOL) s.sendall(send_size) # sending the gradients s.sendall(send_data) recv_size = safe_recv(8, s) recv_size = pickle.loads(recv_size) recv_data = safe_recv(recv_size, s) gradients2 = pickle.loads(recv_data) #print("Recevied gradients of size: ", len(recv_data)) feed_dict = {} for i, grad_var in enumerate(gradients2): feed_dict[placeholder_gradients[i][0]] = gradients2[i] #print(gradients[i].shape) #print(gradients2[i].shape) res = mon_sess.run(train_op, feed_dict=feed_dict)