def test_fw(self): """Tests the forward computation is the same.""" with tf.Graph().as_default(), self.test_session( ) as sess, log.verbose_level(2): config = get_config("resnet-test") config.num_channel = 4 config.height = 8 config.width = 8 np.random.seed(0) xval = np.random.uniform(-1.0, 1.0, [10, 8, 8, 4]).astype(np.float32) x = tf.constant(xval) x1 = x[:5, :, :, :] x2 = x[5:, :, :, :] # We need to split two regular runs because of the complication brought by # batch normalization. with tf.variable_scope("Model", reuse=None): m11 = get_model("resnet", config, inp=x1) with tf.variable_scope("Model", reuse=True): m12 = get_model("resnet", config, inp=x2) with tf.variable_scope("Model", reuse=True): m2 = get_multi_gpu_model("resnet", config, num_replica=2, inp=x) sess.run(tf.global_variables_initializer()) y11, y12, y2 = sess.run([m11.output, m12.output, m2.output]) np.testing.assert_allclose(y11, y2[:5, :], rtol=1e-5) np.testing.assert_allclose(y12, y2[5:, :], rtol=1e-5)
def _test_single_pass(self, method): config = get_config("resnet-test") config.momentum = 0.0 config.base_learn_rate = 1e-1 np.random.seed(0) BSIZE = config.batch_size xval = np.random.uniform( -1.0, 1.0, [BSIZE, config.height, config.width, config.num_channel]).astype(np.float32) yval = np.floor(np.random.uniform(0, 9.9, [BSIZE])).astype(np.int32) # Run multi tower version. with tf.Graph().as_default(), self.test_session( ) as sess, log.verbose_level(2): x = tf.constant(xval) y = tf.constant(yval) with tf.variable_scope("Model", reuse=None): m1 = get_multi_gpu_model( "resnet", config, num_replica=2, inp=x, label=y) sess.run(tf.global_variables_initializer()) m1.assign_lr(sess, config.base_learn_rate) tvars = tf.trainable_variables() tvars_str = map(lambda x: x.name, tvars) saver = tf.train.Saver(tvars) if not os.path.exists(FOLDER): os.makedirs(FOLDER) saver.save(sess, CKPT_FNAME) m1.train_step(sess) tvars_v1 = sess.run(tvars) tvars_d1 = dict(zip(tvars_str, tvars_v1)) # Run MultiPassModel. with tf.Graph().as_default(), self.test_session( ) as sess, log.verbose_level(2): with tf.variable_scope("Model", reuse=True): m2 = MultiPassModel( config, ResNetModel, num_passes=2, debug=True, inp=x, label=y, aggregate_method=method) tvars = tf.trainable_variables() saver = tf.train.Saver(tvars) sess.run(tf.global_variables_initializer()) saver.restore(sess, CKPT_FNAME) m2.assign_lr(sess, config.base_learn_rate) m2.train_step(sess) tvars_v2 = sess.run(tvars) tvars_d2 = dict(zip(tvars_str, tvars_v2)) for vv in tvars_str: log.info(vv, verbose=2) np.testing.assert_allclose( tvars_d1[vv], tvars_d2[vv], rtol=1e-4, atol=1e-6) log.info("...ok", verbose=2)
def _get_config(): if FLAGS.config is not None: return get_config_from_json(FLAGS.config) else: if FLAGS.restore: save_folder = os.path.realpath( os.path.abspath(os.path.join(FLAGS.results, FLAGS.id))) return get_config_from_json(os.path.join(save_folder, "conf.json")) else: return get_config(FLAGS.model)
def _test_grad(self, modelname): """Tests the manual gradients for every layer.""" with tf.Graph().as_default(), self.test_session( ) as sess, log.verbose_level(2): config = get_config(modelname) config.manual_gradients = False config.filter_initialization = "uniform" # Declare a regular bprop model. m1 = get_model(config.model_class, config, is_training=True) name_list1 = map(lambda x: x[1].name, m1.grads_and_vars) grads1 = map(lambda x: x[0], m1.grads_and_vars) tf.get_variable_scope().reuse_variables() # Declare a manual bprop model. config.manual_gradients = True m2 = get_model(config.model_class, config, is_training=True) name_list2 = map(lambda x: x[1].name, m2.grads_and_vars) grads2 = map(lambda x: x[0], m2.grads_and_vars) # Check lengths are equal. self.assertEqual(len(m1.grads_and_vars), len(m2.grads_and_vars)) # Prepare synthetic data. xval = np.random.uniform(-1.0, 1.0, [ config.batch_size, config.height, config.width, config.num_channel ]).astype(np.float32) yval = np.floor( np.random.uniform(0.0, config.num_classes - 0.1, [config.batch_size ])).astype(np.int32) sess.run(tf.global_variables_initializer()) g1 = sess.run(grads1, feed_dict={m1.input: xval, m1.label: yval}) gdict1 = dict(zip(name_list1, g1)) g2 = sess.run(grads2, feed_dict={m2.input: xval, m2.label: yval}) gdict2 = dict(zip(name_list2, g2)) # Check two gradients are the same. check_two_dict(gdict1, gdict2, tol=1e0, name=modelname)
def _test_getmodel(self, modelname): with tf.Graph().as_default(), self.test_session(), log.verbose_level(2): config = get_config(modelname) get_model(config.model_class, config)
def _get_config(): # Manually set config. if FLAGS.config is not None: return get_config_from_json(FLAGS.config) else: return get_config(FLAGS.model)
def test_bk(self): """Tests the backward computation is the same.""" with tf.Graph().as_default(), self.test_session( ) as sess, log.verbose_level(2): config = get_config("resnet-test") config.num_channel = 4 config.height = 8 config.width = 8 np.random.seed(0) xval = np.random.uniform(-1.0, 1.0, [10, 8, 8, 4]).astype(np.float32) yval = np.floor(np.random.uniform(0, 9.9, [10])).astype(np.int32) x = tf.constant(xval) y = tf.constant(yval) # log.fatal(y.get_shape()) x1 = x[:5, :, :, :] x2 = x[5:, :, :, :] y1 = y[:5] y2 = y[5:] with tf.variable_scope("Model", reuse=None): m11 = get_model("resnet", config, inp=x1, label=y1) with tf.variable_scope("Model", reuse=True): m12 = get_model("resnet", config, inp=x2, label=y2) with tf.variable_scope("Model", reuse=True): m2 = get_multi_gpu_model("resnet", config, num_replica=2, inp=x, label=y) sess.run(tf.global_variables_initializer()) tvars = tf.global_variables() name_list11 = map(lambda x: x[1].name, m11.grads_and_vars) grads11 = map(lambda x: x[0], m11.grads_and_vars) g11 = sess.run(grads11) gdict11 = dict(zip(name_list11, g11)) name_list12 = map(lambda x: x[1].name, m12.grads_and_vars) grads12 = map(lambda x: x[0], m12.grads_and_vars) g12 = sess.run(grads12) gdict12 = dict(zip(name_list12, g12)) name_list21 = map(lambda x: x[1].name, m2.tower_grads_and_vars[0]) grads21 = map(lambda x: x[0], m2.tower_grads_and_vars[0]) g21 = sess.run(grads21) gdict21 = dict(zip(name_list21, g21)) name_list22 = map(lambda x: x[1].name, m2.tower_grads_and_vars[1]) grads22 = map(lambda x: x[0], m2.tower_grads_and_vars[1]) g22 = sess.run(grads22) gdict22 = dict(zip(name_list22, g22)) # Check two gradients are the same. check_two_dict(gdict11, gdict21) check_two_dict(gdict12, gdict22) # Check the average gradients are the same. name_list2 = map(lambda x: x[1].name, m2.grads_and_vars) grads2 = map(lambda x: x[0], m2.grads_and_vars) g2 = sess.run(grads2) gdict2 = dict(zip(name_list2, g2)) name_list1 = name_list11 g1 = [(gdict11[kk] + gdict12[kk]) / 2.0 for kk in name_list1] gdict1 = dict(zip(name_list1, g1)) check_two_dict(gdict1, gdict2)