def test_reuse_model(self, avg_pool, is_training): batch_size = 2 num_classes = 10 images = _construct_images(batch_size) config = bagnet_config.get_config() config.num_classes = num_classes config.avg_pool = avg_pool model = bagnet_model.BagNet(config) # Build once. logits1, _ = model(images, is_training) num_params = len(tf.all_variables()) l2_loss1 = tf.losses.get_regularization_loss() # Build twice. logits2, _ = model( images, is_training) l2_loss2 = tf.losses.get_regularization_loss() # Ensure variables are reused. self.assertLen(tf.all_variables(), num_params) init = tf.global_variables_initializer() self.evaluate(init) logits1, logits2 = self.evaluate((logits1, logits2)) l2_loss1, l2_loss2 = self.evaluate((l2_loss1, l2_loss2)) np.testing.assert_almost_equal(logits1, logits2, decimal=9) np.testing.assert_almost_equal(l2_loss1, l2_loss2, decimal=9)
def test_reuse_model(self, policy, is_training): config = saccader_config.get_config() num_times = 2 image_shape = (224, 224, 3) num_classes = 10 config.num_classes = num_classes config.num_times = num_times batch_size = 3 images1 = tf.constant(np.random.rand(*((batch_size, ) + image_shape)), dtype=tf.float32) model = saccader.Saccader(config) logits1 = model(images1, num_times=num_times, is_training=is_training, policy=policy)[0] num_params = len(tf.all_variables()) l2_loss1 = tf.losses.get_regularization_loss() # Build twice with different num_times and different batch size. images2 = tf.constant(np.random.rand(*((batch_size - 1, ) + image_shape)), dtype=tf.float32) logits2 = model(images2, num_times=num_times + 1, is_training=is_training, policy=policy)[0] l2_loss2 = tf.losses.get_regularization_loss() # Ensure variables are reused. self.assertLen(tf.all_variables(), num_params) init = tf.global_variables_initializer() self.evaluate(init) logits1, logits2 = self.evaluate((logits1, logits2)) l2_loss1, l2_loss2 = self.evaluate((l2_loss1, l2_loss2)) np.testing.assert_almost_equal(l2_loss1, l2_loss2, decimal=9)
def test_reuse_model(self, is_training): batch_size = 5 images = _construct_images(batch_size) config = mnist_config.ConfigDict() model = mnist_model.MNISTNetwork(config) # Build once. logits1, _ = model(images, is_training) num_params = len(tf.all_variables()) l2_loss1 = tf.losses.get_regularization_loss() # Build twice. logits2, _ = model(images, is_training) l2_loss2 = tf.losses.get_regularization_loss() # Ensure variables are reused. self.assertLen(tf.all_variables(), num_params) init = tf.global_variables_initializer() with self.test_session() as sess: sess.run(init) # Ensure operations are the same after reuse. err_logits = (np.abs(sess.run(logits1 - logits2))).sum() self.assertAlmostEqual(err_logits, 0, 9) err_losses = (np.abs(sess.run(l2_loss1 - l2_loss2))).sum() self.assertAlmostEqual(err_losses, 0, 9)
def test_reuse_model(self, rank, is_training, special_type): batch_size = 5 num_classes = 10 num_channels = 3 images = _construct_images(batch_size) config = simple_model_config.get_config() config.num_classes = num_classes config.num_channels = num_channels config.batch_norm = False config.kernel_size_list = [3, 3, 3] config.num_filters_list = [64, 64, 64] config.strides_list = [2, 2, 1] config.layer_types = ['conv2d', special_type, 'conv2d'] config.rank = rank model = simple_model.SimpleNetwork(config) # Build once. logits1, _ = model(images, is_training) num_params = len(tf.all_variables()) # Build twice. logits2, _ = model(images, is_training) # Ensure variables are reused. self.assertLen(tf.all_variables(), num_params) init = tf.global_variables_initializer() with self.test_session() as sess: sess.run(init) # Ensure operations are the same after reuse. err_logits = (np.abs(sess.run(logits1 - logits2))).sum() self.assertAlmostEqual(err_logits, 0, 9)
def train_intent(): #config=tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True)) with tf.Session() as sess: labels_train, labels_test, data_filter_train, data_filter_test = get_test_data( ) accuracy, x, y_target, keep_prob, train_step, y, cross_entropy, W_conv1 = create_graph( train=True) path = pathConfigs.model_path if not os.path.exists(path): os.makedirs(path) num_ckpt = 0 # 이미 check point가 있으면 있는거 사용 if len(os.listdir(path)) > 3: print('LOAD CNN MODEL') dir = os.listdir(path) for i in dir: try: new_one = int(i.split('-')[1].split('.')[0]) if num_ckpt < new_one: num_ckpt = new_one except: pass restorer = tf.train.Saver(tf.all_variables()) restorer.restore( sess, pathConfigs.model_path + 'check_point-' + str(num_ckpt) + '.ckpt') else: # 학습된거 없으면 새로 만듦 sess.run(tf.global_variables_initializer()) for i in range(learning_step): sess.run(train_step, feed_dict={ x: data_filter_train, y_target: labels_train, keep_prob: 0.5 }) index = i + num_ckpt if i % 100 == 0: train_accuracy = sess.run(accuracy, feed_dict={ x: data_filter_train, y_target: labels_train, keep_prob: 1 }) print("step %d, training accuracy: %.3f" % (index, train_accuracy)) if i != 0 and i % 100 == 0: saver = tf.train.Saver(tf.all_variables()) saver.save(sess, path + 'check_point-' + str(index) + '.ckpt') print("Save Models checkpoint : %d" % index) print("Current TIME: ", datetime.now())
def restore_best_model(): """Load bestmodel file from eval directory, add variables for adagrad, and save to train directory""" tf.logging.info("Restoring bestmodel for training...") # Initialize all vars in the model sess = tf.Session(config=util.get_config()) print("Initializing all variables...") sess.run(tf.initialize_all_variables()) # Restore the best model from eval dir saver = tf.train.Saver( [v for v in tf.all_variables() if "Adagrad" not in v.name]) print("Restoring all non-adagrad variables from best model in eval dir...") curr_ckpt = util.load_ckpt(saver, sess, "eval") print("Restored %s." % curr_ckpt) # Save this model to train dir and quit new_model_name = curr_ckpt.split("/")[-1].replace("bestmodel", "model") new_fname = os.path.join(FLAGS.log_root, "train", new_model_name) print("Saving model to %s..." % (new_fname)) new_saver = tf.train.Saver( ) # this saver saves all variables that now exist, including Adagrad variables new_saver.save(sess, new_fname) print("Saved.") exit()
def __init__(self, sess, ac_dim, ob_dim, n_layers, size, learning_rate=1e-4, training=True, policy_scope='policy_vars', discrete=False, # unused for now nn_baseline=False, # unused for now **kwargs): super().__init__(**kwargs) # init vars self.sess = sess self.ac_dim = ac_dim self.ob_dim = ob_dim self.n_layers = n_layers self.size = size self.discrete = discrete self.learning_rate = learning_rate self.training = training self.nn_baseline = nn_baseline # build TF graph with tf.variable_scope(policy_scope, reuse=tf.AUTO_REUSE): self.build_graph() # saver for policy variables that are not related to training self.policy_vars = [v for v in tf.all_variables() if policy_scope in v.name and 'train' not in v.name] self.policy_saver = tf.train.Saver(self.policy_vars, max_to_keep=None)
def create_vggish_frozen_graph(): """Create the VGGish frozen graph.""" os.system('git clone https://github.com/tensorflow/models.git') sys.path.append('models/research/audioset/vggish/') import vggish_slim os.system( 'curl -O https://storage.googleapis.com/audioset/vggish_model.ckpt') ckpt_path = 'vggish_model.ckpt' with tf.Graph().as_default(), tf.Session() as sess: vggish_slim.define_vggish_slim(training=False) vggish_slim.load_vggish_slim_checkpoint(sess, ckpt_path) saver = tf.train.Saver(tf.all_variables()) freeze_graph.freeze_graph_with_def_protos( sess.graph_def, saver.as_saver_def(), ckpt_path, 'vggish/fc2/BiasAdd', restore_op_name=None, filename_tensor_name=None, output_graph='/tmp/mediapipe/vggish_new.pb', clear_devices=True, initializer_nodes=None) os.system('rm -rf models/') os.system('rm %s' % ckpt_path)
def __init__(self, n_actions, n_features, learning_rate, suffix): self.n_actions = n_actions self.n_features = n_features self.lr = learning_rate self.suffix = suffix """ self.ep_obs, self.ep_as, self.ep_rs: observation, action, reward, recorded for each batch """ self.ep_obs, self.ep_as, self.ep_rs, self.ep_ss = [], [], [], [] """ self.tput_batch: record throughput for each batch, used to show progress while training self.tput_persisit, self.episode: persist to record throughput, used to be stored and plot later """ self.tput_batch = [] self.tput_persisit = [] self.ss_batch = [] self.ss_persisit = [] self.episode = [] self.ss_perapp_persisit = [] self.ss_coex_persisit = [] self.ss_sum_persisit = [] # TODO self.vio = []: violation self._build_net() self.sess = tf.Session() self.sess.run(tf.global_variables_initializer()) restore = [ 'Actor' + self.suffix + '/fc1' + self.suffix + '/kernel:0', 'Actor' + self.suffix + '/fc1' + self.suffix + '/bias:0', 'Actor' + self.suffix + '/fc2' + self.suffix + '/kernel:0', 'Actor' + self.suffix + '/fc2' + self.suffix + '/bias:0' ] restore_var = [v for v in tf.all_variables() if v.name in restore] self.saver = tf.train.Saver(var_list=restore_var)
def export_weights(self): """ Function to store TensorFlow weights back to into a dict in CoreML format to be used by the C++ implementation Returns ------- tf_export_params: Dictionary Dictionary of weights from TensorFlow stored as {weight_name: weight_value} """ tf_export_params = {} tvars = _tf.trainable_variables() tvars_vals = self.sess.run(tvars) for var, val in zip(tvars, tvars_vals): if 'weight' in var.name: if var.name.startswith('conv'): tf_export_params[var.name.split( ':')[0]] = _utils.convert_conv1d_tf_to_coreml(val) elif var.name.startswith('dense'): tf_export_params[var.name.split( ':')[0]] = _utils.convert_dense_tf_to_coreml(val) elif var.name.startswith('rnn/lstm_cell/kernel'): i2h_i, i2h_c, i2h_f, i2h_o, h2h_i, h2h_c, h2h_f, h2h_o = _utils.convert_lstm_weight_tf_to_coreml( val, CONV_H) tf_export_params['lstm_i2h_i_weight'] = i2h_i tf_export_params['lstm_i2h_c_weight'] = i2h_c tf_export_params['lstm_i2h_f_weight'] = i2h_f tf_export_params['lstm_i2h_o_weight'] = i2h_o tf_export_params['lstm_h2h_i_weight'] = h2h_i tf_export_params['lstm_h2h_c_weight'] = h2h_c tf_export_params['lstm_h2h_f_weight'] = h2h_f tf_export_params['lstm_h2h_o_weight'] = h2h_o elif var.name.startswith('rnn/lstm_cell/bias'): h2h_i_bias, h2h_c_bias, h2h_f_bias, h2h_o_bias = _utils.convert_lstm_bias_tf_to_coreml( val) tf_export_params['lstm_h2h_i_bias'] = h2h_i_bias tf_export_params['lstm_h2h_c_bias'] = h2h_c_bias tf_export_params['lstm_h2h_f_bias'] = h2h_f_bias tf_export_params['lstm_h2h_o_bias'] = h2h_o_bias elif var.name.startswith('batch_normalization'): tf_export_params[ 'bn_' + var.name.split('/')[-1][0:-2]] = _np.array(val) else: tf_export_params[var.name.split(':')[0]] = _np.array(val) tvars = _tf.all_variables() tvars_vals = self.sess.run(tvars) for var, val in zip(tvars, tvars_vals): if 'moving_mean' in var.name: tf_export_params['bn_running_mean'] = _np.array(val) if 'moving_variance' in var.name: tf_export_params['bn_running_var'] = _np.array(val) for layer_name in tf_export_params.keys(): tf_export_params[layer_name] = _np.ascontiguousarray( tf_export_params[layer_name]) return tf_export_params
def __init__(self, n_actions, n_features, learning_rate=0.001, suffix="", safety_requirement=0.1, params=params, idx=None): self.n_actions = n_actions self.n_features = n_features self.lr = learning_rate print("learning rate: {}".format(self.lr)) self.pg_lr = 0.01 self.suffix = suffix self.safety_requirement = safety_requirement """ self.ep_obs, self.ep_as, self.ep_rs: observation, action, reward, recorded for each batch """ self.ep_obs, self.ep_as, self.ep_rs, self.ep_ss = [], [], [], [] """ self.tput_batch: record throughput for each batch, used to show progress while training self.tput_persisit, self.episode: persist to record throughput, used to be stored and plot later """ self.tput_batch, self.tput_persisit, self.safe_batch, self.safe_persisit = [], [], [], [] self.coex_persisit, self.sum_persisit = [], [] self.node_used_persisit = [] self.episode = [] self.node_used = [] self.ss_perapp_persisit = [] self.ss_coex_persisit = [] self.ss_sum_persisit = [] self.start_cpo =False self.count = 0 # TODO self.vio = []: violation ##### PPO CHANGE ##### self.ppo_sample_inter = 3 self.ppo_sample_counter = 0 self.grad_squared = 0 self.params = params self.clip_eps = params['clip_eps'] if idx == None: np.random.seed(1) tf.set_random_seed(1) else: np.random.seed(idx) tf.set_random_seed(idx) ##### PPO CHANGE ##### self._build_net() self.sess = tf.Session() self.sess.run(tf.global_variables_initializer()) restore = ['Actor' + self.suffix + '/fc1' + self.suffix + '/kernel:0', 'Actor' + self.suffix + '/fc1' + self.suffix + '/bias:0', 'Actor' + self.suffix + '/fc2' + self.suffix + '/kernel:0', 'Actor' + self.suffix + '/fc2' + self.suffix + '/bias:0'] restore_var = [v for v in tf.all_variables() if v.name in restore] self.saver = tf.train.Saver(var_list=restore_var)
def begin(self): # TODO(karishmamalkan): Add logging for when variables are loaded variable_references = {var.name: var for var in tf.all_variables()} variable_mappings = {} vars_in_chk = tf.train.list_variables(self.checkpoint_path) for (var, _) in vars_in_chk: variable_mappings[var] = variable_references[ var.replace(self.chk_scopename, self.graph_scopename) + ":0"] tf.train.init_from_checkpoint(self.checkpoint_path, variable_mappings)
def get_train_op(self, total_loss, global_step, learning_rate): """ Return train_op """ optimizer = tf.train.AdamOptimizer(learning_rate) train_op = optimizer.minimize(total_loss, global_step=global_step) adam_vars = [var for var in tf.all_variables() if 'Adam' in var.name] reset_optimizer_op = tf.variables_initializer(adam_vars) return train_op, reset_optimizer_op
def print_vars(all_vars=None, label="Variables"): """Print info about a list of variables.""" if not all_vars: all_vars = tf.all_variables() num_params = _count_total_params(all_vars) tf.logging.info("# %s, num_params=%d" % (label, num_params)) tf.logging.info("Format: <name>, <shape>, <(soft) device placement>") for var in all_vars: tf.logging.info(" %s, %s, %s" % (var.name, str(var.get_shape()), var.op.device))
def get_load_vars(): """Returns a list of variables to load from the checkpoint.""" tf.train.get_or_create_global_step() load_vars = [] for v in tf.all_variables(): if v.name.startswith('augmentation'): continue if 'train_images' in v.name or 'train_labels' in v.name: continue load_vars.append(v) return load_vars
def train(argv=None): """Train RemasteredPilotNet model with given hyper parameters""" # TODO : log directory name according hyper parameters (epoch, batch size, cut layer index of PilotNet model) logDir = FLAGS.log_directory # delete old logs if FLAGS.clear_log: if tf.gfile.Exists(logDir): tf.gfile.DeleteRecursively(logDir) tf.gfile.MakeDirs(logDir) with tf.Graph().as_default(): # Construct a PilotNet model pilotNetModel = PilotNet() # Load weights onto the PilotNet instance with tf.Session() as sess: # saver = tf.train.import_meta_graph(FLAGS.model_file + '.meta') # saver.restore(sess, FLAGS.model_file) saver = tf.train.import_meta_graph( "/home/smaniott/Bureau/test/model/model.ckpt.meta") saver.restore(sess, "/home/smaniott/Bureau/test/model/model.ckpt") # images of the road ahead and steering angles in random order # dataset = NuSceneDB(FLAGS.dataset_dir) # Build RemasteredPilotNet model instance based on previously trained (loaded weights) PilotNet model cutLayerIndex = 10 model = RemasteredPilotNet() ''' 采用随机梯度下降法进行训练(自适应估计方法, Adaptive Moment Estimation, Adam):找到最好的权重值和偏差,以最小化输出误差 1.计算梯度 compute_gradients(loss, <list of variables>) 2.运用梯度 apply_gradients(<list of variables>) ''' train_vars = tf.trainable_variables() # define loss loss = tf.reduce_mean(tf.square(tf.subtract(model.y_, model.steering))) \ + tf.add_n([tf.nn.l2_loss(v) for v in train_vars]) * FLAGS.L2NormConst optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize(loss) ''' TensorFlow's V1 checkpoint format has been deprecated. Consider switching to the more efficient V2 format, now on by default. ''' # saver = tf.train.Saver(write_version=saver_pb2.SaverDef.V1) saver = tf.train.Saver(tf.all_variables()) # create a summary to monitor cost tensor tf.summary.scalar("loss", loss) # merge all summaries into a single op merged_summary_op = tf.summary.merge_all() init = tf.initialize_all_variables() """
def testRateOverrides(self, mocked_parametrized): mocked_parametrized.side_effect = ( lambda x, rate: scale_gradient.scale_gradient(x, -rate)) rate = 7.3 cons = optimization_constraints.OptimizationConstraints() lhs = tf.zeros_like(1.0) rhs = tf.ones_like(1.0) x = cons.add(lhs < rhs, rate=rate)() v = tf.all_variables()[0] dxdl = tf.gradients(x, v) with tf.train.MonitoredSession() as sess: grads = sess.run(dxdl) self.assertAllClose(grads[0], rate)
def test_saccader_pretrain_loss(self): batch_size = 2 num_classes = 1001 images = tf.random_uniform( shape=(batch_size, 224, 224, 3), minval=-1, maxval=1, dtype=tf.float32) config = saccader_config.get_config() config.num_classes = num_classes model = saccader.Saccader(config) model( images, num_times=6, is_training=True, policy="learned") num_params = len(tf.all_variables()) pretrain_loss = losses.saccader_pretraining_loss( model, images, is_training=True) # Test loss does not introduce new variables. self.assertLen(tf.all_variables(), num_params) # Gradients with respect to location variables should exist. for v, g in zip(model.var_list_location, tf.gradients( pretrain_loss, model.var_list_location)): if v.trainable: self.assertIsNotNone(g) # Gradients with respect to classification variables should be None. for g in tf.gradients( pretrain_loss, model.var_list_classification): self.assertIsNone(g) # Test evaluating the loss self.evaluate(tf.global_variables_initializer()) self.evaluate(pretrain_loss)
def initialize_uninitialized(sess): # global_vars = tf.global_variables() # is_not_initialized = sess.run([tf.is_variable_initialized(var) for var in global_vars]) # not_initialized_vars = [v for (v, f) in zip(global_vars, is_not_initialized) if not f] # if len(not_initialized_vars): # sess.run(tf.variables_initializer(not_initialized_vars)) uninit_vars = [] for var in tf.all_variables(): try: sess.run(var) except tf.errors.FailedPreconditionError: uninit_vars.append(var) init_new_vars_op = tf.initialize_variables(uninit_vars) sess.run(init_new_vars_op)
def evaluate(self): config = tf.ConfigProto(allow_soft_placement=True) with tf.Session(config=config) as sess: with tf.device("/gpu:%d" % cfg.GPU_ID): if self.model_path.find('.ckpt') != -1: self.init_opt() print("Reading model parameters from %s" % self.model_path) saver = tf.train.Saver(tf.all_variables()) saver.restore(sess, self.model_path) # self.eval_one_dataset(sess, self.dataset.train, # self.log_dir, subset='train') self.eval_one_dataset(sess, self.dataset.test, self.log_dir, subset='test') else: print("Input a valid model path.")
def testBatchNormIsTraining(self, is_training): feature_dims = (128, 128) inputs = tf.random.uniform((3, 4), dtype=tf.float64, seed=1) projection_head = projection_head_lib.ProjectionHead( feature_dims=feature_dims, use_batch_norm=True) outputs = projection_head(inputs, training=is_training) statistics_vars = [ var for var in tf.all_variables() if 'moving_' in var.name ] self.assertLen(statistics_vars, 2) grads = tf.gradients(outputs, statistics_vars) self.assertLen(grads, 2) if is_training: self.assertAllEqual([None, None], grads) self.assertTrue(tf.get_collection(tf.GraphKeys.UPDATE_OPS)) else: self.assertNotIn(None, grads)
def variables_to_restore(self, moving_avg_variables=None): """ """ name_map = {} if moving_avg_variables is None: moving_avg_variables = tf.trainable_variables() moving_avg_variables += tf.moving_average_variables() # Remove duplicates moving_avg_variables = set(moving_avg_variables) # Collect all the variables with moving average, for v in moving_avg_variables: name_map[self.average_name(v)] = v # Make sure we restore variables without moving average as well. for v in list(set(tf.all_variables()) - moving_avg_variables): if v.op.name not in name_map: name_map[v.op.name] = v return name_map
def main(_): tf.logging.set_verbosity(tf.logging.INFO) reader = tf.train.load_checkpoint(FLAGS.src_ckpt) var_values, var_dtypes = {}, {} for (name, _) in tf.train.list_variables(FLAGS.src_ckpt): # skip global_step and optimizer states in src ckpt if not FLAGS.retain_all if FLAGS.finetune_only and (name.startswith("global_step") or "adam" in name.lower() or "generator" in name.lower()): continue tensor, tgt_name = get_tensor(reader, name) var_values[tgt_name] = tensor var_dtypes[tgt_name] = tensor.dtype if FLAGS.tgt_ckpt: if not tf.io.gfile.exists(os.path.dirname(FLAGS.tgt_ckpt)): tf.io.gfile.makedirs(os.path.dirname(FLAGS.tgt_ckpt)) with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE): tf_vars = [ tf.get_variable(v, shape=var_values[v].shape, dtype=var_dtypes[v]) for v in var_values ] placeholders = [ tf.placeholder(v.dtype, shape=v.shape) for v in tf_vars ] assign_ops = [tf.assign(v, p) for (v, p) in zip(tf_vars, placeholders)] group_assign_op = tf.group(assign_ops) saver = tf.train.Saver(tf.all_variables()) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) # Run grouped assign op feed_dict = {} for p, (name, value) in zip(placeholders, var_values.items()): feed_dict[p] = value sess.run(group_assign_op, feed_dict) # Use the built saver to save the averaged checkpoint. saver.save(sess, FLAGS.tgt_ckpt) tf.logging.info("Modified checkpoint saved to %s", FLAGS.tgt_ckpt)
def __init__(self, model, *args, **kwargs): """""" if args: if len(args) > 1: raise TypeError('Parser takes at most one argument') kwargs['name'] = kwargs.pop('name', model.__name__) super(Network, self).__init__(*args, **kwargs) if not os.path.isdir(self.save_dir): os.mkdir(self.save_dir) with open(os.path.join(self.save_dir, 'config.cfg'), 'w') as f: self._config.write(f) self._global_step = tf.Variable(0., trainable=False) self._global_epoch = tf.Variable(0., trainable=False) self._model = model(self._config, global_step=self.global_step) self._vocabs = [] vocab_files = [(self.word_file, 1, 'Words'), (self.tag_file, [3, 4], 'Tags'), (self.rel_file, 7, 'Rels')] for i, (vocab_file, index, name) in enumerate(vocab_files): vocab = Vocab(vocab_file, index, self._config, name=name, cased=self.cased if not i else True, use_pretrained=(not i), global_step=self.global_step) self._vocabs.append(vocab) self._trainset = Dataset(self.train_file, self._vocabs, model, self._config, name='Trainset') self._validset = Dataset(self.valid_file, self._vocabs, model, self._config, name='Validset') self._testset = Dataset(self.test_file, self._vocabs, model, self._config, name='Testset') self._ops = self._gen_ops() self._save_vars = filter(lambda x: u'Pretrained' not in x.name, tf.all_variables()) self.history = { 'train_loss': [], 'train_accuracy': [], 'valid_loss': [], 'valid_accuracy': [], 'test_acuracy': 0 } return
def reload_checkpoint(self, checkpoint_path, use_legacy_checkpoint=False): if use_legacy_checkpoint: variables_to_restore = atari_lib.maybe_transform_variable_names( tf.all_variables(), legacy_checkpoint_load=True) else: global_vars = set([x.name for x in tf.global_variables()]) ckpt_vars = [ '{}:0'.format(name) for name, _ in tf.train.list_variables(checkpoint_path) ] include_vars = list(global_vars.intersection(set(ckpt_vars))) variables_to_restore = contrib_slim.get_variables_to_restore( include=include_vars) if variables_to_restore: reloader = tf.train.Saver(var_list=variables_to_restore) reloader.restore(self._sess, checkpoint_path) tf.logging.info('Done restoring from %s', checkpoint_path) else: tf.logging.info('Nothing to restore!')
def build_model(self, sess): self.init_opt() sess.run(tf.initialize_all_variables()) if len(self.model_path) > 0: print("Reading model parameters from %s" % self.model_path) restore_vars = tf.all_variables() # all_vars = tf.all_variables() # restore_vars = [var for var in all_vars if # var.name.startswith('g_') or # var.name.startswith('d_')] saver = tf.train.Saver(restore_vars) saver.restore(sess, self.model_path) istart = self.model_path.rfind('_') + 1 iend = self.model_path.rfind('.') counter = self.model_path[istart:iend] counter = int(counter) else: print("Created model with fresh parameters.") counter = 0 return counter
def testGetAllLocalVariables(self, get_non_trainable_variables): def local_custom_getter(getter, *args, **kwargs): kwargs["trainable"] = False if "collections" in kwargs and kwargs["collections"] is not None: kwargs["collections"] += [tf.GraphKeys.LOCAL_VARIABLES] else: kwargs["collections"] = [tf.GraphKeys.LOCAL_VARIABLES] return getter(*args, **kwargs) inputs = tf.ones(dtype=tf.float32, shape=[10, 10]) # Create a new ModuleWithSubmodules that uses all local variables with tf.variable_scope("", custom_getter=local_custom_getter): submodule_a = SimpleModule(name="simple_submodule") submodule_b = ComplexModule(name="complex_submodule") local_module = ModuleWithSubmodules( submodule_a=submodule_a, submodule_b=submodule_b) local_module(inputs) # pylint: disable=not-callable self.assertEmpty(local_module.get_all_variables()) self.assertEmpty(tf.all_variables()) self.assertLen(tf.local_variables(), 12) all_variables = get_non_trainable_variables(local_module) all_variable_names = sorted([str(v.name) for v in all_variables]) self.assertEqual([ "complex_submodule/linear_1/b:0", "complex_submodule/linear_1/w:0", "complex_submodule/linear_2/b:0", "complex_submodule/linear_2/w:0", "module_with_submodules/complex_build/linear_1/b:0", "module_with_submodules/complex_build/linear_1/w:0", "module_with_submodules/complex_build/linear_2/b:0", "module_with_submodules/complex_build/linear_2/w:0", "module_with_submodules/simple_build/b:0", "module_with_submodules/simple_build/w:0", "simple_submodule/b:0", "simple_submodule/w:0", ], all_variable_names)
parameters = 10 else: parameters = 0 log("Trainable parameters:") log([v.name for v in train_params[parameters:]]) # Training hyper parameters train_op = tf.train.AdamOptimizer(learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-08, use_locking=False).minimize( cost, var_list=train_params[parameters:]) uninitialized_vars = [] for var in tf.all_variables(): try: sess.run(var) except tf.errors.FailedPreconditionError: uninitialized_vars.append(var) init_new_vars_op = tf.initialize_variables(uninitialized_vars) sess.run(init_new_vars_op) log('TensorFlow Session starting...') # TensorBoard summary (graph) tf.summary.scalar('cost', cost) merged_summary = tf.summary.merge_all() writer = tf.summary.FileWriter('./tensorboard_test') writer.add_graph(sess.graph)
def __init__(self, sess, model, batch_size=1, confidence=CONFIDENCE, targeted=False, learning_rate=LEARNING_RATE, binary_search_steps=BINARY_SEARCH_STEPS, max_iterations=MAX_ITERATIONS, abort_early=ABORT_EARLY, initial_const=INITIAL_CONST, boxmin=0, boxmax=1, epsilon=0.3): image_size, num_channels, num_labels = model.image_size, model.num_channels, model.num_labels self.sess = sess self.TARGETED = targeted self.LEARNING_RATE = learning_rate self.MAX_ITERATIONS = max_iterations self.BINARY_SEARCH_STEPS = binary_search_steps self.ABORT_EARLY = abort_early self.CONFIDENCE = confidence self.initial_const = initial_const self.batch_size = batch_size self.repeat = binary_search_steps >= 10 self.I_KNOW_WHAT_I_AM_DOING_AND_WANT_TO_OVERRIDE_THE_PRESOFTMAX_CHECK = False shape = (batch_size, image_size, image_size, num_channels) self.uninitialized_vars = [] for var in tf.all_variables(): try: self.sess.run(var) except tf.errors.FailedPreconditionError: self.uninitialized_vars.append(var) # these are variables to be more efficient in sending data to tf self.timg = tf.Variable(np.zeros(shape), dtype=tf.float32) self.tlab = tf.Variable(np.zeros((batch_size, num_labels)), dtype=tf.float32) self.const = tf.Variable(np.ones(batch_size), dtype=tf.float32) # and here's what we use to assign them self.assign_timg = tf.placeholder(tf.float32, shape) self.assign_tlab = tf.placeholder(tf.float32, (batch_size, num_labels)) self.assign_const = tf.placeholder(tf.float32, [batch_size]) # the variable we're going to optimize over #modifier = tf.Variable(np.zeros(shape,dtype=np.float32),name='modifier') modifier = tf.Variable(np.random.uniform(-epsilon, epsilon, shape).astype('float32'), name='modifier') self.modifier = tf.get_variable( 'modifier', shape, trainable=True, constraint=lambda x: tf.clip_by_value(x, -epsilon, epsilon)) # the resulting image, tanh'd to keep bounded from boxmin to boxmax self.boxmul = (boxmax - boxmin) / 2. self.boxplus = (boxmin + boxmax) / 2. self.newimg = tf.clip_by_value(self.modifier + self.timg, 0, 1) ''' matrix = tf.random_normal([500,image_size*image_size*num_channels,128],0.,1.0/tf.sqrt(128.)) self.x_batch = standardized( tf.keras.backend.dot(tf.reshape(self.timg,[image_size*image_size*num_channels]),matrix)) self.y_batch = standardized( tf.keras.backend.dot(tf.reshape(self.newimg,[image_size*image_size*num_channels]),matrix)) ''' tconv = tf.transpose(model.conv1(self.timg), perm=[3, 1, 2, 0]) nconv = tf.transpose(model.conv1(self.newimg), perm=[3, 1, 2, 0]) T_xy, T_x_y = MiNetwork(tconv, nconv) #T_xy , T_x_y = MiNetwork(self.x_batch,self.y_batch) self.MI = tf.reduce_mean(T_xy, axis=0) - tf.log( tf.reduce_mean(tf.exp(T_x_y))) real = model.predict(self.timg, self.tlab) self.real = real #fake = tf.reduce_max(tf.abs(self.timg - self.recon)) fake = model.predict(self.newimg, self.tlab) self.fake = fake self.loss1 = tf.maximum(0.0, fake - real) # sum up the losses self.loss1_1 = tf.reduce_sum(self.const * self.loss1) #self.loss = self.loss1_1 - self.MI self.loss = self.loss1_1 + self.MI # Setup the adam optimizer and keep track of variables we're creating start_vars = set(x.name for x in tf.global_variables()) m_var = [var for var in tf.global_variables() if 'M_' in var.name] #optimizer = tf.train.AdamOptimizer(self.LEARNING_RATE) optimizer = tf.train.GradientDescentOptimizer(self.LEARNING_RATE) self.mi_train = tf.train.AdamOptimizer(0.000015).minimize( -self.MI, var_list=m_var) self.train = optimizer.minimize(self.loss, var_list=[self.modifier]) end_vars = tf.global_variables() new_vars = [x for x in end_vars if x.name not in start_vars] # these are the variables to initialize when we run self.setup = [] self.lamda = [] self.setup.append(self.timg.assign(self.assign_timg)) self.setup.append(self.tlab.assign(self.assign_tlab)) self.lamda.append(self.const.assign(self.assign_const)) self.init = tf.variables_initializer(var_list=[self.modifier] + new_vars) self.mi_init = tf.variables_initializer(var_list=m_var)
def train(self): config = tf.ConfigProto(allow_soft_placement=True) with tf.Session(config=config) as sess: with tf.device("/gpu:%d" % cfg.GPU_ID): counter = self.build_model(sess) saver = tf.train.Saver(tf.all_variables(), keep_checkpoint_every_n_hours=2) # summary_op = tf.merge_all_summaries() summary_writer = tf.train.SummaryWriter( self.log_dir, sess.graph) keys = ["d_loss", "g_loss"] log_vars = [] log_keys = [] for k, v in self.log_vars: if k in keys: log_vars.append(v) log_keys.append(k) # print(k, v) generator_lr = cfg.TRAIN.GENERATOR_LR discriminator_lr = cfg.TRAIN.DISCRIMINATOR_LR num_embedding = cfg.TRAIN.NUM_EMBEDDING lr_decay_step = cfg.TRAIN.LR_DECAY_EPOCH number_example = self.dataset.train._num_examples updates_per_epoch = int(number_example / self.batch_size) epoch_start = int(counter / updates_per_epoch) for epoch in range(epoch_start, self.max_epoch): widgets = [ "epoch #%d|" % epoch, Percentage(), Bar(), ETA() ] pbar = ProgressBar(maxval=updates_per_epoch, widgets=widgets) pbar.start() if epoch % lr_decay_step == 0 and epoch != 0: generator_lr *= 0.5 discriminator_lr *= 0.5 all_log_vals = [] for i in range(updates_per_epoch): pbar.update(i) # training d images, wrong_images, embeddings, _, _ =\ self.dataset.train.next_batch(self.batch_size, num_embedding) feed_dict = { self.images: images, self.wrong_images: wrong_images, self.embeddings: embeddings, self.generator_lr: generator_lr, self.discriminator_lr: discriminator_lr } # train d feed_out = [ self.discriminator_trainer, self.d_sum, self.hist_sum, log_vars ] _, d_sum, hist_sum, log_vals = sess.run( feed_out, feed_dict) summary_writer.add_summary(d_sum, counter) summary_writer.add_summary(hist_sum, counter) all_log_vals.append(log_vals) # train g feed_out = [self.generator_trainer, self.g_sum] _, g_sum = sess.run(feed_out, feed_dict) summary_writer.add_summary(g_sum, counter) # save checkpoint counter += 1 if counter % self.snapshot_interval == 0: snapshot_path = "%s/%s_%s.ckpt" %\ (self.checkpoint_dir, self.exp_name, str(counter)) fn = saver.save(sess, snapshot_path) print("Model saved in file: %s" % fn) img_sum = self.epoch_sum_images(sess, cfg.TRAIN.NUM_COPY) summary_writer.add_summary(img_sum, counter) avg_log_vals = np.mean(np.array(all_log_vals), axis=0) dic_logs = {} for k, v in zip(log_keys, avg_log_vals): dic_logs[k] = v # print(k, v) log_line = "; ".join("%s: %s" % (str(k), str(dic_logs[k])) for k in dic_logs) print("Epoch %d | " % (epoch) + log_line) sys.stdout.flush() if np.any(np.isnan(avg_log_vals)): raise ValueError("NaN detected!")