def test_reuse_model(self, avg_pool, is_training):
batch_size = 2
num_classes = 10
images = _construct_images(batch_size)
config = bagnet_config.get_config()
config.num_classes = num_classes
config.avg_pool = avg_pool
model = bagnet_model.BagNet(config)
# Build once.
logits1, _ = model(images, is_training)
num_params = len(tf.all_variables())
l2_loss1 = tf.losses.get_regularization_loss()
# Build twice.
logits2, _ = model(
images, is_training)
l2_loss2 = tf.losses.get_regularization_loss()
# Ensure variables are reused.
self.assertLen(tf.all_variables(), num_params)
init = tf.global_variables_initializer()
self.evaluate(init)
logits1, logits2 = self.evaluate((logits1, logits2))
l2_loss1, l2_loss2 = self.evaluate((l2_loss1, l2_loss2))
np.testing.assert_almost_equal(logits1, logits2, decimal=9)
np.testing.assert_almost_equal(l2_loss1, l2_loss2, decimal=9)
def test_reuse_model(self, policy, is_training):
config = saccader_config.get_config()
num_times = 2
image_shape = (224, 224, 3)
num_classes = 10
config.num_classes = num_classes
config.num_times = num_times
batch_size = 3
images1 = tf.constant(np.random.rand(*((batch_size, ) + image_shape)),
dtype=tf.float32)
model = saccader.Saccader(config)
logits1 = model(images1,
num_times=num_times,
is_training=is_training,
policy=policy)[0]
num_params = len(tf.all_variables())
l2_loss1 = tf.losses.get_regularization_loss()
# Build twice with different num_times and different batch size.
images2 = tf.constant(np.random.rand(*((batch_size - 1, ) +
image_shape)),
dtype=tf.float32)
logits2 = model(images2,
num_times=num_times + 1,
is_training=is_training,
policy=policy)[0]
l2_loss2 = tf.losses.get_regularization_loss()
# Ensure variables are reused.
self.assertLen(tf.all_variables(), num_params)
init = tf.global_variables_initializer()
self.evaluate(init)
logits1, logits2 = self.evaluate((logits1, logits2))
l2_loss1, l2_loss2 = self.evaluate((l2_loss1, l2_loss2))
np.testing.assert_almost_equal(l2_loss1, l2_loss2, decimal=9)
def test_reuse_model(self, is_training):
batch_size = 5
images = _construct_images(batch_size)
config = mnist_config.ConfigDict()
model = mnist_model.MNISTNetwork(config)
# Build once.
logits1, _ = model(images, is_training)
num_params = len(tf.all_variables())
l2_loss1 = tf.losses.get_regularization_loss()
# Build twice.
logits2, _ = model(images, is_training)
l2_loss2 = tf.losses.get_regularization_loss()
# Ensure variables are reused.
self.assertLen(tf.all_variables(), num_params)
init = tf.global_variables_initializer()
with self.test_session() as sess:
sess.run(init)
# Ensure operations are the same after reuse.
err_logits = (np.abs(sess.run(logits1 - logits2))).sum()
self.assertAlmostEqual(err_logits, 0, 9)
err_losses = (np.abs(sess.run(l2_loss1 - l2_loss2))).sum()
self.assertAlmostEqual(err_losses, 0, 9)
def test_reuse_model(self, rank, is_training, special_type):
batch_size = 5
num_classes = 10
num_channels = 3
images = _construct_images(batch_size)
config = simple_model_config.get_config()
config.num_classes = num_classes
config.num_channels = num_channels
config.batch_norm = False
config.kernel_size_list = [3, 3, 3]
config.num_filters_list = [64, 64, 64]
config.strides_list = [2, 2, 1]
config.layer_types = ['conv2d', special_type, 'conv2d']
config.rank = rank
model = simple_model.SimpleNetwork(config)
# Build once.
logits1, _ = model(images, is_training)
num_params = len(tf.all_variables())
# Build twice.
logits2, _ = model(images, is_training)
# Ensure variables are reused.
self.assertLen(tf.all_variables(), num_params)
init = tf.global_variables_initializer()
with self.test_session() as sess:
sess.run(init)
# Ensure operations are the same after reuse.
err_logits = (np.abs(sess.run(logits1 - logits2))).sum()
self.assertAlmostEqual(err_logits, 0, 9)
def train_intent():
#config=tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))
with tf.Session() as sess:
labels_train, labels_test, data_filter_train, data_filter_test = get_test_data(
)
accuracy, x, y_target, keep_prob, train_step, y, cross_entropy, W_conv1 = create_graph(
train=True)
path = pathConfigs.model_path
if not os.path.exists(path):
os.makedirs(path)
num_ckpt = 0
# 이미 check point가 있으면 있는거 사용
if len(os.listdir(path)) > 3:
print('LOAD CNN MODEL')
dir = os.listdir(path)
for i in dir:
try:
new_one = int(i.split('-')[1].split('.')[0])
if num_ckpt < new_one:
num_ckpt = new_one
except:
pass
restorer = tf.train.Saver(tf.all_variables())
restorer.restore(
sess, pathConfigs.model_path + 'check_point-' + str(num_ckpt) +
'.ckpt')
else: # 학습된거 없으면 새로 만듦
sess.run(tf.global_variables_initializer())
for i in range(learning_step):
sess.run(train_step,
feed_dict={
x: data_filter_train,
y_target: labels_train,
keep_prob: 0.5
})
index = i + num_ckpt
if i % 100 == 0:
train_accuracy = sess.run(accuracy,
feed_dict={
x: data_filter_train,
y_target: labels_train,
keep_prob: 1
})
print("step %d, training accuracy: %.3f" %
(index, train_accuracy))
if i != 0 and i % 100 == 0:
saver = tf.train.Saver(tf.all_variables())
saver.save(sess, path + 'check_point-' + str(index) + '.ckpt')
print("Save Models checkpoint : %d" % index)
print("Current TIME: ", datetime.now())
def restore_best_model():
"""Load bestmodel file from eval directory, add variables for adagrad, and save to train directory"""
tf.logging.info("Restoring bestmodel for training...")
# Initialize all vars in the model
sess = tf.Session(config=util.get_config())
print("Initializing all variables...")
sess.run(tf.initialize_all_variables())
# Restore the best model from eval dir
saver = tf.train.Saver(
[v for v in tf.all_variables() if "Adagrad" not in v.name])
print("Restoring all non-adagrad variables from best model in eval dir...")
curr_ckpt = util.load_ckpt(saver, sess, "eval")
print("Restored %s." % curr_ckpt)
# Save this model to train dir and quit
new_model_name = curr_ckpt.split("/")[-1].replace("bestmodel", "model")
new_fname = os.path.join(FLAGS.log_root, "train", new_model_name)
print("Saving model to %s..." % (new_fname))
new_saver = tf.train.Saver(
) # this saver saves all variables that now exist, including Adagrad variables
new_saver.save(sess, new_fname)
print("Saved.")
exit()
def __init__(self,
sess,
ac_dim,
ob_dim,
n_layers,
size,
learning_rate=1e-4,
training=True,
policy_scope='policy_vars',
discrete=False, # unused for now
nn_baseline=False, # unused for now
**kwargs):
super().__init__(**kwargs)
# init vars
self.sess = sess
self.ac_dim = ac_dim
self.ob_dim = ob_dim
self.n_layers = n_layers
self.size = size
self.discrete = discrete
self.learning_rate = learning_rate
self.training = training
self.nn_baseline = nn_baseline
# build TF graph
with tf.variable_scope(policy_scope, reuse=tf.AUTO_REUSE):
self.build_graph()
# saver for policy variables that are not related to training
self.policy_vars = [v for v in tf.all_variables() if policy_scope in v.name and 'train' not in v.name]
self.policy_saver = tf.train.Saver(self.policy_vars, max_to_keep=None)
def create_vggish_frozen_graph():
"""Create the VGGish frozen graph."""
os.system('git clone https://github.com/tensorflow/models.git')
sys.path.append('models/research/audioset/vggish/')
import vggish_slim
os.system(
'curl -O https://storage.googleapis.com/audioset/vggish_model.ckpt')
ckpt_path = 'vggish_model.ckpt'
with tf.Graph().as_default(), tf.Session() as sess:
vggish_slim.define_vggish_slim(training=False)
vggish_slim.load_vggish_slim_checkpoint(sess, ckpt_path)
saver = tf.train.Saver(tf.all_variables())
freeze_graph.freeze_graph_with_def_protos(
sess.graph_def,
saver.as_saver_def(),
ckpt_path,
'vggish/fc2/BiasAdd',
restore_op_name=None,
filename_tensor_name=None,
output_graph='/tmp/mediapipe/vggish_new.pb',
clear_devices=True,
initializer_nodes=None)
os.system('rm -rf models/')
os.system('rm %s' % ckpt_path)
def __init__(self, n_actions, n_features, learning_rate, suffix):
self.n_actions = n_actions
self.n_features = n_features
self.lr = learning_rate
self.suffix = suffix
"""
self.ep_obs, self.ep_as, self.ep_rs: observation, action, reward, recorded for each batch
"""
self.ep_obs, self.ep_as, self.ep_rs, self.ep_ss = [], [], [], []
"""
self.tput_batch: record throughput for each batch, used to show progress while training
self.tput_persisit, self.episode: persist to record throughput, used to be stored and plot later
"""
self.tput_batch = []
self.tput_persisit = []
self.ss_batch = []
self.ss_persisit = []
self.episode = []
self.ss_perapp_persisit = []
self.ss_coex_persisit = []
self.ss_sum_persisit = []
# TODO self.vio = []: violation
self._build_net()
self.sess = tf.Session()
self.sess.run(tf.global_variables_initializer())
restore = [
'Actor' + self.suffix + '/fc1' + self.suffix + '/kernel:0',
'Actor' + self.suffix + '/fc1' + self.suffix + '/bias:0',
'Actor' + self.suffix + '/fc2' + self.suffix + '/kernel:0',
'Actor' + self.suffix + '/fc2' + self.suffix + '/bias:0'
]
restore_var = [v for v in tf.all_variables() if v.name in restore]
self.saver = tf.train.Saver(var_list=restore_var)
def export_weights(self):
"""
Function to store TensorFlow weights back to into a dict in CoreML format to be used
by the C++ implementation
Returns
-------
tf_export_params: Dictionary
Dictionary of weights from TensorFlow stored as {weight_name: weight_value}
"""
tf_export_params = {}
tvars = _tf.trainable_variables()
tvars_vals = self.sess.run(tvars)
for var, val in zip(tvars, tvars_vals):
if 'weight' in var.name:
if var.name.startswith('conv'):
tf_export_params[var.name.split(
':')[0]] = _utils.convert_conv1d_tf_to_coreml(val)
elif var.name.startswith('dense'):
tf_export_params[var.name.split(
':')[0]] = _utils.convert_dense_tf_to_coreml(val)
elif var.name.startswith('rnn/lstm_cell/kernel'):
i2h_i, i2h_c, i2h_f, i2h_o, h2h_i, h2h_c, h2h_f, h2h_o = _utils.convert_lstm_weight_tf_to_coreml(
val, CONV_H)
tf_export_params['lstm_i2h_i_weight'] = i2h_i
tf_export_params['lstm_i2h_c_weight'] = i2h_c
tf_export_params['lstm_i2h_f_weight'] = i2h_f
tf_export_params['lstm_i2h_o_weight'] = i2h_o
tf_export_params['lstm_h2h_i_weight'] = h2h_i
tf_export_params['lstm_h2h_c_weight'] = h2h_c
tf_export_params['lstm_h2h_f_weight'] = h2h_f
tf_export_params['lstm_h2h_o_weight'] = h2h_o
elif var.name.startswith('rnn/lstm_cell/bias'):
h2h_i_bias, h2h_c_bias, h2h_f_bias, h2h_o_bias = _utils.convert_lstm_bias_tf_to_coreml(
val)
tf_export_params['lstm_h2h_i_bias'] = h2h_i_bias
tf_export_params['lstm_h2h_c_bias'] = h2h_c_bias
tf_export_params['lstm_h2h_f_bias'] = h2h_f_bias
tf_export_params['lstm_h2h_o_bias'] = h2h_o_bias
elif var.name.startswith('batch_normalization'):
tf_export_params[
'bn_' + var.name.split('/')[-1][0:-2]] = _np.array(val)
else:
tf_export_params[var.name.split(':')[0]] = _np.array(val)
tvars = _tf.all_variables()
tvars_vals = self.sess.run(tvars)
for var, val in zip(tvars, tvars_vals):
if 'moving_mean' in var.name:
tf_export_params['bn_running_mean'] = _np.array(val)
if 'moving_variance' in var.name:
tf_export_params['bn_running_var'] = _np.array(val)
for layer_name in tf_export_params.keys():
tf_export_params[layer_name] = _np.ascontiguousarray(
tf_export_params[layer_name])
return tf_export_params
def __init__(self,
n_actions,
n_features,
learning_rate=0.001,
suffix="",
safety_requirement=0.1,
params=params,
idx=None):
self.n_actions = n_actions
self.n_features = n_features
self.lr = learning_rate
print("learning rate: {}".format(self.lr))
self.pg_lr = 0.01
self.suffix = suffix
self.safety_requirement = safety_requirement
"""
self.ep_obs, self.ep_as, self.ep_rs: observation, action, reward, recorded for each batch
"""
self.ep_obs, self.ep_as, self.ep_rs, self.ep_ss = [], [], [], []
"""
self.tput_batch: record throughput for each batch, used to show progress while training
self.tput_persisit, self.episode: persist to record throughput, used to be stored and plot later
"""
self.tput_batch, self.tput_persisit, self.safe_batch, self.safe_persisit = [], [], [], []
self.coex_persisit, self.sum_persisit = [], []
self.node_used_persisit = []
self.episode = []
self.node_used = []
self.ss_perapp_persisit = []
self.ss_coex_persisit = []
self.ss_sum_persisit = []
self.start_cpo =False
self.count = 0
# TODO self.vio = []: violation
##### PPO CHANGE #####
self.ppo_sample_inter = 3
self.ppo_sample_counter = 0
self.grad_squared = 0
self.params = params
self.clip_eps = params['clip_eps']
if idx == None:
np.random.seed(1)
tf.set_random_seed(1)
else:
np.random.seed(idx)
tf.set_random_seed(idx)
##### PPO CHANGE #####
self._build_net()
self.sess = tf.Session()
self.sess.run(tf.global_variables_initializer())
restore = ['Actor' + self.suffix + '/fc1' + self.suffix + '/kernel:0', 'Actor' + self.suffix + '/fc1' + self.suffix + '/bias:0', 'Actor' + self.suffix + '/fc2' + self.suffix + '/kernel:0', 'Actor' + self.suffix + '/fc2' + self.suffix + '/bias:0']
restore_var = [v for v in tf.all_variables() if v.name in restore]
self.saver = tf.train.Saver(var_list=restore_var)
def begin(self):
# TODO(karishmamalkan): Add logging for when variables are loaded
variable_references = {var.name: var for var in tf.all_variables()}
variable_mappings = {}
vars_in_chk = tf.train.list_variables(self.checkpoint_path)
for (var, _) in vars_in_chk:
variable_mappings[var] = variable_references[
var.replace(self.chk_scopename, self.graph_scopename) + ":0"]
tf.train.init_from_checkpoint(self.checkpoint_path, variable_mappings)
def get_train_op(self, total_loss, global_step, learning_rate):
"""
Return train_op
"""
optimizer = tf.train.AdamOptimizer(learning_rate)
train_op = optimizer.minimize(total_loss, global_step=global_step)
adam_vars = [var for var in tf.all_variables() if 'Adam' in var.name]
reset_optimizer_op = tf.variables_initializer(adam_vars)
return train_op, reset_optimizer_op
def print_vars(all_vars=None, label="Variables"):
"""Print info about a list of variables."""
if not all_vars:
all_vars = tf.all_variables()
num_params = _count_total_params(all_vars)
tf.logging.info("# %s, num_params=%d" % (label, num_params))
tf.logging.info("Format: <name>, <shape>, <(soft) device placement>")
for var in all_vars:
tf.logging.info(" %s, %s, %s" %
(var.name, str(var.get_shape()), var.op.device))
def get_load_vars():
"""Returns a list of variables to load from the checkpoint."""
tf.train.get_or_create_global_step()
load_vars = []
for v in tf.all_variables():
if v.name.startswith('augmentation'):
continue
if 'train_images' in v.name or 'train_labels' in v.name:
continue
load_vars.append(v)
return load_vars
def train(argv=None):
"""Train RemasteredPilotNet model with given hyper parameters"""
# TODO : log directory name according hyper parameters (epoch, batch size, cut layer index of PilotNet model)
logDir = FLAGS.log_directory
# delete old logs
if FLAGS.clear_log:
if tf.gfile.Exists(logDir):
tf.gfile.DeleteRecursively(logDir)
tf.gfile.MakeDirs(logDir)
with tf.Graph().as_default():
# Construct a PilotNet model
pilotNetModel = PilotNet()
# Load weights onto the PilotNet instance
with tf.Session() as sess:
# saver = tf.train.import_meta_graph(FLAGS.model_file + '.meta')
# saver.restore(sess, FLAGS.model_file)
saver = tf.train.import_meta_graph(
"/home/smaniott/Bureau/test/model/model.ckpt.meta")
saver.restore(sess, "/home/smaniott/Bureau/test/model/model.ckpt")
# images of the road ahead and steering angles in random order
# dataset = NuSceneDB(FLAGS.dataset_dir)
# Build RemasteredPilotNet model instance based on previously trained (loaded weights) PilotNet model
cutLayerIndex = 10
model = RemasteredPilotNet()
'''
采用随机梯度下降法进行训练(自适应估计方法, Adaptive Moment Estimation, Adam)：找到最好的权重值和偏差,以最小化输出误差
1.计算梯度 compute_gradients(loss, <list of variables>)
2.运用梯度 apply_gradients(<list of variables>)
'''
train_vars = tf.trainable_variables()
# define loss
loss = tf.reduce_mean(tf.square(tf.subtract(model.y_, model.steering))) \
+ tf.add_n([tf.nn.l2_loss(v) for v in train_vars]) * FLAGS.L2NormConst
optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize(loss)
'''
TensorFlow's V1 checkpoint format has been deprecated.
Consider switching to the more efficient V2 format, now on by default.
'''
# saver = tf.train.Saver(write_version=saver_pb2.SaverDef.V1)
saver = tf.train.Saver(tf.all_variables())
# create a summary to monitor cost tensor
tf.summary.scalar("loss", loss)
# merge all summaries into a single op
merged_summary_op = tf.summary.merge_all()
init = tf.initialize_all_variables()
"""
def testRateOverrides(self, mocked_parametrized):
mocked_parametrized.side_effect = (
lambda x, rate: scale_gradient.scale_gradient(x, -rate))
rate = 7.3
cons = optimization_constraints.OptimizationConstraints()
lhs = tf.zeros_like(1.0)
rhs = tf.ones_like(1.0)
x = cons.add(lhs < rhs, rate=rate)()
v = tf.all_variables()[0]
dxdl = tf.gradients(x, v)
with tf.train.MonitoredSession() as sess:
grads = sess.run(dxdl)
self.assertAllClose(grads[0], rate)
def test_saccader_pretrain_loss(self):
batch_size = 2
num_classes = 1001
images = tf.random_uniform(
shape=(batch_size, 224, 224, 3), minval=-1, maxval=1, dtype=tf.float32)
config = saccader_config.get_config()
config.num_classes = num_classes
model = saccader.Saccader(config)
model(
images,
num_times=6,
is_training=True,
policy="learned")
num_params = len(tf.all_variables())
pretrain_loss = losses.saccader_pretraining_loss(
model, images, is_training=True)
# Test loss does not introduce new variables.
self.assertLen(tf.all_variables(), num_params)
# Gradients with respect to location variables should exist.
for v, g in zip(model.var_list_location, tf.gradients(
pretrain_loss, model.var_list_location)):
if v.trainable:
self.assertIsNotNone(g)
# Gradients with respect to classification variables should be None.
for g in tf.gradients(
pretrain_loss, model.var_list_classification):
self.assertIsNone(g)
# Test evaluating the loss
self.evaluate(tf.global_variables_initializer())
self.evaluate(pretrain_loss)
def initialize_uninitialized(sess):
# global_vars = tf.global_variables()
# is_not_initialized = sess.run([tf.is_variable_initialized(var) for var in global_vars])
# not_initialized_vars = [v for (v, f) in zip(global_vars, is_not_initialized) if not f]
# if len(not_initialized_vars):
# sess.run(tf.variables_initializer(not_initialized_vars))
uninit_vars = []
for var in tf.all_variables():
try:
sess.run(var)
except tf.errors.FailedPreconditionError:
uninit_vars.append(var)
init_new_vars_op = tf.initialize_variables(uninit_vars)
sess.run(init_new_vars_op)
def evaluate(self):
config = tf.ConfigProto(allow_soft_placement=True)
with tf.Session(config=config) as sess:
with tf.device("/gpu:%d" % cfg.GPU_ID):
if self.model_path.find('.ckpt') != -1:
self.init_opt()
print("Reading model parameters from %s" % self.model_path)
saver = tf.train.Saver(tf.all_variables())
saver.restore(sess, self.model_path)
# self.eval_one_dataset(sess, self.dataset.train,
# self.log_dir, subset='train')
self.eval_one_dataset(sess,
self.dataset.test,
self.log_dir,
subset='test')
else:
print("Input a valid model path.")
def testBatchNormIsTraining(self, is_training):
feature_dims = (128, 128)
inputs = tf.random.uniform((3, 4), dtype=tf.float64, seed=1)
projection_head = projection_head_lib.ProjectionHead(
feature_dims=feature_dims, use_batch_norm=True)
outputs = projection_head(inputs, training=is_training)
statistics_vars = [
var for var in tf.all_variables() if 'moving_' in var.name
]
self.assertLen(statistics_vars, 2)
grads = tf.gradients(outputs, statistics_vars)
self.assertLen(grads, 2)
if is_training:
self.assertAllEqual([None, None], grads)
self.assertTrue(tf.get_collection(tf.GraphKeys.UPDATE_OPS))
else:
self.assertNotIn(None, grads)
def variables_to_restore(self, moving_avg_variables=None):
""" """
name_map = {}
if moving_avg_variables is None:
moving_avg_variables = tf.trainable_variables()
moving_avg_variables += tf.moving_average_variables()
# Remove duplicates
moving_avg_variables = set(moving_avg_variables)
# Collect all the variables with moving average,
for v in moving_avg_variables:
name_map[self.average_name(v)] = v
# Make sure we restore variables without moving average as well.
for v in list(set(tf.all_variables()) - moving_avg_variables):
if v.op.name not in name_map:
name_map[v.op.name] = v
return name_map
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
reader = tf.train.load_checkpoint(FLAGS.src_ckpt)
var_values, var_dtypes = {}, {}
for (name, _) in tf.train.list_variables(FLAGS.src_ckpt):
# skip global_step and optimizer states in src ckpt if not FLAGS.retain_all
if FLAGS.finetune_only and (name.startswith("global_step")
or "adam" in name.lower()
or "generator" in name.lower()):
continue
tensor, tgt_name = get_tensor(reader, name)
var_values[tgt_name] = tensor
var_dtypes[tgt_name] = tensor.dtype
if FLAGS.tgt_ckpt:
if not tf.io.gfile.exists(os.path.dirname(FLAGS.tgt_ckpt)):
tf.io.gfile.makedirs(os.path.dirname(FLAGS.tgt_ckpt))
with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
tf_vars = [
tf.get_variable(v,
shape=var_values[v].shape,
dtype=var_dtypes[v]) for v in var_values
]
placeholders = [
tf.placeholder(v.dtype, shape=v.shape) for v in tf_vars
]
assign_ops = [tf.assign(v, p) for (v, p) in zip(tf_vars, placeholders)]
group_assign_op = tf.group(assign_ops)
saver = tf.train.Saver(tf.all_variables())
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
# Run grouped assign op
feed_dict = {}
for p, (name, value) in zip(placeholders, var_values.items()):
feed_dict[p] = value
sess.run(group_assign_op, feed_dict)
# Use the built saver to save the averaged checkpoint.
saver.save(sess, FLAGS.tgt_ckpt)
tf.logging.info("Modified checkpoint saved to %s", FLAGS.tgt_ckpt)
def __init__(self, model, *args, **kwargs):
""""""
if args:
if len(args) > 1:
raise TypeError('Parser takes at most one argument')
kwargs['name'] = kwargs.pop('name', model.__name__)
super(Network, self).__init__(*args, **kwargs)
if not os.path.isdir(self.save_dir):
os.mkdir(self.save_dir)
with open(os.path.join(self.save_dir, 'config.cfg'), 'w') as f:
self._config.write(f)
self._global_step = tf.Variable(0., trainable=False)
self._global_epoch = tf.Variable(0., trainable=False)
self._model = model(self._config, global_step=self.global_step)
self._vocabs = []
vocab_files = [(self.word_file, 1, 'Words'),
(self.tag_file, [3, 4], 'Tags'),
(self.rel_file, 7, 'Rels')]
for i, (vocab_file, index, name) in enumerate(vocab_files):
vocab = Vocab(vocab_file, index, self._config,
name=name,
cased=self.cased if not i else True,
use_pretrained=(not i),
global_step=self.global_step)
self._vocabs.append(vocab)
self._trainset = Dataset(self.train_file, self._vocabs, model, self._config, name='Trainset')
self._validset = Dataset(self.valid_file, self._vocabs, model, self._config, name='Validset')
self._testset = Dataset(self.test_file, self._vocabs, model, self._config, name='Testset')
self._ops = self._gen_ops()
self._save_vars = filter(lambda x: u'Pretrained' not in x.name, tf.all_variables())
self.history = {
'train_loss': [],
'train_accuracy': [],
'valid_loss': [],
'valid_accuracy': [],
'test_acuracy': 0
}
return
def reload_checkpoint(self, checkpoint_path, use_legacy_checkpoint=False):
if use_legacy_checkpoint:
variables_to_restore = atari_lib.maybe_transform_variable_names(
tf.all_variables(), legacy_checkpoint_load=True)
else:
global_vars = set([x.name for x in tf.global_variables()])
ckpt_vars = [
'{}:0'.format(name)
for name, _ in tf.train.list_variables(checkpoint_path)
]
include_vars = list(global_vars.intersection(set(ckpt_vars)))
variables_to_restore = contrib_slim.get_variables_to_restore(
include=include_vars)
if variables_to_restore:
reloader = tf.train.Saver(var_list=variables_to_restore)
reloader.restore(self._sess, checkpoint_path)
tf.logging.info('Done restoring from %s', checkpoint_path)
else:
tf.logging.info('Nothing to restore!')
def build_model(self, sess):
self.init_opt()
sess.run(tf.initialize_all_variables())
if len(self.model_path) > 0:
print("Reading model parameters from %s" % self.model_path)
restore_vars = tf.all_variables()
# all_vars = tf.all_variables()
# restore_vars = [var for var in all_vars if
# var.name.startswith('g_') or
# var.name.startswith('d_')]
saver = tf.train.Saver(restore_vars)
saver.restore(sess, self.model_path)
istart = self.model_path.rfind('_') + 1
iend = self.model_path.rfind('.')
counter = self.model_path[istart:iend]
counter = int(counter)
else:
print("Created model with fresh parameters.")
counter = 0
return counter
def testGetAllLocalVariables(self, get_non_trainable_variables):
def local_custom_getter(getter, *args, **kwargs):
kwargs["trainable"] = False
if "collections" in kwargs and kwargs["collections"] is not None:
kwargs["collections"] += [tf.GraphKeys.LOCAL_VARIABLES]
else:
kwargs["collections"] = [tf.GraphKeys.LOCAL_VARIABLES]
return getter(*args, **kwargs)
inputs = tf.ones(dtype=tf.float32, shape=[10, 10])
# Create a new ModuleWithSubmodules that uses all local variables
with tf.variable_scope("", custom_getter=local_custom_getter):
submodule_a = SimpleModule(name="simple_submodule")
submodule_b = ComplexModule(name="complex_submodule")
local_module = ModuleWithSubmodules(
submodule_a=submodule_a, submodule_b=submodule_b)
local_module(inputs) # pylint: disable=not-callable
self.assertEmpty(local_module.get_all_variables())
self.assertEmpty(tf.all_variables())
self.assertLen(tf.local_variables(), 12)
all_variables = get_non_trainable_variables(local_module)
all_variable_names = sorted([str(v.name) for v in all_variables])
self.assertEqual([
"complex_submodule/linear_1/b:0",
"complex_submodule/linear_1/w:0",
"complex_submodule/linear_2/b:0",
"complex_submodule/linear_2/w:0",
"module_with_submodules/complex_build/linear_1/b:0",
"module_with_submodules/complex_build/linear_1/w:0",
"module_with_submodules/complex_build/linear_2/b:0",
"module_with_submodules/complex_build/linear_2/w:0",
"module_with_submodules/simple_build/b:0",
"module_with_submodules/simple_build/w:0",
"simple_submodule/b:0",
"simple_submodule/w:0",
], all_variable_names)
parameters = 10
else:
parameters = 0
log("Trainable parameters:")
log([v.name for v in train_params[parameters:]])
# Training hyper parameters
train_op = tf.train.AdamOptimizer(learning_rate=0.001,
beta1=0.9,
beta2=0.999,
epsilon=1e-08,
use_locking=False).minimize(
cost, var_list=train_params[parameters:])
uninitialized_vars = []
for var in tf.all_variables():
try:
sess.run(var)
except tf.errors.FailedPreconditionError:
uninitialized_vars.append(var)
init_new_vars_op = tf.initialize_variables(uninitialized_vars)
sess.run(init_new_vars_op)
log('TensorFlow Session starting...')
# TensorBoard summary (graph)
tf.summary.scalar('cost', cost)
merged_summary = tf.summary.merge_all()
writer = tf.summary.FileWriter('./tensorboard_test')
writer.add_graph(sess.graph)
def __init__(self,
sess,
model,
batch_size=1,
confidence=CONFIDENCE,
targeted=False,
learning_rate=LEARNING_RATE,
binary_search_steps=BINARY_SEARCH_STEPS,
max_iterations=MAX_ITERATIONS,
abort_early=ABORT_EARLY,
initial_const=INITIAL_CONST,
boxmin=0,
boxmax=1,
epsilon=0.3):
image_size, num_channels, num_labels = model.image_size, model.num_channels, model.num_labels
self.sess = sess
self.TARGETED = targeted
self.LEARNING_RATE = learning_rate
self.MAX_ITERATIONS = max_iterations
self.BINARY_SEARCH_STEPS = binary_search_steps
self.ABORT_EARLY = abort_early
self.CONFIDENCE = confidence
self.initial_const = initial_const
self.batch_size = batch_size
self.repeat = binary_search_steps >= 10
self.I_KNOW_WHAT_I_AM_DOING_AND_WANT_TO_OVERRIDE_THE_PRESOFTMAX_CHECK = False
shape = (batch_size, image_size, image_size, num_channels)
self.uninitialized_vars = []
for var in tf.all_variables():
try:
self.sess.run(var)
except tf.errors.FailedPreconditionError:
self.uninitialized_vars.append(var)
# these are variables to be more efficient in sending data to tf
self.timg = tf.Variable(np.zeros(shape), dtype=tf.float32)
self.tlab = tf.Variable(np.zeros((batch_size, num_labels)),
dtype=tf.float32)
self.const = tf.Variable(np.ones(batch_size), dtype=tf.float32)
# and here's what we use to assign them
self.assign_timg = tf.placeholder(tf.float32, shape)
self.assign_tlab = tf.placeholder(tf.float32, (batch_size, num_labels))
self.assign_const = tf.placeholder(tf.float32, [batch_size])
# the variable we're going to optimize over
#modifier = tf.Variable(np.zeros(shape,dtype=np.float32),name='modifier')
modifier = tf.Variable(np.random.uniform(-epsilon, epsilon,
shape).astype('float32'),
name='modifier')
self.modifier = tf.get_variable(
'modifier',
shape,
trainable=True,
constraint=lambda x: tf.clip_by_value(x, -epsilon, epsilon))
# the resulting image, tanh'd to keep bounded from boxmin to boxmax
self.boxmul = (boxmax - boxmin) / 2.
self.boxplus = (boxmin + boxmax) / 2.
self.newimg = tf.clip_by_value(self.modifier + self.timg, 0, 1)
'''
matrix = tf.random_normal([500,image_size*image_size*num_channels,128],0.,1.0/tf.sqrt(128.))
self.x_batch = standardized( tf.keras.backend.dot(tf.reshape(self.timg,[image_size*image_size*num_channels]),matrix))
self.y_batch = standardized( tf.keras.backend.dot(tf.reshape(self.newimg,[image_size*image_size*num_channels]),matrix))
'''
tconv = tf.transpose(model.conv1(self.timg), perm=[3, 1, 2, 0])
nconv = tf.transpose(model.conv1(self.newimg), perm=[3, 1, 2, 0])
T_xy, T_x_y = MiNetwork(tconv, nconv)
#T_xy , T_x_y = MiNetwork(self.x_batch,self.y_batch)
self.MI = tf.reduce_mean(T_xy, axis=0) - tf.log(
tf.reduce_mean(tf.exp(T_x_y)))
real = model.predict(self.timg, self.tlab)
self.real = real
#fake = tf.reduce_max(tf.abs(self.timg - self.recon))
fake = model.predict(self.newimg, self.tlab)
self.fake = fake
self.loss1 = tf.maximum(0.0, fake - real)
# sum up the losses
self.loss1_1 = tf.reduce_sum(self.const * self.loss1)
#self.loss = self.loss1_1 - self.MI
self.loss = self.loss1_1 + self.MI
# Setup the adam optimizer and keep track of variables we're creating
start_vars = set(x.name for x in tf.global_variables())
m_var = [var for var in tf.global_variables() if 'M_' in var.name]
#optimizer = tf.train.AdamOptimizer(self.LEARNING_RATE)
optimizer = tf.train.GradientDescentOptimizer(self.LEARNING_RATE)
self.mi_train = tf.train.AdamOptimizer(0.000015).minimize(
-self.MI, var_list=m_var)
self.train = optimizer.minimize(self.loss, var_list=[self.modifier])
end_vars = tf.global_variables()
new_vars = [x for x in end_vars if x.name not in start_vars]
# these are the variables to initialize when we run
self.setup = []
self.lamda = []
self.setup.append(self.timg.assign(self.assign_timg))
self.setup.append(self.tlab.assign(self.assign_tlab))
self.lamda.append(self.const.assign(self.assign_const))
self.init = tf.variables_initializer(var_list=[self.modifier] +
new_vars)
self.mi_init = tf.variables_initializer(var_list=m_var)
def train(self):
config = tf.ConfigProto(allow_soft_placement=True)
with tf.Session(config=config) as sess:
with tf.device("/gpu:%d" % cfg.GPU_ID):
counter = self.build_model(sess)
saver = tf.train.Saver(tf.all_variables(),
keep_checkpoint_every_n_hours=2)
# summary_op = tf.merge_all_summaries()
summary_writer = tf.train.SummaryWriter(
self.log_dir, sess.graph)
keys = ["d_loss", "g_loss"]
log_vars = []
log_keys = []
for k, v in self.log_vars:
if k in keys:
log_vars.append(v)
log_keys.append(k)
# print(k, v)
generator_lr = cfg.TRAIN.GENERATOR_LR
discriminator_lr = cfg.TRAIN.DISCRIMINATOR_LR
num_embedding = cfg.TRAIN.NUM_EMBEDDING
lr_decay_step = cfg.TRAIN.LR_DECAY_EPOCH
number_example = self.dataset.train._num_examples
updates_per_epoch = int(number_example / self.batch_size)
epoch_start = int(counter / updates_per_epoch)
for epoch in range(epoch_start, self.max_epoch):
widgets = [
"epoch #%d|" % epoch,
Percentage(),
Bar(),
ETA()
]
pbar = ProgressBar(maxval=updates_per_epoch,
widgets=widgets)
pbar.start()
if epoch % lr_decay_step == 0 and epoch != 0:
generator_lr *= 0.5
discriminator_lr *= 0.5
all_log_vals = []
for i in range(updates_per_epoch):
pbar.update(i)
# training d
images, wrong_images, embeddings, _, _ =\
self.dataset.train.next_batch(self.batch_size,
num_embedding)
feed_dict = {
self.images: images,
self.wrong_images: wrong_images,
self.embeddings: embeddings,
self.generator_lr: generator_lr,
self.discriminator_lr: discriminator_lr
}
# train d
feed_out = [
self.discriminator_trainer, self.d_sum,
self.hist_sum, log_vars
]
_, d_sum, hist_sum, log_vals = sess.run(
feed_out, feed_dict)
summary_writer.add_summary(d_sum, counter)
summary_writer.add_summary(hist_sum, counter)
all_log_vals.append(log_vals)
# train g
feed_out = [self.generator_trainer, self.g_sum]
_, g_sum = sess.run(feed_out, feed_dict)
summary_writer.add_summary(g_sum, counter)
# save checkpoint
counter += 1
if counter % self.snapshot_interval == 0:
snapshot_path = "%s/%s_%s.ckpt" %\
(self.checkpoint_dir,
self.exp_name,
str(counter))
fn = saver.save(sess, snapshot_path)
print("Model saved in file: %s" % fn)
img_sum = self.epoch_sum_images(sess, cfg.TRAIN.NUM_COPY)
summary_writer.add_summary(img_sum, counter)
avg_log_vals = np.mean(np.array(all_log_vals), axis=0)
dic_logs = {}
for k, v in zip(log_keys, avg_log_vals):
dic_logs[k] = v
# print(k, v)
log_line = "; ".join("%s: %s" % (str(k), str(dic_logs[k]))
for k in dic_logs)
print("Epoch %d | " % (epoch) + log_line)
sys.stdout.flush()
if np.any(np.isnan(avg_log_vals)):
raise ValueError("NaN detected!")
