def __init__(self, train=False, save=False, load=False):
self.index = 0
self.input_placeholder, self.label_placeholder, self.is_training, self.lr = self.input_placeholder(
)
#self.logits = self.buildResNet(self.input_placeholder, FLAGS.num_blocks, self.is_training)
self.logits = self.infer(self.input_placeholder, FLAGS.num_blocks,
self.is_training)
self.loss, self.total_loss = self.loss(self.logits,
self.label_placeholder)
self.acc = self.accuracy_op(self.logits, self.label_placeholder)
self.train_op = self.train_op(self.lr)
#self.train_op = self.train_op()
init_op = tf.global_variables_initializer()
self.sess = tf.Session()
self.sess.run(init_op)
self.writer = self.boardWrite()
#self.load_model()
self.train_dir = 'output'
self.model_name = 'resnet'
self.params = self.get_all_params()
self.grad_total_loss_op = tf.gradients(self.total_loss, self.params)
self.grad_loss_no_reg_op = tf.gradients(self.loss, self.params)
self.v_placeholder = [
tf.placeholder(tf.float32, shape=a.get_shape())
for a in self.params
]
self.hessian_vector = hessian_vector_product(self.total_loss,
self.params,
self.v_placeholder)
self.grad_loss_wrt_input_op = tf.gradients(self.total_loss,
self.input_placeholder)
self.vec_to_list = self.get_vec_to_list_fn()
if train:
self.train()
if save:
self.saver = self.save_model()
if load:
self.load_model(load_model_dir + "99")
print("create train_grad_loss_list")
self.train_grad_loss_list = []
for i in tqdm(range(50000)):
temp = np.load(params_dir + "/train_{}.npz".format(i))['loss_val']
self.train_grad_loss_list.append(temp)
def __init__(self, **kwargs):
np.random.seed(0)
tf.set_random_seed(0)
self.batch_size = kwargs.pop('batch_size')
self.data_sets = kwargs.pop('data_sets')
self.train_dir = kwargs.pop('train_dir', 'output')
log_dir = kwargs.pop('log_dir', 'log')
self.model_name = kwargs.pop('model_name')
self.num_classes = kwargs.pop('num_classes')
self.initial_learning_rate = kwargs.pop('initial_learning_rate')
self.decay_epochs = kwargs.pop('decay_epochs')
if 'keep_probs' in kwargs: self.keep_probs = kwargs.pop('keep_probs')
else: self.keep_probs = None
if 'mini_batch' in kwargs: self.mini_batch = kwargs.pop('mini_batch')
else: self.mini_batch = True
if 'damping' in kwargs: self.damping = kwargs.pop('damping')
else: self.damping = 0.0
if not os.path.exists(self.train_dir):
os.makedirs(self.train_dir)
# Initialize session
config = tf.ConfigProto()
self.sess = tf.Session(config=config)
K.set_session(self.sess)
# Setup input
self.input_placeholder, self.labels_placeholder = self.placeholder_inputs(
)
self.num_train_examples = self.data_sets.train.labels.shape[0]
self.num_test_examples = self.data_sets.test.labels.shape[0]
# Setup inference and training
if self.keep_probs is not None:
self.keep_probs_placeholder = tf.placeholder(tf.float32, shape=(2))
self.logits = self.inference(self.input_placeholder,
self.keep_probs_placeholder)
elif hasattr(self, 'inference_needs_labels'):
self.logits = self.inference(self.input_placeholder,
self.labels_placeholder)
else:
self.logits = self.inference(self.input_placeholder)
self.total_loss, self.loss_no_reg, self.indiv_loss_no_reg = self.loss(
self.logits, self.labels_placeholder)
self.global_step = tf.Variable(0, name='global_step', trainable=False)
self.learning_rate = tf.Variable(self.initial_learning_rate,
name='learning_rate',
trainable=False)
self.learning_rate_placeholder = tf.placeholder(tf.float32)
self.update_learning_rate_op = tf.assign(
self.learning_rate, self.learning_rate_placeholder)
# self.train_op = self.get_train_op(self.total_loss, self.global_step, self.learning_rate)
self.train_op = self.get_train_sgd_op(self.total_loss,
self.global_step,
self.learning_rate)
self.accuracy_op = self.get_accuracy_op(self.logits,
self.labels_placeholder)
self.preds = self.predictions(self.logits)
# Setup misc
self.saver = tf.train.Saver()
# Setup gradients and Hessians
self.params = self.get_all_params()
self.grad_total_loss_op = tf.gradients(self.total_loss, self.params)
self.grad_loss_no_reg_op = tf.gradients(self.loss_no_reg, self.params)
self.v_placeholder = [
tf.placeholder(tf.float32, shape=a.get_shape())
for a in self.params
]
self.u_placeholder = [
tf.placeholder(tf.float32, shape=a.get_shape())
for a in self.params
]
self.hessian_vector = hessian_vector_product(self.total_loss,
self.params,
self.v_placeholder)
self.grad_loss_wrt_input_op = tf.gradients(self.total_loss,
self.input_placeholder)
# Because tf.gradients auto accumulates, we probably don't need the add_n (or even reduce_sum)
self.influence_op = tf.add_n([
tf.reduce_sum(tf.multiply(a, array_ops.stop_gradient(b)))
for a, b in zip(self.grad_total_loss_op, self.v_placeholder)
])
self.grad_influence_wrt_input_op = tf.gradients(
self.influence_op, self.input_placeholder)
self.checkpoint_file = os.path.join(self.train_dir,
"%s-checkpoint" % self.model_name)
self.all_train_feed_dict = self.fill_feed_dict_with_all_ex(
self.data_sets.train)
self.all_test_feed_dict = self.fill_feed_dict_with_all_ex(
self.data_sets.test)
init = tf.global_variables_initializer()
self.sess.run(init)
self.vec_to_list = self.get_vec_to_list_fn()
self.adversarial_loss, self.indiv_adversarial_loss = self.adversarial_loss(
self.logits, self.labels_placeholder)
if self.adversarial_loss is not None:
self.grad_adversarial_loss_op = tf.gradients(
self.adversarial_loss, self.params)
# Setup misc
saver = tf.train.Saver()
all_params = []
for layer in ['softmax_linear']:
for var_name in ['weights']:
temp_tensor = tf.get_default_graph().get_tensor_by_name("%s/%s:0" % (layer, var_name))
all_params.append(temp_tensor)
params=all_params
grad_total_loss_op = tf.gradients(total_loss, params)
grad_loss_no_reg_op =tf.gradients(loss_no_reg,params)
v_placeholder = [tf.placeholder(tf.float32, shape=a.get_shape()) for a in params]
u_placeholder = [tf.placeholder(tf.float32, shape=a.get_shape()) for a in params]
hessian_vector = hessian_vector_product(total_loss, params, v_placeholder)
grad_loss_wrt_input_op = tf.gradients(total_loss, input_placeholder)
influence_op = tf.add_n(
[tf.reduce_sum(tf.multiply(a, array_ops.stop_gradient(b))) for a, b in zip(grad_total_loss_op, v_placeholder)])
grad_influence_wrt_input_op = tf.gradients(influence_op, input_placeholder)
checkpoint_file = os.path.join(train_dir, "%s-checkpoint" % model_name)
# Here I have reduced number of datapoints learned in the train().
all_train_feed_dict = {input_placeholder: data_sets.train.x,labels_placeholder: data_sets.train.labels}
all_test_feed_dict = {input_placeholder: data_sets.test.x,labels_placeholder: data_sets.test.labels}
init = tf.global_variables_initializer()
sess.run(init)