def __init__(self, remote_vars, local_vars, *args, **kwargs):
if local_vars:
self.has_init = tf.is_variable_initialized(local_vars[0])
else:
self.has_init = tf.constant(True)
ready_for_local_init_op = tf.report_uninitialized_variables(
var_list=remote_vars)
self.init_op = tf.group(tf.initialize_variables(local_vars),
*tf.get_collection(tf.GraphKeys.LOCAL_INIT_OP))
if "scaffold" in kwargs:
# TODO(lmetz) I think this could technically be supported?
raise ValueError("Do not set scaffold on the session creator.")
scaffold = tf.train.Scaffold(
ready_for_local_init_op=ready_for_local_init_op,
ready_op=ready_for_local_init_op,
local_init_op=ready_for_local_init_op,
init_op=ready_for_local_init_op,
init_fn=self._maybe_initialize_local_vars_and_state_fn,
summary_op=tf.summary.merge([tf.summary.scalar("dummy", 0)]))
kwargs["scaffold"] = scaffold
super(WorkerSessionCreator, self).__init__(*args, **kwargs)
def __init__(self, sess, state_dim, action_dim, act_range, lr, tau_in):
# self.env_dim = inp_dim
# self.act_dim = out_dim
self.act_range = act_range
self.tau_in = tau_in
self.lr = lr
self.state_dim = state_dim
self.action_dim = action_dim
self.sess = sess
# Create network and target network
self.state_input, self.action_output, \
self.net = self.create_network(state_dim, action_dim)
self.channel_selection_net = self.net[0:6]
self.power_allocation_net = self.net[6:12]
self.pre_action_channel_input, self.pre_action_output, \
self.pre_power_net = self.create_pre_train_network(state_dim, action_dim)
self.pre_target_action_channel_input, self.pre_target_action_output, \
self.pre_target_power_update, self.pre_target_power_net = \
self.create_pre_train_target_network(state_dim, action_dim, self.pre_power_net)
self.init_pre_power_allocation = tf.initialize_variables(
self.pre_power_net)
self.target_state_input, self.target_action_output, \
self.target_update, self.target_net = self.create_target_network(state_dim, action_dim, self.net)
self.network_transfer = [
tf.assign(e, t) for e, t in zip(self.power_allocation_net,
self.pre_target_power_net)
]
print("self.net: ", self.net)
self.init_channel_selection = tf.initialize_variables(
self.channel_selection_net)
self.init_power_allocation = tf.initialize_variables(
self.power_allocation_net)
self.target_to_estimation = [
tf.assign(e, t) for e, t in zip(self.net, self.target_net)
]
self.create_training_method()
self.sess.run(tf.initialize_all_variables())
self.update_target()
def __init__(self, make_loss_and_init_fn):
"""Wraps a model in the Problem interface.
make_loss_and_init argument is a callable that returns a tuple of
two other callables as follows.
The first will construct most of the graph and return the problem loss. It
is essential that this graph contains the totality of the model's variables,
but none of its queues. TODO(siege): I forget why this restriction is there.
The second will return construct the model initialization graph given a list
of parameters and return a callable that is passed an instance of
tf.Session, and should initialize the models' parameters.
An argument value function would look like this:
```python
def make_loss_and_init_fn():
inputs = queued_reader()
def make_loss():
return create_model_with_variables(inputs)
def make_init_fn(parameters):
saver = tf.Saver(parameters)
def init_fn(sess):
sess.restore(sess, ...)
return init_fn
return make_loss, make_init_fn
```
Args:
make_loss_and_init_fn: a callable, as described aboce
"""
make_loss_fn, make_init_fn = make_loss_and_init_fn()
self.make_loss_fn = make_loss_fn
self.parameters, self.constants = _get_variables(make_loss_fn)
if make_init_fn is not None:
init_fn = make_init_fn(self.parameters + self.constants)
else:
init_op = tf.initialize_variables(self.parameters + self.constants)
init_fn = lambda sess: sess.run(init_op)
tf.logging.info("ModelAdapter parameters: %s",
[op.name for op in self.parameters])
tf.logging.info("ModelAdapter constants: %s",
[op.name for op in self.constants])
super(ModelAdapter, self).__init__([],
random_seed=None,
noise_stdev=0.0,
init_fn=init_fn)
def initialize_uninitialized(sess):
# global_vars = tf.global_variables()
# is_not_initialized = sess.run([tf.is_variable_initialized(var) for var in global_vars])
# not_initialized_vars = [v for (v, f) in zip(global_vars, is_not_initialized) if not f]
# if len(not_initialized_vars):
# sess.run(tf.variables_initializer(not_initialized_vars))
uninit_vars = []
for var in tf.all_variables():
try:
sess.run(var)
except tf.errors.FailedPreconditionError:
uninit_vars.append(var)
init_new_vars_op = tf.initialize_variables(uninit_vars)
sess.run(init_new_vars_op)
def build_training_graph(task_name,
optimizer_name,
num_batchs_per_evaluation=5):
"""Build the tensorflow graph.
Args:
task_name: Name of task to build.
optimizer_name: Name of the optimizer to use.
num_batchs_per_evaluation: Number of batches to use when running a
single evaluation op. Note, this op is run multiple times per evauation by
training code.
Returns:
A dict containing TensorFlow tensors and operations used for training.
"""
global_step = tf.train.get_or_create_global_step()
task_mod = registry.task_registry.get_instance(task_name)
params = task_mod.current_params()
loss = task_mod.call_split(params, datasets.Split.TRAIN)
opt = registry.optimizers_registry.get_instance(optimizer_name)
train_op = opt.minimize(loss,
var_list=list(params.values()),
global_step=global_step)
train_op = tf.group(train_op, name="train_op")
(train_loss, valid_inner_loss, valid_outer_loss,
test_loss), _ = compute_averaged_loss(task_mod, params,
num_batchs_per_evaluation)
init_op = tf.initialize_variables(task_mod.get_variables())
return GraphEndpoints(train_op=train_op,
global_step=global_step,
init_op=init_op,
test_loss=test_loss,
valid_inner_loss=valid_inner_loss,
valid_outer_loss=valid_outer_loss,
train_loss=train_loss)
# Training hyper parameters
train_op = tf.train.AdamOptimizer(learning_rate=0.001,
beta1=0.9,
beta2=0.999,
epsilon=1e-08,
use_locking=False).minimize(
cost, var_list=train_params[parameters:])
uninitialized_vars = []
for var in tf.all_variables():
try:
sess.run(var)
except tf.errors.FailedPreconditionError:
uninitialized_vars.append(var)
init_new_vars_op = tf.initialize_variables(uninitialized_vars)
sess.run(init_new_vars_op)
log('TensorFlow Session starting...')
# TensorBoard summary (graph)
tf.summary.scalar('cost', cost)
merged_summary = tf.summary.merge_all()
writer = tf.summary.FileWriter('./tensorboard_test')
writer.add_graph(sess.graph)
log('TensorBoard infos in ./tensorboard_test')
# Save path depending on the training behaviour
if not args.transfer_model and args.transfer_cnn:
save_path = args.save_dir + '/cnn_s2p_' + appliance_name + '_transf_' + args.cnn + '_pointnet_model'
else:
def attack_batch(self, imgs, labs):
"""
Run the attack on a batch of images and labels.
"""
def compare(x, y):
if not isinstance(x, (float, int, np.int64)):
x = np.copy(x)
if self.TARGETED:
x[y] -= self.CONFIDENCE
else:
x[y] += self.CONFIDENCE
x = np.argmax(x)
if self.TARGETED:
return x == y
else:
return x != y
batch_size = self.batch_size
# convert to tanh-space
#imgs = np.arctanh((imgs - self.boxplus) / self.boxmul * 0.999999)
cc = []
ll = []
tl = []
mm = []
# set the lower and upper bounds accordingly
lower_bound = np.zeros(batch_size)
CONST = np.ones(batch_size) * self.initial_const
cc.append(CONST[0])
upper_bound = np.ones(batch_size) * 1e10
# the best l2, score, and image attack
o_bestl2 = [1e10] * batch_size
o_bestscore = [-1] * batch_size
o_bestattack = [np.zeros(imgs[0].shape)] * batch_size
f_bestl2 = [1e10] * batch_size
f_bestscore = [-1] * batch_size
f_bestattack = [np.zeros(imgs[0].shape)] * batch_size
print(o_bestl2)
# completely reset adam's internal state.
self.sess.run(self.init)
self.sess.run(self.mi_init)
'''
uninitialized_vars = []
for var in tf.all_variables():
try:
self.sess.run(var)
except tf.errors.FailedPreconditionError:
uninitialized_vars.append(var)
'''
self.sess.run(tf.initialize_variables(self.uninitialized_vars))
batch = imgs[:batch_size]
batchlab = labs[:batch_size]
bestl2 = [1e10] * batch_size
bestscore = [-1] * batch_size
fbestl2 = [1e10] * batch_size
fbestscore = [-1] * batch_size
# set the variables so that we don't have to send them over again
self.sess.run(self.setup, {
self.assign_timg: batch,
self.assign_tlab: batchlab
})
prev = np.inf
success = False
for iteration in range(self.MAX_ITERATIONS):
self.sess.run(self.lamda, {self.assign_const: CONST})
#_, l, ls1,scores, nimg = self.sess.run([self.train, self.loss1, self.loss, self.output, self.newimg])
_, l, ls1, nimg = self.sess.run(
[self.train, self.loss1, self.loss, self.newimg])
for xx in range(20):
#if iteration ==0:
_, l2s = self.sess.run([self.mi_train, self.MI])
mmi = self.sess.run(self.MI)
#if iteration % 10 ==0:
CONST = (1 - 0.1 * (1 / ((iteration + 1)**0.5))) * CONST + 0.1 * l
if CONST < 0:
CONST = np.zeros(batch_size)
'''
if np.all(scores>=-.0001) and np.all(scores <= 1.0001):
if np.allclose(np.sum(scores,axis=1), 1.0, atol=1e-3):
if not self.I_KNOW_WHAT_I_AM_DOING_AND_WANT_TO_OVERRIDE_THE_PRESOFTMAX_CHECK:
raise Exception("The output of model.predict should return the pre-softmax layer. It looks like you are returning the probability vector (post-softmax). If you are sure you want to do that, set attack.I_KNOW_WHAT_I_AM_DOING_AND_WANT_TO_OVERRIDE_THE_PRESOFTMAX_CHECK = True")
'''
# print out the losses every 10%
if iteration % (self.MAX_ITERATIONS // 10) == 0:
print(iteration, self.sess.run(
(self.loss, self.loss1, self.MI)))
# check if we should abort search if we're getting nowhere.
if self.ABORT_EARLY and iteration % (self.MAX_ITERATIONS //
10) == 0:
if l > prev * .9999:
break
prev = l
'''
# adjust the best result found so far
for e,(l2,sc,ii) in enumerate(zip(l2s,scores,nimg)):
if l2 > bestl2[e] and compare(sc, np.argmax(batchlab[e])):
bestl2[e] = l2
bestscore[e] = np.argmax(sc)
if l2 > o_bestl2[e] and compare(sc, np.argmax(batchlab[e])):
o_bestl2[e] = l2
o_bestscore[e] = np.argmax(sc)
o_bestattack[e] = ii
'''
for e, (l2, ii, fx) in enumerate(zip(l2s, nimg, [l])):
if fx == 0 and l2 > 0:
success = True
if l2 < bestl2[e] and l2 > 0:
bestl2[e] = l2
if l2 < o_bestl2[e] and l2 > 0:
o_bestl2[e] = l2
o_bestattack[e] = ii
if fx < fbestl2[e]:
fbestl2[e] = fx
if fx < f_bestl2[e]:
f_bestl2[e] = fx
f_bestattack[e] = ii
# return the best solution found
o_bestl2 = np.array(o_bestl2)
if success:
return o_bestattack, o_bestl2
else:
return f_bestattack, f_bestl2
def build_graph(
loss_module_fn,
learner_fn,
trainer_class,
np_global_step=None, # pylint: disable=unused-argument
):
"""Build the inner / outer training graph."""
local_device, remote_device, index_remote_device =\
device_utils.get_local_remote_device_fn(FLAGS.ps_tasks)
with tf.device(local_device):
with tf.device(remote_device):
global_step = tf.train.get_or_create_global_step()
loss_module = loss_module_fn()
learner, theta_mod = learner_fn(
loss_module=loss_module,
remote_device=remote_device,
)
trainer = trainer_class(
local_device=local_device,
remote_device=remote_device,
index_remote_device=index_remote_device,
learner=learner,
)
truncated_trainer_endpoints = trainer.build_endpoints()
trainable_theta_vars = theta_mod.get_variables(
tf.GraphKeys.TRAINABLE_VARIABLES)
logging.info("GOT %d trainable variables", len(trainable_theta_vars))
# The following is a sort of accounting of variables. It ensures variables
# are where you think they are, one is not creating extras.
# Variable management in distributed setting has caused great
# issues in the past.
# While verbose, this seems to mitigate a lot of it by being very explicit
# and throwing errors.
local_vars = list(learner.get_variables(tf.GraphKeys.GLOBAL_VARIABLES))
local_vars += list(
learner.loss_module.get_variables(tf.GraphKeys.GLOBAL_VARIABLES))
local_vars += trainer.get_local_variables()
saved_remote_vars = list(
theta_mod.get_variables(tf.GraphKeys.GLOBAL_VARIABLES))
saved_remote_vars += [global_step
] + trainer.get_saved_remote_variables()
not_saved_remote_vars = trainer.get_not_saved_remote_variables()
# TODO(lmetz) remove this line. For now, the meta_opt places variables in
# the wrong scopes
saved_remote_vars = list(set(saved_remote_vars))
all_remote_vars = saved_remote_vars + not_saved_remote_vars
logging.info("Remote Saved Variables")
for v in saved_remote_vars:
logging.info(" %s\t\t %s \t %s", v.shape.as_list(), v.device,
v.op.name)
logging.info("Remote Not Saved Variables")
for v in not_saved_remote_vars:
logging.info(" %s\t\t %s \t %s", v.shape.as_list(), v.device,
v.op.name)
logging.info("Local Variables")
for v in local_vars:
logging.info(" %s\t\t %s \t %s", v.shape.as_list(), v.device,
v.op.name)
logging.info("Trainable Theta Variables")
for v in theta_mod.get_variables(tf.GraphKeys.TRAINABLE_VARIABLES):
logging.info(" %s\t\t %s \t %s", v.shape.as_list(), v.device,
v.op.name)
device_utils.check_variables_accounting(local_vars, all_remote_vars)
device_utils.check_variables_are_local(local_vars)
device_utils.check_variables_are_remote(all_remote_vars)
chief_summary_op = tf.summary.merge([
tf.summary.scalar("global_step", global_step),
])
# Ops to run when a parameter server is reset.
ps_was_reset_ops = [tf.initialize_variables(not_saved_remote_vars)]
ps_was_reset_ops.append(trainer.ps_was_reset_op())
ps_was_reset_op = tf.group(*ps_was_reset_ops, name="ps_was_reset_op")
if FLAGS.ps_tasks == 0:
chief_device = ""
else:
chief_device = "/job:chief"
with tf.device(chief_device):
chief_is_ready = tf.get_variable(name="chief_is_ready",
initializer=tf.constant(False))
set_chief_is_ready = chief_is_ready.assign(True)
# this dictionary is result of / merged with trainer.
return dict(
global_step=global_step,
chief_summary_op=chief_summary_op,
saved_remote_vars=saved_remote_vars,
remote_vars=all_remote_vars,
local_vars=local_vars,
chief_is_ready=chief_is_ready,
set_chief_is_ready=set_chief_is_ready,
trainer_trainer_ops=truncated_trainer_endpoints,
ps_was_reset_op=ps_was_reset_op,
)
def init_learner_state(self):
learner_init_op = tf.initialize_variables(
self.learner.learner.get_variables(tf.GraphKeys.GLOBAL_VARIABLES))
local_inits = tf.get_collection(tf.GraphKeys.LOCAL_INIT_OP)
with tf.control_dependencies(local_inits + [learner_init_op]):
return self.learner.assign_state(self.learner.initial_state())
def main(_):
game = pyspiel.load_game(FLAGS.game)
# Information state length
info_state_shape = game.information_state_tensor_shape()
flat_info_state_length = np.prod(info_state_shape)
# Output
num_actions = game.num_distinct_actions()
with tf.Session() as sess:
net_input = tf.placeholder(tf.float32, [None, flat_info_state_length],
name="input")
# pylint: disable=unused-variable
output = tf.placeholder(tf.float32, [None, num_actions], name="output")
legals_mask = tf.placeholder(tf.float32, [None, num_actions],
name="legals_mask")
policy_net = tf.layers.dense(net_input, 128, activation=tf.nn.relu)
policy_net = tf.layers.dense(policy_net, 128, activation=tf.nn.relu)
policy_net = tf.layers.dense(policy_net, num_actions)
# Note: subtracting the max here is to help with numerical stability.
# However, there can still be numerical problems. If you are doing a softmax
# here, it can return NaN when the max for the policy net is high on one of
# the illegal actions, because policy_net - max will be small for legal
# actions, giving all exp(small) == 0 in the denominator, returning NaN at
# the end. One fix is to set the logits to -inf and define a custom cross
# entropy op that ignores over the illegal actions.
policy_net = policy_net - tf.reduce_max(
policy_net, axis=-1, keepdims=True)
masked_exp_logit = tf.multiply(tf.exp(policy_net), legals_mask)
renormalizing_factor = tf.reduce_sum(masked_exp_logit,
axis=-1,
keepdims=True)
# pylint: disable=unused-variable
policy_softmax = tf.where(tf.equal(legals_mask, 0.),
tf.zeros_like(masked_exp_logit),
tf.divide(masked_exp_logit,
renormalizing_factor),
name="policy_softmax")
policy_targets = tf.placeholder(shape=[None, num_actions],
dtype=tf.float32)
policy_cost = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits_v2(logits=policy_net,
labels=policy_targets),
axis=0)
# We make one sample.
sampled_actions = tf.random.categorical(tf.log(policy_softmax),
1,
name="sampled_actions")
# pylint: disable=unused-variable
optimizer = tf.train.AdamOptimizer(0.0001).minimize(policy_cost,
name="train")
# pylint: disable=unused-variable
init = tf.initialize_variables(tf.all_variables(),
name="init_all_vars_op")
print("Writing file: {}/{}".format(FLAGS.dir, FLAGS.filename))
tf.train.write_graph(sess.graph_def,
FLAGS.dir,
FLAGS.filename,
as_text=False)
def initialize():
new_variables = set(tf.all_variables()) - ALREADY_INITIALIZED
get_session().run(tf.initialize_variables(new_variables))
ALREADY_INITIALIZED.update(new_variables)
