def nn_layer(X, n_neurons, name, activation=None): # pylint: disable=invalid-name
"""Creates NN layer.
Creates NN layer with W's initialized as truncated normal and
b's as zeros.
Args:
X: Input tensor.
n_neurons: An integer. Number of neurons in the layer.
name: A string. Name of the layer.
activation: Activation function. Optional.
"""
with tf.name_scope(name):
# dimension of each x in X
n_inputs = int(X.get_shape()[1])
stddev = 2.0 / np.sqrt(n_inputs)
init = tf.truncated_normal((n_inputs, n_neurons), stddev=stddev)
with tf.device(_gpu_device_name(0)):
W = tf.Variable(init, name='W') # pylint: disable=invalid-name
b = tf.Variable(tf.zeros([n_neurons]), name='b') # pylint: disable=invalid-name
Z = tf.matmul(X, W) + b # pylint: disable=invalid-name
if activation is not None: # pylint: disable=no-else-return
return activation(Z)
else:
return Z
def apply_gradients(self, grads_and_vars, beta): # pylint: disable=arguments-differ
with tf.device(_gpu_device_name(self.replica_id)):
c = tf.sqrt(np.float32(2*self.learning_rate/beta)) # pylint: disable=invalid-name
ops_ = [tf.assign(
v,
v - self.learning_rate*g + c*tf.random_normal(v.shape, stddev=1))
for g, v in grads_and_vars]
return tf.group(ops_)
def apply_gradients(self, grads_and_vars):
"""Applies gradients.
Args:
grads_and_vars: list of tuples as returned by
optimizer.compute_gradients()
Returns:
An op for gradient computation.
"""
with tf.device(_gpu_device_name(self.replica_id)):
ops_ = [tf.assign(v, v - self.learning_rate*g)
for g, v in grads_and_vars]
train_op = tf.group(ops_)
return train_op
def apply_gradients(self, grads_and_vars, stddev): # pylint: disable=arguments-differ
"""Applies gradients and adds normal noise with `stddev`.
Args:
grads_and_vars: list of tuples as returned by
optimizer.compute_gradients()
stddev: standard deviation of normal noise
Returns:
An op for gradient computation.
"""
with tf.device(_gpu_device_name(self.replica_id)):
ops_ = [tf.assign(
var,
(var
- self.learning_rate*grad
+ tf.random_normal(var.shape, stddev=stddev)))
for grad, var in grads_and_vars]
train_op = tf.group(ops_)
return train_op
def cnn_cifar10_model(graph):
with graph.as_default():
with tf.name_scope('Inputs'):
with tf.name_scope('X'):
X = tf.placeholder(tf.float32, shape=(None, 3072), name='X')
X_reshaped = tf.reshape(X,
shape=tf.TensorShape([-1, 32, 32, 3]))
with tf.name_scope('y'):
y = tf.placeholder(tf.int64, shape=(None), name='y')
with tf.device(_gpu_device_name(0)):
with tf.name_scope('conv1'):
kernel = _variable_with_weight_decay(name='kernel1',
shape=[5, 5, 3, 64],
stddev=5e-2,
wd=None)
conv = tf.nn.conv2d(input=X_reshaped,
filter=kernel,
strides=[1, 1, 1, 1],
padding='SAME')
biases = tf.get_variable(
name='biases1',
shape=[64],
initializer=tf.constant_initializer(0.0))
pre_activation = tf.nn.bias_add(conv, biases)
conv1 = tf.nn.relu(pre_activation)
with tf.name_scope('pool1'):
pool1 = tf.nn.max_pool(conv1,
ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool1')
with tf.name_scope('norm1'):
norm1 = tf.nn.lrn(pool1,
4,
bias=1.0,
alpha=0.001 / 9.0,
beta=0.75,
name='norm1')
with tf.name_scope('conv2'):
kernel = _variable_with_weight_decay('kernel2',
shape=[5, 5, 64, 64],
stddev=5e-2,
wd=None)
conv = tf.nn.conv2d(input=norm1,
filter=kernel,
strides=[1, 1, 1, 1],
padding='SAME')
biases = tf.get_variable(
name='biases2',
shape=[64],
initializer=tf.constant_initializer(0.1))
pre_activation = tf.nn.bias_add(conv, biases)
conv2 = tf.nn.relu(pre_activation)
with tf.name_scope('norm2'):
norm2 = tf.nn.lrn(conv2,
4,
bias=1.0,
alpha=0.001 / 9.0,
beta=0.75,
name='norm2')
with tf.name_scope('pool2'):
pool2 = tf.nn.max_pool(norm2,
ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool2')
with tf.name_scope('fully_connected1'):
reshaped = tf.reshape(
pool2, [X_reshaped.get_shape().as_list()[0], -1])
fc1 = nn_layer(reshaped, 384, 'fully_connected1', tf.nn.relu)
keep_prob = tf.placeholder(tf.float32, name='keep_prob')
with tf.name_scope('dropout1'):
fc1_dropout = tf.nn.dropout(fc1, keep_prob)
with tf.name_scope('fully_connected2'):
fc2 = nn_layer(fc1_dropout, 192, 'fully_connected2',
tf.nn.relu)
with tf.name_scope('dropout2'):
fc2_dropout = tf.nn.dropout(fc2, keep_prob)
with tf.name_scope('logits'):
logits = nn_layer(fc2_dropout, 10, 'logits')
return X, y, keep_prob, logits
def cnn_cifar10_model2(graph):
height = 28
width = 28
channels = 1
n_inputs = height * width
conv1_fmaps = 32
conv1_ksize = 3
conv1_stride = 1
conv1_pad = "SAME"
conv2_fmaps = 64
conv2_ksize = 3
conv2_stride = 2
conv2_pad = "SAME"
pool3_fmaps = conv2_fmaps
n_fc1 = 64
n_outputs = 10
with graph.as_default():
with tf.name_scope('Input'):
X = tf.placeholder(tf.float32,
shape=[None, width * height * n_channels],
name='X')
y = tf.placeholder(tf.float32, shape=[None, n_classes])
X_reshaped = tf.reshape(X,
shape=[-1, width, height, n_channels],
name='y')
#keep_prob = tf.placeholder(tf.float32, shape=[], name='keep_proba')
#keep_prob_val = 1.0 - keep_prob
#keep_prob_val = tf.Variable(tf.zeros([]), name='keep_prob_val')
#keep_prob_val = tf.assign(keep_prob_val, 1.0 - keep_prob, name='assigned_prob')
with tf.device(_gpu_device_name(0)):
with tf.name_scope('conv1'):
conv1 = tf.layers.conv2d(X_reshaped, )
"""
with tf.name_scope('conv1') as scope:
W = tf.Variable('W', )
conv = tf.layers.conv2d(inputs=X_reshaped, filters=32,
kernel_size=[3,3], padding='SAME', activation=tf.nn.relu)
conv = tf.layers.conv2d(inputs=conv, filters=64,
kernel_size=[3,3], padding='SAME', activation=tf.nn.relu)
pool = tf.layers.max_pooling2d(conv, pool_size=[2,2],
strides=2, padding='SAME')
drop = tf.layers.dropout(pool, rate=keep_prob_val, name=scope)
with tf.name_scope('conv2'):
conv = tf.layers.conv2d(inputs=drop, filters=128,
kernel_size=[2,2], padding='SAME', activation=tf.nn.relu)
pool = tf.layers.max_pooling2d(conv, pool_size=[2,2],
strides=2, padding='SAME')
conv = tf.layers.conv2d(inputs=pool, filters=128,
kernel_size=[2,2], padding='SAME', activation=tf.nn.relu)
pool = tf.layers.max_pooling2d(conv, pool_size=[2,2],
strides=2, padding='SAME')
drop = tf.layers.dropout(pool, rate=keep_prob_val)
with tf.name_scope('fully_connected'):
flat = tf.reshape(drop, [-1, 4 * 4 * 128])
fc = nn_layer(flat, 1500, name='fully_connected',
activation=tf.nn.relu)
drop = tf.layers.dropout(fc, rate=keep_prob_val)
logits = nn_layer(drop, 10, name='logits', activation=tf.nn.softmax)
"""
return X, y, keep_prob, logits
def cnn_cifar10_model3(graph):
height = 32
width = 32
channels = 3
n_inputs = height * width * channels
conv1_fmaps = 32
conv1_ksize = 3
conv1_stride = 1
conv1_pad = "SAME"
conv2_fmaps = 64
conv2_ksize = 3
conv2_stride = 2
conv2_pad = "SAME"
pool3_fmaps = conv2_fmaps
n_fc1 = 64
n_outputs = 10
gpu_device_name = _gpu_device_name(0)
with graph.as_default():
with tf.name_scope('Input'):
with tf.name_scope('X'):
X = tf.placeholder(tf.float32,
shape=[None, n_inputs],
name='X')
X_reshaped = tf.reshape(X, shape=[-1, height, width, channels])
with tf.name_scope('y'):
y = tf.placeholder(tf.int32, shape=[None], name='y')
with tf.device(gpu_device_name):
with tf.name_scope('conv1'):
conv1 = tf.layers.conv2d(X_reshaped,
filters=conv1_fmaps,
kernel_size=conv1_ksize,
strides=conv1_stride,
padding=conv1_pad,
activation=tf.nn.relu,
name='conv1')
with tf.name_scope('conv2'):
conv2 = tf.layers.conv2d(conv1,
filters=conv2_fmaps,
kernel_size=conv2_ksize,
strides=conv2_stride,
padding=conv2_pad,
activation=tf.nn.relu,
name='conv2')
with tf.name_scope('pool3'):
pool3 = tf.nn.max_pool(conv2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='VALID')
pool3_flat = tf.reshape(pool3, shape=[-1, pool3_fmaps * 7 * 7])
with tf.name_scope('fully_connected'):
fc = nn_layer(pool3_flat, n_fc1, activation=tf.nn.relu, name='fc')
with tf.device(gpu_device_name):
with tf.name_scope('dropout'):
keep_prob = tf.placeholder(tf.float32, name='keep_prob')
fc_dropout = tf.nn.dropout(fc, keep_prob)
with tf.name_scope('logits'):
logits = tf.layers.dense(fc_dropout, n_outputs, name='logits')
return X, y, keep_prob, logits
def compute_gradients(self, loss):
"""Wrapper for tf.train.Optimizer.minimize()"""
var_list = self._get_dependencies(loss)
with tf.device(_gpu_device_name(self.replica_id)):
grads_and_vars = self.tf_optimizer.compute_gradients(loss, var_list)
return grads_and_vars
def minimize(self, loss):
self.trainable_variables = self._get_dependencies(loss) # pylint: disable=attribute-defined-outside-init
with tf.device(_gpu_device_name(self.replica_id)):
self.train_op = self.tf_optimizer.minimize(loss) # pylint: disable=attribute-defined-outside-init
return self.train_op
def copy_variable_to_graph(org_instance, to_graph, namespace): # pylint: disable=too-many-locals
"""Copies the Variable instance 'org_instance' into the graph
'to_graph', under the given namespace.
The dict 'COPIED_VARIABLES', if provided, will be updated with
mapping the new variable's name to the instance.
"""
# global COPIED_VARIABLES
if not isinstance(org_instance, tf.Variable):
raise TypeError(str(org_instance) + " is not a Variable")
#The name of the new variable
if namespace != '':
new_name = (namespace + '/' +
org_instance.name[:org_instance.name.index(':')])
else:
new_name = org_instance.name[:org_instance.name.index(':')]
###############################################################
if namespace != '':
replica_id = int(namespace.split('_')[1])
else:
replica_id = -1
###############################################################
#Get the collections that the new instance needs to be added to.
#The new collections will also be a part of the given namespace,
#except the special ones required for variable initialization and
#training.
collections = []
for name, collection in org_instance.graph._collections.items(): # pylint: disable=protected-access
if org_instance in collection:
if (name == ops.GraphKeys.GLOBAL_VARIABLES
or name == ops.GraphKeys.TRAINABLE_VARIABLES
or namespace == ''):
collections.append(name)
else:
collections.append(namespace + '/' + name)
#See if its trainable.
trainable = (org_instance in org_instance.graph.get_collection(
ops.GraphKeys.TRAINABLE_VARIABLES))
#Get the initial value
with org_instance.graph.as_default():
temp_session = tf.Session() # pylint: disable=unused-variable
# init_value = temp_session.run(org_instance.initialized_value())
#Initialize the new variable
with to_graph.as_default():
'''
new_var = PLACER.copy_and_init_variable_on_cpu( org_instance,
new_name,
trainable=trainable,
collections=collections,
validate_shape=True)
'''
######################################################
if (replica_id >= 0 and 'gpu' in org_instance.device.lower()):
device_name = _gpu_device_name(replica_id)
else:
device_name = '/cpu:0'
with tf.device(device_name):
n_inputs = int(org_instance.get_shape()[0])
try:
n_neurons = int(org_instance.get_shape()[1])
stddev = 2.0 / np.sqrt(n_inputs)
init = tf.truncated_normal((n_inputs, n_neurons),
stddev=stddev)
except IndexError:
init = tf.zeros([n_inputs])
new_var = tf.Variable(init,
trainable=trainable,
name=new_name,
collections=collections,
validate_shape=True)
######################################################
'''
new_var = tf.Variable(init_value,
trainable=trainable,
name=new_name,
collections=collections,
validate_shape=True)
'''
#Add to the COPIED_VARIABLES dict
COPIED_VARIABLES[new_var.name] = new_var
'''
if (replica_id >=0 and
'gpu' in org_instance.device.lower()):
new_var._set_device(_gpu_device_name(replica_id))
'''
#print('var:', org_instance.device, new_var.device)
return new_var
def copy_to_graph(org_instance, to_graph, namespace="", exclude=None): # pylint: disable=too-many-locals, too-many-statements, too-many-branches
"""
Makes a copy of the Operation/Tensor instance 'org_instance'
for the graph 'to_graph', recursively. Therefore, all required
structures linked to org_instance will be automatically copied.
'COPIED_VARIABLES' should be a dict mapping pertinent copied variable
names to the copied instances.
The new instances are automatically inserted into the given 'namespace'.
If namespace='', it is inserted into the graph's global namespace.
However, to avoid naming conflicts, its better to provide a namespace.
If the instance(s) happens to be a part of collection(s), they are
are added to the appropriate collections in to_graph as well.
For example, for collection 'C' which the instance happens to be a
part of, given a namespace 'N', the new instance will be a part of
'N/C' in to_graph.
Returns the corresponding instance with respect to to_graph.
TODO: Order of insertion into collections is not preserved
"""
#print(org_instance.name)
#print(org_instance.name)
####################################################################
if namespace != '':
replica_id = int(namespace.split('_')[1])
else:
replica_id = -1
global COPIED_VARIABLES # pylint: disable=global-statement
if exclude:
for exc in exclude:
if org_instance.name.split(':')[0] == exc.name.split(':')[0]:
return exc
####################################################################
#The name of the new instance
if namespace != '':
new_name = namespace + '/' + org_instance.name
else:
new_name = org_instance.name
#If a variable by the new name already exists, return the
#correspondng tensor that will act as an input
if new_name in COPIED_VARIABLES:
return to_graph.get_tensor_by_name(COPIED_VARIABLES[new_name].name)
#If an instance of the same name exists, return appropriately
try:
already_present = to_graph.as_graph_element(new_name,
allow_tensor=True,
allow_operation=True)
return already_present
except: # pylint: disable=bare-except
pass
#Get the collections that the new instance needs to be added to.
#The new collections will also be a part of the given namespace.
collections = []
for name, collection in org_instance.graph._collections.items(): # pylint: disable=protected-access
if org_instance in collection:
if namespace == '':
collections.append(name)
else:
collections.append(namespace + '/' + name)
#Take action based on the class of the instance
if isinstance(org_instance, tf.Tensor): # pylint: disable=no-else-return
#If its a Tensor, it is one of the outputs of the underlying
#op. Therefore, copy the op itself and return the appropriate
#output.
op = org_instance.op # pylint: disable=invalid-name
new_op = copy_to_graph(op, to_graph, namespace, exclude=exclude)
output_index = op.outputs.index(org_instance)
new_tensor = new_op.outputs[output_index]
#Add to collections if any
for collection in collections:
to_graph.add_to_collection(collection, new_tensor)
return new_tensor
elif isinstance(org_instance, tf.Operation):
op = org_instance # pylint: disable=invalid-name
#If it has an original_op parameter, copy it
if op._original_op is not None: # pylint: disable=protected-access
new_original_op = copy_to_graph(
op._original_op,
to_graph, # pylint: disable=protected-access
namespace,
exclude=exclude)
else:
new_original_op = None
#If it has control inputs, call this function recursively on each.
new_control_inputs = [
copy_to_graph(x, to_graph, namespace, exclude=exclude)
for x in op.control_inputs
]
#If it has inputs, call this function recursively on each.
new_inputs = [
copy_to_graph(x, to_graph, namespace, exclude=exclude)
for x in op.inputs
]
#Make a new node_def based on that of the original.
#An instance of tensorflow.core.framework.graph_pb2.NodeDef, it
#stores String-based info such as name, device and type of the op.
#Unique to every Operation instance.
new_node_def = deepcopy(op.node_def)
#Change the name
new_node_def.name = new_name
#Copy the other inputs needed for initialization
output_types = op._output_types[:] # pylint: disable=protected-access
input_types = op._input_types[:] # pylint: disable=protected-access
#print('name:', new_name)
#print('output_types:',output_types)
#print('input types',input_types)
#print('new inputs', new_inputs)
#print('##########################')
#Make a copy of the op_def too.
#Its unique to every _type_ of Operation.
op_def = deepcopy(op.op_def)
#Initialize a new Operation instance
new_op = tf.Operation(new_node_def, to_graph, new_inputs, output_types,
new_control_inputs, input_types, new_original_op,
op_def)
#Use Graph's hidden methods to add the op
to_graph._add_op(new_op) # pylint: disable=protected-access
to_graph._record_op_seen_by_control_dependencies(new_op) # pylint: disable=protected-access
#print(to_graph._device_function_stack)
for device_function in reversed(to_graph._device_function_stack): # pylint: disable=protected-access
new_op._set_device(device_function(new_op)) # pylint: disable=protected-access
#########################################################
#print(device_function(new_op))
#new_op = PLACER.set_on_gpu(new_op, replica_id)
########################################################
########################################################
if (replica_id >= 0 and 'gpu' in op.device.lower()):
new_op._set_device(_gpu_device_name(replica_id)) # pylint: disable=protected-access
return new_op
########################################################
'''
return new_op
'''
else:
raise TypeError("Could not copy instance: " + str(org_instance))