def infer_outputs(self):
"""
Use TensorFlow's shape and dtype inference to determine the number of
outputs as well as their shapes and dtypes, based on the node's op type
string, its attribute values, and what inputs are connected to it.
Inference will only function properly if the currently-loaded version of
TensorFlow knows about the specified op type and the current
configuration of this op's inputs is compatible with the combination of
op type string and parameters.
Overwrites the previous value of the `outputs` property.
Raises:
TBD
"""
# TF lack a supported API for invoking shape inference directly,
# so we instantiate a dummy graph and create a dummy Operation object
temp_graph = tf.Graph()
with temp_graph.as_default():
input_placeholders = [
tf.placeholder(shape=t.shape, dtype=t.dtype)
for t in self._inputs
]
# See the docs for tf.Operation for important notes about the semantics
# of each arg to the following constructor.
dummy_op = tf.Operation(self.to_node_def(),
temp_graph,
inputs=input_placeholders)
self.set_outputs_from_pairs([(o.dtype, o.shape)
for o in dummy_op.outputs])
def copy_op_to_graph(op, to_graph, cached_tensors, scope=''):
if scope != '':
new_name = scope + '/' + op.name
else:
new_name = op.name
try:
already_present = to_graph.as_graph_element(new_name,
allow_tensor=True,
allow_operation=True)
return already_present
except:
pass
if op._original_op is not None:
new_original_op = copy_op_to_graph(op._original_op, to_graph,
cached_tensors, scope)
else:
new_original_op = None
new_control_inputs = [
copy_op_to_graph(x, to_graph, cached_tensors, scope)
for x in op.control_inputs
]
#If it has inputs, call this function recursively on each.
new_inputs = [
copy_tensor_to_graph(x, to_graph, cached_tensors, scope)
for x in op.inputs
]
#Make a new node_def based on that of the original.
#An instance of tensorflow.core.framework.node_def_pb2.NodeDef, it
#stores String-based info such as name, device and type of the op.
#Unique to every Operation instance.
new_node_def = deepcopy(op.node_def)
#Change the name
new_node_def.name = new_name
#Copy the other inputs needed for initialization
output_types = op._output_types[:]
input_types = op._input_types[:]
#Make a copy of the op_def too.
#Its unique to every _type_ of Operation.
op_def = deepcopy(op.op_def)
#Initialize a new Operation instance
new_op = tf.Operation(new_node_def, to_graph, new_inputs, output_types,
new_control_inputs, input_types, new_original_op,
op_def)
#Use Graph's hidden methods to add the op
to_graph._record_op_seen_by_control_dependencies(new_op)
# pylint: disable=protected-access
for device_function in to_graph._device_functions_outer_to_inner:
new_op._set_device(device_function(new_op))
# pylint: enable=protected-access
return new_op
def copy_op(op, new_name):
nnd = deepcopy(op.node_def) # ~7% of test script runtime
nnd.name = new_name
op_def = deepcopy(op.op_def) # ~2% of test script runtime
new_op = tf.Operation( # ~22% of test script runtime
nnd,
op.graph,
list(op.inputs),
op._output_types[:], # ~2% of runtime
op.control_inputs[:],
op._input_types[:],
op,
op_def,
)
return new_op
def infer_outputs(self):
"""
Use TensorFlow's shape and dtype inference to determine the number of
outputs as well as their shapes and dtypes, based on the node's op type
string, its attribute values, and what inputs are connected to it.
Inference will only function properly if the currently-loaded version of
TensorFlow knows about the specified op type and the current
configuration of this op's inputs is compatible with the combination of
op type string and parameters.
Overwrites the previous value of the `outputs` property.
Raises:
TBD
"""
if self.op_type == "Assign":
# SPECIAL CASE: Assign op takes a reference as input. Don't build up a
# graph and invoke shape inference, because the APIs for references are
# in flux. Instead, just trust the attributes.
# First input is the reference, second is the value to put in place.
# Assign op returns the reference that it just assigned to.
input_ref = self._inputs[0]
self.set_outputs_from_pairs([(input_ref.dtype, input_ref.shape)])
else:
# Common case: Use shape inference.
# TF lacks a supported API for invoking shape inference directly,
# so we instantiate a dummy graph and create a dummy Operation object.
temp_graph = tf.Graph()
with temp_graph.as_default():
input_placeholders = [
tf.placeholder(shape=t.shape, dtype=t.dtype)
for t in self._inputs
]
# See the docs for tf.Operation for important notes about the semantics
# of each arg to the following constructor.
dummy_op = tf.Operation(self.to_node_def(),
temp_graph,
inputs=input_placeholders)
self.set_outputs_from_pairs([(o.dtype, o.shape)
for o in dummy_op.outputs])
def copy(org_instance,
dict_swap=None,
scope="copied",
replace_itself=False,
copy_q=False):
"""Build a new node in the TensorFlow graph from `org_instance`,
where any of its ancestors existing in `dict_swap` are
replaced with `dict_swap`'s corresponding value.
The copying is done recursively, so any `Operation` whose output
is required to evaluate `org_instance` is also copied (if it isn't
already copied within the new scope). This is with the exception of
`tf.Variable`s, `tf.placeholder`s, and nodes of type `Queue`, which
are reused and not newly copied.
Parameters
----------
org_instance : RandomVariable, tf.Variable, tf.Tensor, or tf.Operation
Node to add in graph with replaced ancestors.
dict_swap : dict, optional
Random variables, variables, tensors, or operations to swap with.
Its keys are what `org_instance` may depend on, and its values are
the corresponding object (not necessarily of the same class
instance, but must have the same type, e.g., float32) that is used
in exchange.
scope : str, optional
A scope for the new node(s). This is used to avoid name
conflicts with the original node(s).
replace_itself : bool, optional
Whether to replace `org_instance` itself if it exists in
`dict_swap`. (This is used for the recursion.)
copy_q : bool, optional
Whether to copy the replaced tensors too (if not already
copied within the new scope). Otherwise will reuse them.
Returns
-------
RandomVariable, tf.Variable, tf.Tensor, or tf.Operation
The copied node.
Raises
------
TypeError
If `org_instance` is not one of the above types.
Examples
--------
>>> x = tf.constant(2.0)
>>> y = tf.constant(3.0)
>>> z = x * y
>>>
>>> qx = tf.constant(4.0)
>>> # The TensorFlow graph is currently
>>> # `x` -> `z` <- y`, `qx`
>>>
>>> # This adds a subgraph with newly copied nodes,
>>> # `copied/qx` -> `copied/z` <- `copied/y`
>>> z_new = copy(z, {x: qx})
>>>
>>> sess = tf.Session()
>>> sess.run(z)
6.0
>>> sess.run(z_new)
12.0
"""
if not isinstance(org_instance, RandomVariable) and \
not isinstance(org_instance, tf.Variable) and \
not isinstance(org_instance, tf.Tensor) and \
not isinstance(org_instance, tf.Operation):
raise TypeError("Could not copy instance: " + str(org_instance))
if dict_swap is None:
dict_swap = {}
# Swap instance if in dictionary.
if org_instance in dict_swap and replace_itself:
org_instance = dict_swap[org_instance]
if not copy_q:
return org_instance
elif isinstance(org_instance, tf.Tensor) and replace_itself:
# Deal with case when `org_instance` is the associated tensor
# from the RandomVariable, e.g., `z.value()`. If
# `dict_swap={z: qz}`, we aim to swap it with `qz.value()`.
for key, value in six.iteritems(dict_swap):
if isinstance(key, RandomVariable):
if org_instance == key.value():
if isinstance(value, RandomVariable):
org_instance = value.value()
else:
org_instance = value
if not copy_q:
return org_instance
break
graph = tf.get_default_graph()
new_name = scope + '/' + org_instance.name
# If an instance of the same name exists, return appropriately.
# Do this for random variables.
random_variables = {
x.name: x
for x in graph.get_collection('_random_variable_collection_')
}
if new_name in random_variables:
return random_variables[new_name]
# Do this for tensors and operations.
try:
already_present = graph.as_graph_element(new_name,
allow_tensor=True,
allow_operation=True)
return already_present
except:
pass
# If instance is a variable, return it; do not re-copy any.
# Note we check variables via their name and not their type. This
# is because if we get variables through an op's inputs, it has
# type tf.Tensor: we can only tell it is a variable via its name.
variables = {
x.name: x
for x in graph.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
}
if org_instance.name in variables:
return graph.get_tensor_by_name(variables[org_instance.name].name)
# Do the same for placeholders. Determine via its op's type.
if isinstance(org_instance, tf.Tensor) and \
"Placeholder" in org_instance.op.type:
return org_instance
if isinstance(org_instance, RandomVariable):
rv = org_instance
# If it has copiable arguments, copy them.
args = []
for arg in rv._args:
if isinstance(arg, RandomVariable) or \
isinstance(arg, tf.Variable) or \
isinstance(arg, tf.Tensor) or \
isinstance(arg, tf.Operation):
arg = copy(arg, dict_swap, scope, True, copy_q)
args.append(arg)
kwargs = {}
for key, value in six.iteritems(rv._kwargs):
if isinstance(value, list):
kwargs[key] = [
copy_rv(v, dict_swap, scope, True, copy_q) for v in value
]
else:
kwargs[key] = copy_rv(value, dict_swap, scope, True, copy_q)
kwargs['name'] = new_name
# Create new random variable with copied arguments.
new_rv = rv.__class__(*args, **kwargs)
return new_rv
elif isinstance(org_instance, tf.Tensor):
tensor = org_instance
# A tensor is one of the outputs of its underlying
# op. Therefore copy the op itself.
op = tensor.op
new_op = copy(op, dict_swap, scope, True, copy_q)
output_index = op.outputs.index(tensor)
new_tensor = new_op.outputs[output_index]
# Add copied tensor to collections that the original one is in.
for name, collection in tensor.graph._collections.items():
if tensor in collection:
graph.add_to_collection(name, new_tensor)
return new_tensor
elif isinstance(org_instance, tf.Operation):
op = org_instance
# Do not copy queue operations
if 'Queue' in op.type:
return op
# If it has an original op, copy it.
if op._original_op is not None:
new_original_op = copy(op._original_op, dict_swap, scope, True,
copy_q)
else:
new_original_op = None
# If it has control inputs, copy them.
new_control_inputs = []
for x in op.control_inputs:
elem = copy(x, dict_swap, scope, True, copy_q)
if not isinstance(elem, tf.Operation):
elem = tf.convert_to_tensor(elem)
new_control_inputs += [elem]
# If it has inputs, copy them.
new_inputs = []
for x in op.inputs:
elem = copy(x, dict_swap, scope, True, copy_q)
if not isinstance(elem, tf.Operation):
elem = tf.convert_to_tensor(elem)
new_inputs += [elem]
# Make a copy of the node def.
# As an instance of tensorflow.core.framework.graph_pb2.NodeDef, it
# stores string-based info such as name, device, and type of the op.
# It is unique to every Operation instance.
new_node_def = deepcopy(op.node_def)
new_node_def.name = new_name
# Copy the other inputs needed for initialization.
output_types = op._output_types[:]
input_types = op._input_types[:]
# Make a copy of the op def.
# It is unique to every Operation type.
op_def = deepcopy(op.op_def)
ret = tf.Operation(new_node_def, graph, new_inputs, output_types,
new_control_inputs, input_types, new_original_op,
op_def)
# Use Graph's private methods to add the op, following
# implementation of `tf.Graph().create_op()`.
compute_shapes = True
compute_device = True
op_type = new_name
if compute_shapes:
set_shapes_for_outputs(ret)
graph._add_op(ret)
graph._record_op_seen_by_control_dependencies(ret)
if compute_device:
graph._apply_device_functions(ret)
if graph._colocation_stack:
all_colocation_groups = []
for colocation_op in graph._colocation_stack:
all_colocation_groups.extend(colocation_op.colocation_groups())
if colocation_op.device:
# Make this device match the device of the colocated op, to
# provide consistency between the device and the colocation
# property.
if ret.device and ret.device != colocation_op.device:
logging.warning(
"Tried to colocate %s with an op %s that had "
"a different device: %s vs %s. "
"Ignoring colocation property.", name,
colocation_op.name, ret.device,
colocation_op.device)
else:
ret._set_device(colocation_op.device)
all_colocation_groups = sorted(set(all_colocation_groups))
ret.node_def.attr["_class"].CopyFrom(
attr_value_pb2.AttrValue(
list=attr_value_pb2.AttrValue.ListValue(
s=all_colocation_groups)))
# Sets "container" attribute if
# (1) graph._container is not None
# (2) "is_stateful" is set in OpDef
# (3) "container" attribute is in OpDef
# (4) "container" attribute is None
if (graph._container and op_type in graph._registered_ops
and graph._registered_ops[op_type].is_stateful
and "container" in ret.node_def.attr
and not ret.node_def.attr["container"].s):
ret.node_def.attr["container"].CopyFrom(
attr_value_pb2.AttrValue(s=compat.as_bytes(graph._container)))
return ret
else:
raise TypeError("Could not copy instance: " + str(org_instance))
def copy_to_graph(org_instance, to_graph, namespace="", exclude=None):
"""Creates a copy of the `Operation`/`Tensor` `org_instance.
The copying is done recursively with the help of `COPIED_VARIABLES`
dictionary which stores already copied instances. Additionaly,
it is possible to exclude additional `Tensor`s or `Operation`s by
putting already copied instances to the `exclude` list. The copied
instances are inserted under the provided `namespace`. To avoid
naming conflicts it is better to provide a value for `namespace`.
Args:
`org_instance`: A instance of `Operation` or `Tensor` to be copied
to the `to_graph`.
`to_graph`: A graph where `org_instance` should be copied.
`namespace`: A namespace under which the `org_instance` will be
copied.
`exclude`: A list of variables/tensors/ops that should not be
copied.
Returns:
A copied instance.
Raises:
`ValueError`: If the intance couldn't be copied.
"""
####################################################################
if namespace != '':
replica_id = int(namespace.split('_')[1])
else:
replica_id = -1
global COPIED_VARIABLES # pylint: disable=global-statement
if exclude:
for exc in exclude:
if org_instance.name.split(':')[0] == exc.name.split(':')[0]:
return exc
####################################################################
# The name of the new instance
if namespace != '':
new_name = namespace + '/' + org_instance.name
else:
new_name = org_instance.name
# If a variable by the new name already exists, return the
# correspondng tensor that will act as an input
if new_name in COPIED_VARIABLES:
print('copied',new_name)
return to_graph.get_tensor_by_name(
COPIED_VARIABLES[new_name].name)
# If an instance of the same name exists, return appropriately
try:
already_present = to_graph.as_graph_element(
new_name,
allow_tensor=True,
allow_operation=True)
return already_present
except: # pylint: disable=bare-except
pass
# Get the collections that the new instance needs to be added to.
# The new collections will also be a part of the given namespace.
collections = []
for name, collection in org_instance.graph._collections.items(): # pylint: disable=protected-access
if org_instance in collection:
if namespace == '':
collections.append(name)
else:
collections.append(namespace + '/' + name)
# Take action based on the class of the instance
if isinstance(org_instance, tf.Tensor): # pylint: disable=no-else-return
# If its a Tensor, it is one of the outputs of the underlying
# op. Therefore, copy the op itself and return the appropriate
# output.
op = org_instance.op # pylint: disable=invalid-name
new_op = copy_to_graph(op, to_graph, namespace, exclude=exclude)
output_index = op.outputs.index(org_instance)
new_tensor = new_op.outputs[output_index]
#Add to collections if any
for collection in collections:
to_graph.add_to_collection(collection, new_tensor)
return new_tensor
elif isinstance(org_instance, tf.Operation):
op = org_instance # pylint: disable=invalid-name
# If it has an original_op parameter, copy it
if op._original_op is not None: # pylint: disable=protected-access
new_original_op = copy_to_graph(
op._original_op, to_graph, # pylint: disable=protected-access
namespace, exclude=exclude)
else:
new_original_op = None
# If it has control inputs, call this function recursively on each.
new_control_inputs = [copy_to_graph(x, to_graph,
namespace,
exclude=exclude)
for x in op.control_inputs]
# If it has inputs, call this function recursively on each.
new_inputs = [copy_to_graph(x, to_graph,
namespace, exclude=exclude)
for x in op.inputs]
# Make a new node_def based on that of the original.
# An instance of tensorflow.core.framework.graph_pb2.NodeDef, it
# stores String-based info such as name, device and type of the op.
# Unique to every Operation instance.
new_node_def = deepcopy(op.node_def)
# Change the name
new_node_def.name = new_name
# Copy the other inputs needed for initialization
output_types = op._output_types[:] # pylint: disable=protected-access
input_types = op._input_types[:] # pylint: disable=protected-access
# Make a copy of the op_def too.
# Its unique to every _type_ of Operation.
op_def = deepcopy(op.op_def)
# Initialize a new Operation instance
new_op = tf.Operation(
new_node_def,
to_graph,
new_inputs,
output_types,
new_control_inputs,
input_types,
new_original_op,
op_def)
########################################################
if StrictVersion(tf.__version__) == StrictVersion('1.12.0'):
to_graph._record_op_seen_by_control_dependencies(new_op)
for device_function in to_graph._device_functions_outer_to_inner:
new_op._set_device(device_function(new_op))
elif StrictVersion(tf.__version__) == StrictVersion('1.9.0'):
to_graph._add_op(new_op)
for device_function in reversed(to_graph._device_function_stack):
new_op._set_device(device_function(new_op)) # pylint: disable=protected-access
to_graph._record_op_seen_by_control_dependencies(new_op)
else:
raise ValueError('Not supported tensorflow version.')
if (replica_id >= 0 and
'gpu' in op.device.lower()):
new_op._set_device(_gpu_device_name(replica_id)) # pylint: disable=protected-access
return new_op
########################################################
else:
raise ValueError("Could not copy instance: " + str(org_instance))
def copy_to_graph(org_instance, to_graph, namespace=""):
"""
Makes a copy of the Operation/Tensor instance 'org_instance'
for the graph 'to_graph', recursively. Therefore, all required
structures linked to org_instance will be automatically copied.
'copied_variables' should be a dict mapping pertinent copied variable
names to the copied instances.
The new instances are automatically inserted into the given 'namespace'.
If namespace='', it is inserted into the graph's global namespace.
However, to avoid naming conflicts, its better to provide a namespace.
If the instance(s) happens to be a part of collection(s), they are
are added to the appropriate collections in to_graph as well.
For example, for collection 'C' which the instance happens to be a
part of, given a namespace 'N', the new instance will be a part of
'N/C' in to_graph.
Returns the corresponding instance with respect to to_graph.
copy_graph
TODO: Order of insertion into collections is not preserved
"""
# The name of the new instance
if namespace != '':
new_name = namespace + '/' + org_instance.name
print(new_name)
else:
new_name = org_instance.name
# If an instance of the same name exists, return appropriately
try:
already_present = to_graph.as_graph_element(new_name,
allow_tensor=True,
allow_operation=True)
return already_present
except:
pass
# Get the collections that the new instance needs to be added to.
# The new collections will also be a part of the given namespace.
collections = []
for name, collection in org_instance.graph._collections.items():
if org_instance in collection:
if namespace == '':
collections.append(name)
else:
collections.append(namespace + '/' + name)
# Take action based on the class of the instance
if isinstance(org_instance, tf.Tensor):
# If its a Tensor, it is one of the outputs of the underlying
# op. Therefore, copy the op itself and return the appropriate
# output.
op = org_instance.op
new_op = copy_to_graph(op, to_graph, namespace)
output_index = op.outputs.index(org_instance)
new_tensor = new_op.outputs[output_index]
# Add to collections if any
for collection in collections:
to_graph.add_to_collection(collection, new_tensor)
return new_tensor
elif isinstance(org_instance, tf.IndexedSlices):
values = org_instance.values
indices = org_instance.indices
dense_shape = org_instance.dense_shape
new_values = copy_to_graph(values, to_graph, namespace)
new_indices = copy_to_graph(indices, to_graph, namespace)
new_dense_shape = copy_to_graph(
dense_shape, to_graph,
namespace) if dense_shape is not None else None
return tf.IndexedSlices(new_values, new_indices, new_dense_shape)
elif isinstance(org_instance, tf.Operation):
op = org_instance
# If it has an original_op parameter, copy it
if op._original_op is not None:
new_original_op = copy_to_graph(op._original_op, to_graph,
namespace)
else:
new_original_op = None
# If it has control inputs, call this function recursively on each.
new_control_inputs = [
copy_to_graph(x, to_graph, namespace) for x in op.control_inputs
]
# If it has inputs, call this function recursively on each.
new_inputs = [copy_to_graph(x, to_graph, namespace) for x in op.inputs]
# Make a new node_def based on that of the original.
# An instance of tensorflow.core.framework.graph_pb2.NodeDef, it
# stores String-based info such as name, device and type of the op.
# Unique to every Operation instance.
new_node_def = deepcopy(op._node_def)
# Change the name
new_node_def.name = new_name
# Copy the other inputs needed for initialization
output_types = op._output_types[:]
input_types = op._input_types[:]
# Make a copy of the op_def too.
# Its unique to every _type_ of Operation.
op_def = deepcopy(op._op_def)
# Initialize a new Operation instance
new_op = tf.Operation(new_node_def, to_graph, new_inputs, output_types,
new_control_inputs, input_types, new_original_op,
op_def)
# Use Graph's hidden methods to add the op
to_graph._add_op(new_op)
to_graph._record_op_seen_by_control_dependencies(new_op)
for device_function in reversed(to_graph._device_function_stack):
new_op._set_device(device_function(new_op))
return new_op
else:
raise TypeError("Could not copy instance: " + str(org_instance))
def copy(org_instance,
dict_swap=None,
scope="copied",
replace_itself=False,
copy_q=False,
copy_parent_rvs=True):
"""Build a new node in the TensorFlow graph from `org_instance`,
where any of its ancestors existing in `dict_swap` are
replaced with `dict_swap`'s corresponding value.
Copying is done recursively. Any `Operation` whose output is
required to copy `org_instance` is also copied (if it isn't already
copied within the new scope).
`tf.Variable`s, `tf.placeholder`s, and nodes of type `Queue` are
always reused and not copied. In addition, `tf.Operation`s with
operation-level seeds are copied with a new operation-level seed.
Args:
org_instance: RandomVariable, tf.Operation, tf.Tensor, or tf.Variable.
Node to add in graph with replaced ancestors.
dict_swap: dict.
Random variables, variables, tensors, or operations to swap with.
Its keys are what `org_instance` may depend on, and its values are
the corresponding object (not necessarily of the same class
instance, but must have the same type, e.g., float32) that is used
in exchange.
scope: str.
A scope for the new node(s). This is used to avoid name
conflicts with the original node(s).
replace_itself: bool.
Whether to replace `org_instance` itself if it exists in
`dict_swap`. (This is used for the recursion.)
copy_q: bool.
Whether to copy the replaced tensors too (if not already
copied within the new scope). Otherwise will reuse them.
copy_parent_rvs:
Whether to copy parent random variables `org_instance` depends
on. Otherwise will copy only the sample tensors and not the
random variable class itself.
Returns:
RandomVariable, tf.Variable, tf.Tensor, or tf.Operation.
The copied node.
Raises:
TypeError.
If `org_instance` is not one of the above types.
#### Examples
```python
x = tf.constant(2.0)
y = tf.constant(3.0)
z = x * y
qx = tf.constant(4.0)
# The TensorFlow graph is currently
# `x` -> `z` <- y`, `qx`
# This adds a subgraph with newly copied nodes,
# `qx` -> `copied/z` <- `copied/y`
z_new = ed.copy(z, {x: qx})
sess = tf.Session()
sess.run(z)
6.0
sess.run(z_new)
12.0
```
"""
if not isinstance(org_instance,
(RandomVariable, tf.Operation, tf.Tensor, tf.Variable)):
raise TypeError("Could not copy instance: " + str(org_instance))
if dict_swap is None:
dict_swap = {}
if scope[-1] != '/':
scope += '/'
# Swap instance if in dictionary.
if org_instance in dict_swap and replace_itself:
org_instance = dict_swap[org_instance]
if not copy_q:
return org_instance
elif isinstance(org_instance, tf.Tensor) and replace_itself:
# Deal with case when `org_instance` is the associated tensor
# from the RandomVariable, e.g., `z.value()`. If
# `dict_swap={z: qz}`, we aim to swap it with `qz.value()`.
for key, value in six.iteritems(dict_swap):
if isinstance(key, RandomVariable):
if org_instance == key.value():
if isinstance(value, RandomVariable):
org_instance = value.value()
else:
org_instance = value
if not copy_q:
return org_instance
break
# If instance is a tf.Variable, return it; do not copy any. Note we
# check variables via their name. If we get variables through an
# op's inputs, it has type tf.Tensor and not tf.Variable.
if isinstance(org_instance, (tf.Tensor, tf.Variable)):
for variable in tf.global_variables():
if org_instance.name == variable.name:
if variable in dict_swap and replace_itself:
# Deal with case when `org_instance` is the associated _ref
# tensor for a tf.Variable.
org_instance = dict_swap[variable]
if not copy_q or isinstance(org_instance, tf.Variable):
return org_instance
for variable in tf.global_variables():
if org_instance.name == variable.name:
return variable
break
else:
return variable
graph = tf.get_default_graph()
new_name = scope + org_instance.name
# If an instance of the same name exists, return it.
if isinstance(org_instance, RandomVariable):
for rv in random_variables():
if new_name == rv.name:
return rv
elif isinstance(org_instance, (tf.Tensor, tf.Operation)):
try:
return graph.as_graph_element(new_name,
allow_tensor=True,
allow_operation=True)
except:
pass
# Preserve ordering of random variables. Random variables are always
# copied first (from parent -> child) before any deterministic
# operations that depend on them.
if copy_parent_rvs and \
isinstance(org_instance, (RandomVariable, tf.Tensor, tf.Variable)):
for v in get_parents(org_instance):
copy(v, dict_swap, scope, True, copy_q, True)
if isinstance(org_instance, RandomVariable):
rv = org_instance
# If it has copiable arguments, copy them.
args = [
_copy_default(arg, dict_swap, scope, True, copy_q, False)
for arg in rv._args
]
kwargs = {}
for key, value in six.iteritems(rv._kwargs):
if isinstance(value, list):
kwargs[key] = [
_copy_default(v, dict_swap, scope, True, copy_q, False)
for v in value
]
else:
kwargs[key] = _copy_default(value, dict_swap, scope, True,
copy_q, False)
kwargs['name'] = new_name
# Create new random variable with copied arguments.
try:
new_rv = type(rv)(*args, **kwargs)
except ValueError:
# Handle case where parameters are copied under absolute name
# scope. This can cause an error when creating a new random
# variable as tf.identity name ops are called on parameters ("op
# with name already exists"). To avoid remove absolute name scope.
kwargs['name'] = new_name[:-1]
new_rv = type(rv)(*args, **kwargs)
return new_rv
elif isinstance(org_instance, tf.Tensor):
tensor = org_instance
# Do not copy tf.placeholders.
if 'Placeholder' in tensor.op.type:
return tensor
# A tensor is one of the outputs of its underlying
# op. Therefore copy the op itself.
op = tensor.op
new_op = copy(op, dict_swap, scope, True, copy_q, False)
output_index = op.outputs.index(tensor)
new_tensor = new_op.outputs[output_index]
# Add copied tensor to collections that the original one is in.
for name, collection in six.iteritems(tensor.graph._collections):
if tensor in collection:
graph.add_to_collection(name, new_tensor)
return new_tensor
elif isinstance(org_instance, tf.Operation):
op = org_instance
# Do not copy queue operations.
if 'Queue' in op.type:
return op
# Copy the node def.
# It is unique to every Operation instance. Replace the name and
# its operation-level seed if it has one.
node_def = deepcopy(op.node_def)
node_def.name = new_name
# when copying control flow contexts,
# we need to make sure frame definitions are copied
if 'frame_name' in node_def.attr and node_def.attr[
'frame_name'].s != b'':
node_def.attr['frame_name'].s = (scope.encode('utf-8') +
node_def.attr['frame_name'].s)
if 'seed2' in node_def.attr and tf.get_seed(None)[1] is not None:
node_def.attr['seed2'].i = tf.get_seed(None)[1]
# Copy other arguments needed for initialization.
output_types = op._output_types[:]
# If it has an original op, copy it.
if op._original_op is not None:
original_op = copy(op._original_op, dict_swap, scope, True, copy_q,
False)
else:
original_op = None
# Copy the op def.
# It is unique to every Operation type.
op_def = deepcopy(op.op_def)
new_op = tf.Operation(
node_def,
graph,
[], # inputs; will add them afterwards
output_types,
[], # control inputs; will add them afterwards
[], # input types; will add them afterwards
original_op,
op_def)
# advertise op early to break recursions
graph._add_op(new_op)
# If it has control inputs, copy them.
control_inputs = []
for x in op.control_inputs:
elem = copy(x, dict_swap, scope, True, copy_q, False)
if not isinstance(elem, tf.Operation):
elem = tf.convert_to_tensor(elem)
control_inputs.append(elem)
new_op._add_control_inputs(control_inputs)
# If it has inputs, copy them.
for x in op.inputs:
elem = copy(x, dict_swap, scope, True, copy_q, False)
if not isinstance(elem, tf.Operation):
elem = tf.convert_to_tensor(elem)
new_op._add_input(elem)
# Copy the control flow context.
control_flow_context = _copy_context(op._get_control_flow_context(),
{}, dict_swap, scope, copy_q)
new_op._set_control_flow_context(control_flow_context)
# Use Graph's private methods to add the op, following
# implementation of `tf.Graph().create_op()`.
compute_shapes = True
compute_device = True
op_type = new_name
if compute_shapes:
#set_shapes_for_outputs(new_op)
set_shape_and_handle_data_for_outputs(new_op)
graph._record_op_seen_by_control_dependencies(new_op)
if compute_device:
graph._apply_device_functions(new_op)
if graph._colocation_stack:
all_colocation_groups = []
for colocation_op in graph._colocation_stack:
all_colocation_groups.extend(colocation_op.colocation_groups())
if colocation_op.device:
# Make this device match the device of the colocated op, to
# provide consistency between the device and the colocation
# property.
if new_op.device and new_op.device != colocation_op.device:
logging.warning(
"Tried to colocate %s with an op %s that had "
"a different device: %s vs %s. "
"Ignoring colocation property.", name,
colocation_op.name, new_op.device,
colocation_op.device)
all_colocation_groups = sorted(set(all_colocation_groups))
new_op.node_def.attr["_class"].CopyFrom(
attr_value_pb2.AttrValue(
list=attr_value_pb2.AttrValue.ListValue(
s=all_colocation_groups)))
# Sets "container" attribute if
# (1) graph._container is not None
# (2) "is_stateful" is set in OpDef
# (3) "container" attribute is in OpDef
# (4) "container" attribute is None
if (graph._container and op_type in graph._registered_ops
and graph._registered_ops[op_type].is_stateful
and "container" in new_op.node_def.attr
and not new_op.node_def.attr["container"].s):
new_op.node_def.attr["container"].CopyFrom(
attr_value_pb2.AttrValue(s=compat.as_bytes(graph._container)))
return new_op
else:
raise TypeError("Could not copy instance: " + str(org_instance))
def copy_to_graph(org_instance, to_graph, namespace="", exclude=None): # pylint: disable=too-many-locals, too-many-statements, too-many-branches
"""
Makes a copy of the Operation/Tensor instance 'org_instance'
for the graph 'to_graph', recursively. Therefore, all required
structures linked to org_instance will be automatically copied.
'COPIED_VARIABLES' should be a dict mapping pertinent copied variable
names to the copied instances.
The new instances are automatically inserted into the given 'namespace'.
If namespace='', it is inserted into the graph's global namespace.
However, to avoid naming conflicts, its better to provide a namespace.
If the instance(s) happens to be a part of collection(s), they are
are added to the appropriate collections in to_graph as well.
For example, for collection 'C' which the instance happens to be a
part of, given a namespace 'N', the new instance will be a part of
'N/C' in to_graph.
Returns the corresponding instance with respect to to_graph.
TODO: Order of insertion into collections is not preserved
"""
#print(org_instance.name)
#print(org_instance.name)
####################################################################
if namespace != '':
replica_id = int(namespace.split('_')[1])
else:
replica_id = -1
global COPIED_VARIABLES # pylint: disable=global-statement
if exclude:
for exc in exclude:
if org_instance.name.split(':')[0] == exc.name.split(':')[0]:
return exc
####################################################################
#The name of the new instance
if namespace != '':
new_name = namespace + '/' + org_instance.name
else:
new_name = org_instance.name
#If a variable by the new name already exists, return the
#correspondng tensor that will act as an input
if new_name in COPIED_VARIABLES:
return to_graph.get_tensor_by_name(COPIED_VARIABLES[new_name].name)
#If an instance of the same name exists, return appropriately
try:
already_present = to_graph.as_graph_element(new_name,
allow_tensor=True,
allow_operation=True)
return already_present
except: # pylint: disable=bare-except
pass
#Get the collections that the new instance needs to be added to.
#The new collections will also be a part of the given namespace.
collections = []
for name, collection in org_instance.graph._collections.items(): # pylint: disable=protected-access
if org_instance in collection:
if namespace == '':
collections.append(name)
else:
collections.append(namespace + '/' + name)
#Take action based on the class of the instance
if isinstance(org_instance, tf.Tensor): # pylint: disable=no-else-return
#If its a Tensor, it is one of the outputs of the underlying
#op. Therefore, copy the op itself and return the appropriate
#output.
op = org_instance.op # pylint: disable=invalid-name
new_op = copy_to_graph(op, to_graph, namespace, exclude=exclude)
output_index = op.outputs.index(org_instance)
new_tensor = new_op.outputs[output_index]
#Add to collections if any
for collection in collections:
to_graph.add_to_collection(collection, new_tensor)
return new_tensor
elif isinstance(org_instance, tf.Operation):
op = org_instance # pylint: disable=invalid-name
#If it has an original_op parameter, copy it
if op._original_op is not None: # pylint: disable=protected-access
new_original_op = copy_to_graph(
op._original_op,
to_graph, # pylint: disable=protected-access
namespace,
exclude=exclude)
else:
new_original_op = None
#If it has control inputs, call this function recursively on each.
new_control_inputs = [
copy_to_graph(x, to_graph, namespace, exclude=exclude)
for x in op.control_inputs
]
#If it has inputs, call this function recursively on each.
new_inputs = [
copy_to_graph(x, to_graph, namespace, exclude=exclude)
for x in op.inputs
]
#Make a new node_def based on that of the original.
#An instance of tensorflow.core.framework.graph_pb2.NodeDef, it
#stores String-based info such as name, device and type of the op.
#Unique to every Operation instance.
new_node_def = deepcopy(op.node_def)
#Change the name
new_node_def.name = new_name
#Copy the other inputs needed for initialization
output_types = op._output_types[:] # pylint: disable=protected-access
input_types = op._input_types[:] # pylint: disable=protected-access
#print('name:', new_name)
#print('output_types:',output_types)
#print('input types',input_types)
#print('new inputs', new_inputs)
#print('##########################')
#Make a copy of the op_def too.
#Its unique to every _type_ of Operation.
op_def = deepcopy(op.op_def)
#Initialize a new Operation instance
new_op = tf.Operation(new_node_def, to_graph, new_inputs, output_types,
new_control_inputs, input_types, new_original_op,
op_def)
#Use Graph's hidden methods to add the op
to_graph._add_op(new_op) # pylint: disable=protected-access
to_graph._record_op_seen_by_control_dependencies(new_op) # pylint: disable=protected-access
#print(to_graph._device_function_stack)
for device_function in reversed(to_graph._device_function_stack): # pylint: disable=protected-access
new_op._set_device(device_function(new_op)) # pylint: disable=protected-access
#########################################################
#print(device_function(new_op))
#new_op = PLACER.set_on_gpu(new_op, replica_id)
########################################################
########################################################
if (replica_id >= 0 and 'gpu' in op.device.lower()):
new_op._set_device(_gpu_device_name(replica_id)) # pylint: disable=protected-access
return new_op
########################################################
'''
return new_op
'''
else:
raise TypeError("Could not copy instance: " + str(org_instance))
