def get_paths(base_dir, parser):
"""Gets a list of Paths in a given directory.
Args:
base_dir: directory.
parser: a function which gets the raw Path and can augment it with
information such as the export_version, or ignore the path by returning
None. An example parser may extract the export version from a path
such as "/tmp/exports/100" an another may extract from a full file
name such as "/tmp/checkpoint-99.out".
Returns:
A list of Paths contained in the base directory with the parsing function
applied.
By default the following fields are populated,
- Path.path
The parsing function is responsible for populating,
- Path.export_version
"""
raw_paths = gfile.ListDirectory(base_dir)
paths = []
for r in raw_paths:
p = parser(Path(os.path.join(compat.as_str_any(base_dir),
compat.as_str_any(r)),
None))
if p:
paths.append(p)
return sorted(paths)
def get_matching_files_v2(pattern):
"""Returns a list of files that match the given pattern(s).
Args:
pattern: string or iterable of strings. The glob pattern(s).
Returns:
A list of strings containing filenames that match the given pattern(s).
Raises:
errors.OpError: If there are filesystem / directory listing errors.
"""
with errors.raise_exception_on_not_ok_status() as status:
if isinstance(pattern, six.string_types):
return [
# Convert the filenames to string from bytes.
compat.as_str_any(matching_filename)
for matching_filename in pywrap_tensorflow.GetMatchingFiles(
compat.as_bytes(pattern), status)
]
else:
return [
# Convert the filenames to string from bytes.
compat.as_str_any(matching_filename)
for single_filename in pattern
for matching_filename in pywrap_tensorflow.GetMatchingFiles(
compat.as_bytes(single_filename), status)
]
def get_matching_files_v2(pattern):
"""Returns a list of files that match the given pattern(s).
Args:
pattern: string or iterable of strings. The glob pattern(s).
Returns:
A list of strings containing filenames that match the given pattern(s).
Raises:
errors.OpError: If there are filesystem / directory listing errors.
"""
if isinstance(pattern, six.string_types):
return [
# Convert the filenames to string from bytes.
compat.as_str_any(matching_filename)
for matching_filename in pywrap_tensorflow.GetMatchingFiles(
compat.as_bytes(pattern))
]
else:
return [
# Convert the filenames to string from bytes.
compat.as_str_any(matching_filename) # pylint: disable=g-complex-comprehension
for single_filename in pattern
for matching_filename in pywrap_tensorflow.GetMatchingFiles(
compat.as_bytes(single_filename))
]
def request_stop(self, ex=None):
"""Request that the threads stop.
After this is called, calls to `should_stop()` will return `True`.
Note: If an exception is being passed in, in must be in the context of
handling the exception (i.e. `try: ... except Exception as ex: ...`) and not
a newly created one.
Args:
ex: Optional `Exception`, or Python `exc_info` tuple as returned by
`sys.exc_info()`. If this is the first call to `request_stop()` the
corresponding exception is recorded and re-raised from `join()`.
"""
with self._lock:
ex = self._filter_exception(ex)
# If we have already joined the coordinator the exception will not have a
# chance to be reported, so just raise it normally. This can happen if
# you continue to use a session have having stopped and joined the
# coordinator threads.
if self._joined:
if isinstance(ex, tuple):
six.reraise(*ex)
elif ex is not None:
# NOTE(touts): This is bogus if request_stop() is not called
# from the exception handler that raised ex.
six.reraise(*sys.exc_info())
if not self._stop_event.is_set():
if ex and self._exc_info_to_raise is None:
if isinstance(ex, tuple):
logging.info("Error reported to Coordinator: %s, %s",
type(ex[1]),
compat.as_str_any(ex[1]))
self._exc_info_to_raise = ex
else:
logging.info("Error reported to Coordinator: %s, %s",
type(ex),
compat.as_str_any(ex))
self._exc_info_to_raise = sys.exc_info()
# self._exc_info_to_raise should contain a tuple containing exception
# (type, value, traceback)
if (len(self._exc_info_to_raise) != 3 or
not self._exc_info_to_raise[0] or
not self._exc_info_to_raise[1]):
# Raise, catch and record the exception here so that error happens
# where expected.
try:
raise ValueError(
"ex must be a tuple or sys.exc_info must return the current "
"exception: %s"
% self._exc_info_to_raise)
except ValueError:
# Record this error so it kills the coordinator properly.
# NOTE(touts): As above, this is bogus if request_stop() is not
# called from the exception handler that raised ex.
self._exc_info_to_raise = sys.exc_info()
self._stop_event.set()
def test_add_pruned_collection_proto_in_bytes_list(self):
# Note: This also tests _is_removed_mentioned().
collection_name = 'proto_collection'
base_meta_graph_def = meta_graph_pb2.MetaGraphDef()
base_meta_graph_def.collection_def[collection_name].bytes_list.value.extend(
[compat.as_bytes(compat.as_str_any(_make_asset_file_def_any('node1'))),
compat.as_bytes(compat.as_str_any(_make_asset_file_def_any('node2'))),
compat.as_bytes(compat.as_str_any(_make_asset_file_def_any('node3'))),
compat.as_bytes(compat.as_str_any(_make_asset_file_def_any('node4'))),
compat.as_bytes(compat.as_str_any(_make_asset_file_def_any('/a/a_1'))),
compat.as_bytes(compat.as_str_any(_make_asset_file_def_any('/b/b_1')))
])
meta_graph_def = meta_graph_pb2.MetaGraphDef()
removed_op_names = ['node2', 'node4', 'node5', '/a', '/b/b_1']
meta_graph_transform._add_pruned_collection(
base_meta_graph_def, meta_graph_def, collection_name, removed_op_names)
collection = meta_graph_def.collection_def[collection_name]
expected_values = [
compat.as_bytes(compat.as_str_any(_make_asset_file_def_any('node1'))),
compat.as_bytes(compat.as_str_any(_make_asset_file_def_any('node3'))),
compat.as_bytes(compat.as_str_any(_make_asset_file_def_any('/a/a_1'))),
]
self.assertEqual(expected_values, collection.bytes_list.value[:])
def parser(path):
# Modify the path object for RegEx match for Windows Paths
if os.name == 'nt':
match = re.match("^" + compat.as_str_any(base_dir).replace('\\','/') + "/(\\d+)$",
compat.as_str_any(path.path).replace('\\','/'))
else:
match = re.match("^" + compat.as_str_any(base_dir) + "/(\\d+)$",
compat.as_str_any(path.path))
if not match:
return None
return path._replace(export_version=int(match.group(1)))
def test_asset_path_returned(self):
root = tracking.AutoTrackable()
root.path = tracking.TrackableAsset(self._vocab_path)
save_dir = os.path.join(self.get_temp_dir(), "saved_model")
root.get_asset = def_function.function(lambda: root.path.asset_path)
save.save(root, save_dir, signatures=root.get_asset.get_concrete_function())
second_dir = os.path.join(self.get_temp_dir(), "second_dir")
file_io.rename(save_dir, second_dir)
imported_path = _import_and_infer(second_dir, {})["output_0"]
self.assertIn(compat.as_str_any(second_dir),
compat.as_str_any(imported_path))
def _export_eval_result(self, eval_result, checkpoint_path):
"""Export `eval_result` according to strategies in `EvalSpec`."""
export_dir_base = os.path.join(
compat.as_str_any(self._estimator.model_dir),
compat.as_str_any('export'))
for strategy in self._eval_spec.export_strategies:
strategy.export(self._estimator,
os.path.join(
compat.as_str_any(export_dir_base),
compat.as_str_any(strategy.name)),
checkpoint_path=checkpoint_path,
eval_result=eval_result)
def test_asset_path_returned(self):
root = tracking.AutoTrackable()
root.path = tracking.Asset(self._vocab_path)
save_dir = os.path.join(self.get_temp_dir(), "saved_model")
root.get_asset = def_function.function(lambda: root.path.asset_path)
save.save(root,
save_dir,
signatures=root.get_asset.get_concrete_function())
second_dir = os.path.join(self.get_temp_dir(), "second_dir")
file_io.rename(save_dir, second_dir)
imported_path = _import_and_infer(second_dir, {})["output_0"]
self.assertIn(compat.as_str_any(second_dir),
compat.as_str_any(imported_path))
def parser(path):
# Modify the path object for RegEx match for Windows Paths
if os.name == "nt":
match = re.match(
r"^" + compat.as_str_any(base_dir).replace("\\", "/") +
r"/(\d+)$",
compat.as_str_any(path.path).replace("\\", "/"))
else:
match = re.match(r"^" + compat.as_str_any(base_dir) + r"/(\d+)$",
compat.as_str_any(path.path))
if not match:
return None
return path._replace(export_version=int(match.group(1)))
def _add_mean(op, context):
input_name = compat.as_str_any(op.inputs[0].name)
output_name = compat.as_str_any(op.outputs[0].name)
axis_ind = context.consts[op.inputs[1].name]
input_shape = context.shape_dict[input_name]
output_shape = context.shape_dict[output_name]
if context.use_dfs_shape_infer:
status = interpret_shape(input_name, context)
else:
status = False
if status:
labeled_shape = context.dim_labels[input_name]
if isinstance(axis_ind, np.ndarray):
axis = ''
for i in axis_ind:
if input_shape[i] != 1:
axis += labeled_shape[i]
axis = ''.join(sorted(axis))
else:
axis = labeled_shape[axis_ind]
assert axis in ['S', 'C', 'H', 'W', 'CHW', 'HW'], (
'Axis value %s not supported. '
'Reduction supported along C, H, W, HW, CHW dimensions only.' % axis)
else:
if len(input_shape) == 4 and (
np.array_equal(axis_ind, np.array([0, 1, 2])) or
np.array_equal(axis_ind, np.array([1, 2]))):
axis = 'HW'
else:
assert False, ('Mean axis case not handled currently. '
'Input shape = {}, output shape = {}, axis_ind = {}'.
format(str(input_shape), str(output_shape), str(axis_ind)))
mode = 'avg'
# The simple case; reduction along non sequence axis
if axis != 'S':
context.builder.add_reduce(output_name, input_name, output_name, axis, mode)
# Need to permute, reduce and then permute back
else:
context.builder.add_permute(
output_name, (1, 0, 2, 3), input_name, output_name + '_swap_Seq_C')
context.builder.add_reduce(
output_name, output_name + '_swap_Seq_C',
output_name + '_pre_permute', 'C', mode)
context.builder.add_permute(
output_name, (1, 0, 2, 3), output_name + '_pre_permute', output_name)
context.translated[output_name] = True
def _export_eval_result(self, eval_result, checkpoint_path):
"""Export `eval_result` according to strategies in `EvalSpec`."""
export_dir_base = os.path.join(
compat.as_str_any(self._estimator.model_dir),
compat.as_str_any('export'))
for strategy in self._eval_spec.export_strategies:
strategy.export(
self._estimator,
os.path.join(
compat.as_str_any(export_dir_base),
compat.as_str_any(strategy.name)),
checkpoint_path=checkpoint_path,
eval_result=eval_result)
def _export_eval_result(self, eval_result, is_the_final_export):
"""Export `eval_result` according to exporters in `EvalSpec`."""
export_dir_base = os.path.join(
compat.as_str_any(self._estimator.model_dir),
compat.as_str_any('export'))
for exporter in self._eval_spec.exporters:
exporter.export(estimator=self._estimator,
export_path=os.path.join(
compat.as_str_any(export_dir_base),
compat.as_str_any(exporter.name)),
checkpoint_path=eval_result.checkpoint_path,
eval_result=eval_result.metrics,
is_the_final_export=is_the_final_export)
def _export_eval_result(self, eval_result, is_the_final_export):
"""Export `eval_result` according to exporters in `EvalSpec`."""
export_dir_base = os.path.join(
compat.as_str_any(self._estimator.model_dir),
compat.as_str_any('export'))
for exporter in self._eval_spec.exporters:
exporter.export(
estimator=self._estimator,
export_path=os.path.join(
compat.as_str_any(export_dir_base),
compat.as_str_any(exporter.name)),
checkpoint_path=eval_result.checkpoint_path,
eval_result=eval_result.metrics,
is_the_final_export=is_the_final_export)
def strided_slice(op, context):
input_name = compat.as_str_any(op.inputs[0].name)
output_name = compat.as_str_any(op.outputs[0].name)
make_tensor(op.inputs[0], context)
[x, y] = context.session.run([input_name, output_name],
feed_dict=context.input_feed_dict)
assert op.inputs[1].name in context.consts, \
'Strided Slice: begin index must be a constant'
assert op.inputs[2].name in context.consts, \
'Strided Slice: end index must be a constant'
assert op.inputs[3].name in context.consts, \
'Strided Slice: strides must be a constant'
begin = context.consts[compat.as_str_any(op.inputs[1].name)]
end = context.consts[compat.as_str_any(op.inputs[2].name)]
strides = context.consts[compat.as_str_any(op.inputs[3].name)]
begin_mask = op.get_attr('begin_mask')
end_mask = op.get_attr('end_mask')
ellipsis_mask = op.get_attr('ellipsis_mask')
new_axis_mask = op.get_attr('new_axis_mask')
shrink_axis_mask = op.get_attr('shrink_axis_mask')
input_shape = context.shape_dict[input_name]
if len(input_shape) == 1 and len(begin) == 1 and len(end) == 1 and \
len(strides) == 1:
if begin_mask:
begin[0] = 0
if end_mask:
end[0] = input_shape[0]
context.builder.add_slice(
output_name, input_name, output_name,
'channel', begin[0], end[0], strides[0])
elif len(x.shape) == 4 and len(y.shape) == 3 and x.shape[:3] == y.shape:
context.builder.add_slice(
output_name, input_name, output_name,
'channel', begin[-1], end[-1], 1)
elif input_name in context.consts:
#this means all the inputs to the strided slice layer are constant
add_const(context, output_name, y, output_name)
elif np.array_equal(np.squeeze(x),np.squeeze(y)):
skip(op,context)
else:
assert False, 'Strided Slice case not handled'
context.translated[output_name] = True
def lrn(op, context):
input_name = compat.as_str_any(op.inputs[0].name)
output_name = compat.as_str_any(op.outputs[0].name)
input_shape = context.shape_dict[input_name]
C = input_shape[-1]
alpha = op.get_attr('alpha')
beta = op.get_attr('beta')
bias = op.get_attr('bias')
depth_radius = op.get_attr('depth_radius')
context.builder.add_lrn(output_name, input_name, output_name,
alpha=alpha * C,
beta=beta,
local_size=depth_radius,
k=bias)
context.translated[output_name] = True
def mul(op, context):
output_name = compat.as_str_any(op.outputs[0].name)
# input_names: names of input tensors
input_names = [make_tensor(ts, context) for ts in op.inputs]
# input_shapes: shapes of input tensors
input_shapes = [context.shape_dict[ts.name] for ts in op.inputs]
mult_input_names = input_names
# For rank-4 inputs, CoreML only allows [1], [C], [1,H,W] blobs to be
# broadcasted in elementwise operations. To handle other broadcasting cases,
# (e.g. [1,1,W] --> [C,H,W]), we insert up-sampling layers
input_ranks = [len(shape) for shape in input_shapes]
if 4 in input_ranks:
broadcasted_shape4 = _get_broadcasted_shape4(input_shapes)
for idx, in_name in enumerate(input_names):
input_shape = input_shapes[idx]
axis = _broadcast_axis(broadcasted_shape4, input_shape)
if axis is not None:
# add upsample layer
upsampled_in_name = in_name + '__upsampled'
mult_input_names[idx] = upsampled_in_name
input_axis_dim = 1 if axis >= len(input_shape) else input_shape[axis]
scale = broadcasted_shape4[axis] // input_axis_dim
if axis == 1:
context.builder.add_upsample(
upsampled_in_name, scale, 1, in_name, upsampled_in_name)
else:
context.builder.add_upsample(
upsampled_in_name, 1, scale, in_name, upsampled_in_name)
context.builder.add_elementwise(
output_name, mult_input_names, output_name, 'MULTIPLY')
context.translated[output_name] = True
def _is_removed_mentioned(s, removed_op_names):
"""Determine whether any removed op is mentioned in the given object.
This relies on the string representation of the object. This is used for
proto messages that may mention ops by name in nested fields. The string
representation of the proto includes those field values, so this string
search approach is sufficient.
Args:
s: an object to search for removed op names.
removed_op_names: An iterable of names of ops that were removed.
Returns:
True if any removed op is mentioned in the given object, False otherwise.
"""
# A common approach taken by some of the transforms in gtt is to add new nodes
# that have the same prefix as the node they are removing. For example, if
# the original node name was /foo, they may remove that node and add in
# /foo/bar. This regex ensures that we handle these two nodes
# as separate entities. It matches on nodes having names in the form of
# '/foo/bar_x' as well as nodes having names in the form of 'foo.'
s_names = _re.findall(r'((?:[\/]?[a-zA-Z0-9\_]*)*)', compat.as_str_any(s))
for removed_op_name in removed_op_names:
for s_name in s_names:
if s_name.endswith(removed_op_name):
return True
return False
def _normalize_outputs(outputs, function_name, signature_key):
"""Construct an output dictionary from unnormalized function outputs."""
if isinstance(outputs, collections.Mapping):
for key, value in outputs.items():
if not isinstance(value, ops.Tensor):
raise ValueError(
("Got a dictionary containing non-Tensor value {} for key {} "
"in the output of the function {} used to generate a SavedModel "
"signature. Dictionaries outputs for functions used as signatures "
"should have one Tensor output per string key.")
.format(value, key, compat.as_str_any(function_name)))
return outputs
else:
original_outputs = outputs
if not isinstance(outputs, collections.Sequence):
outputs = [outputs]
if not _is_flat(outputs):
raise ValueError(
("Got non-flat outputs '{}' from '{}' for SavedModel "
"signature '{}'. Signatures have one Tensor per output, so "
"to have predictable names Python functions used to generate "
"these signatures should avoid outputting Tensors in nested "
"structures.")
.format(original_outputs, function_name, signature_key))
return {("output_{}".format(output_index)): output
for output_index, output
in enumerate(outputs)}
def get_summary_description(node_def):
"""Given a TensorSummary node_def, retrieve its SummaryDescription.
When a Summary op is instantiated, a SummaryDescription of associated
metadata is stored in its NodeDef. This method retrieves the description.
Args:
node_def: the node_def_pb2.NodeDef of a TensorSummary op
Returns:
a summary_pb2.SummaryDescription
Raises:
ValueError: if the node is not a summary op.
@compatibility(eager)
Not compatible with eager execution. To write TensorBoard
summaries under eager execution, use `tf.contrib.summary` instead.
@end_compatibility
"""
if node_def.op != 'TensorSummary':
raise ValueError("Can't get_summary_description on %s" % node_def.op)
description_str = _compat.as_str_any(node_def.attr['description'].s)
summary_description = SummaryDescription()
_json_format.Parse(description_str, summary_description)
return summary_description
def get_summary_description(node_def):
"""Given a TensorSummary node_def, retrieve its SummaryDescription.
When a Summary op is instantiated, a SummaryDescription of associated
metadata is stored in its NodeDef. This method retrieves the description.
Args:
node_def: the node_def_pb2.NodeDef of a TensorSummary op
Returns:
a summary_pb2.SummaryDescription
Raises:
ValueError: if the node is not a summary op.
@compatibility(eager)
Not compatible with eager execution. To write TensorBoard
summaries under eager execution, use `tf.contrib.summary` instead.
@end_compatbility
"""
if node_def.op != 'TensorSummary':
raise ValueError("Can't get_summary_description on %s" % node_def.op)
description_str = _compat.as_str_any(node_def.attr['description'].s)
summary_description = SummaryDescription()
_json_format.Parse(description_str, summary_description)
return summary_description
def join(path, *paths):
r"""Join one or more path components intelligently.
TensorFlow specific filesystems will be joined
like a url (using "/" as the path seperator) on all platforms:
On Windows or Linux/Unix-like:
>>> tf.io.gfile.join("gcs://folder", "file.py")
'gcs://folder/file.py'
>>> tf.io.gfile.join("ram://folder", "file.py")
'ram://folder/file.py'
But the native filesystem is handled just like os.path.join:
>>> path = tf.io.gfile.join("folder", "file.py")
>>> if os.name == "nt":
... expected = "folder\\file.py" # Windows
... else:
... expected = "folder/file.py" # Linux/Unix-like
>>> path == expected
True
Args:
path: string, path to a directory
paths: string, additional paths to concatenate
Returns:
path: the joined path.
"""
# os.path.join won't take mixed bytes/str, so don't overwrite the incoming `path` var
path_ = compat.as_str_any(compat.path_to_str(path))
if "://" in path_[1:]:
return urljoin(path, *paths)
return os.path.join(path, *paths)
def list_directory_v2(path):
"""Returns a list of entries contained within a directory.
The list is in arbitrary order. It does not contain the special entries "."
and "..".
Args:
path: string, path to a directory
Returns:
[filename1, filename2, ... filenameN] as strings
Raises:
errors.NotFoundError if directory doesn't exist
"""
if not is_directory(path):
raise errors.NotFoundError(
node_def=None,
op=None,
message="Could not find directory {}".format(path))
with errors.raise_exception_on_not_ok_status() as status:
# Convert each element to string, since the return values of the
# vector of string should be interpreted as strings, not bytes.
return [
compat.as_str_any(filename)
for filename in pywrap_tensorflow.GetChildren(
compat.as_bytes(path), status)
]
def list_directory_v2(path):
"""Returns a list of entries contained within a directory.
The list is in arbitrary order. It does not contain the special entries "."
and "..".
Args:
path: string, path to a directory
Returns:
[filename1, filename2, ... filenameN] as strings
Raises:
errors.NotFoundError if directory doesn't exist
"""
if not is_directory(path):
raise errors.NotFoundError(
node_def=None,
op=None,
message="Could not find directory {}".format(path))
# Convert each element to string, since the return values of the
# vector of string should be interpreted as strings, not bytes.
return [
compat.as_str_any(filename)
for filename in pywrap_tensorflow.GetChildren(compat.as_bytes(path))
]
def _is_removed_mentioned(s, removed_op_names):
"""Determine whether any removed op is mentioned in the given object.
This relies on the string representation of the object. This is used for
proto messages that may mention ops by name in nested fields. The string
representation of the proto includes those field values, so this string
search approach is sufficient.
Args:
s: an object to search for removed op names.
removed_op_names: An iterable of names of ops that were removed.
Returns:
True if any removed op is mentioned in the given object, False otherwise.
"""
# A common approach taken by some of the transforms in gtt is to add new nodes
# that have the same prefix as the node they are removing. For example, if
# the original node name was /foo, they may remove that node and add in
# /foo/bar. This regex ensures that we handle these two nodes
# as separate entities. It matches on nodes having names in the form of
# '/foo/bar_x' as well as nodes having names in the form of 'foo.'
s_names = _re.findall(r'((?:[\/]?[a-zA-Z0-9\_]*)*)', compat.as_str_any(s))
for removed_op_name in removed_op_names:
for s_name in s_names:
if s_name.endswith(removed_op_name):
return True
return False
def _normalize_outputs(outputs, function_name, signature_key):
"""Construct an output dictionary from unnormalized function outputs."""
if isinstance(outputs, collections.Mapping):
for key, value in outputs.items():
if not isinstance(value, ops.Tensor):
raise ValueError((
"Got a dictionary containing non-Tensor value {} for key {} "
"in the output of the function {} used to generate a SavedModel "
"signature. Dictionaries outputs for functions used as signatures "
"should have one Tensor output per string key.").format(
value, key, compat.as_str_any(function_name)))
return outputs
else:
original_outputs = outputs
if not isinstance(outputs, collections.Sequence):
outputs = [outputs]
if not _is_flat(outputs):
raise ValueError(
("Got non-flat outputs '{}' from '{}' for SavedModel "
"signature '{}'. Signatures have one Tensor per output, so "
"to have predictable names Python functions used to generate "
"these signatures should avoid outputting Tensors in nested "
"structures.").format(original_outputs, function_name,
signature_key))
return {("output_{}".format(output_index)): output
for output_index, output in enumerate(outputs)}
def list_directory(dirname):
"""Returns a list of entries contained within a directory.
The list is in arbitrary order. It does not contain the special entries "."
and "..".
Args:
dirname: string, path to a directory
Returns:
[filename1, filename2, ... filenameN] as strings
Raises:
errors.NotFoundError if directory doesn't exist
"""
if not is_directory(dirname):
raise errors.NotFoundError(None, None, "Could not find directory")
with errors.raise_exception_on_not_ok_status() as status:
# Convert each element to string, since the return values of the
# vector of string should be interpreted as strings, not bytes.
return [
compat.as_str_any(filename)
for filename in pywrap_tensorflow.GetChildren(
compat.as_bytes(dirname), status)
]
def identity(op, context, input_name = None):
is_network_output = False
for out in op.outputs:
if out.name in context.output_names:
is_network_output = True
break
if input_name is None:
input_name = compat.as_str_any(op.inputs[0].name)
for out in op.outputs:
output_name = compat.as_str_any(out.name)
if op.inputs[0].op.type != 'Const':
if is_network_output:
context.builder.add_activation(
output_name, 'LINEAR', input_name, output_name, [1.0, 0])
else:
skip(op, context)
context.translated[output_name] = True
def real_div(op, context):
output_name = compat.as_str_any(op.outputs[0].name)
input_names = []
for inp in op.inputs:
input_names.append(make_tensor(inp, context))
add_tensor_div(
context.builder, output_name, input_names[0], input_names[1], output_name)
context.translated[output_name] = True
def IsTensorFlowEventsFile(path):
"""Check the path name to see if it is probably a TF Events file."""
if 'tfevents' not in compat.as_str_any(os.path.basename(path)):
return False
if _CNS_DELETED_FILE_PATTERN.search(path):
logging.info('Ignoring deleted Colossus file: %s', path)
return False
return True
def check(op, context):
for inp in op.inputs:
inp_name = compat.as_str_any(inp.name)
assert inp_name in context.translated, (
'No translation found for {}'.format(inp_name))
for out in op.outputs:
assert out.name in context.shape_dict, (
'Shape for {} is not fully defined'.format(out.name))
def translation_required(op, context):
for out in op.outputs:
out_name = compat.as_str_any(out.name)
if out_name in context.translated:
continue
else:
return True
return False
def sub(op, context):
assert len(op.inputs) == 2, 'Sub op currently supports only two inputs'
output_name = compat.as_str_any(op.outputs[0].name)
input_1_name = make_tensor(op.inputs[0], context)
input_2_name = make_tensor(op.inputs[1], context)
add_tensor_sub(
context.builder, output_name, input_1_name, input_2_name, output_name)
context.translated[output_name] = True
def request_stop(self, ex=None):
"""Request that the threads stop.
After this is called, calls to `should_stop()` will return `True`.
Note: If an exception is being passed in, in must be in the context of
handling the exception (i.e. `try: ... except Exception as ex: ...`) and not
a newly created one.
Args:
ex: Optional `Exception`, or Python `exc_info` tuple as returned by
`sys.exc_info()`. If this is the first call to `request_stop()` the
corresponding exception is recorded and re-raised from `join()`.
"""
ex = self._filter_exception(ex)
with self._lock:
if not self._stop_event.is_set():
if ex and self._exc_info_to_raise is None:
if isinstance(ex, tuple):
logging.info("Error reported to Coordinator: %s, %s",
type(ex[1]),
compat.as_str_any(ex[1]))
self._exc_info_to_raise = ex
else:
logging.info("Error reported to Coordinator: %s, %s",
type(ex),
compat.as_str_any(ex))
self._exc_info_to_raise = sys.exc_info()
# self._exc_info_to_raise should contain a tuple containing exception
# (type, value, traceback)
if (len(self._exc_info_to_raise) != 3 or
not self._exc_info_to_raise[0] or
not self._exc_info_to_raise[1]):
# Raise, catch and record the exception here so that error happens
# where expected.
try:
raise ValueError(
"ex must be a tuple or sys.exc_info must return the current "
"exception: %s"
% self._exc_info_to_raise)
except ValueError:
# Record this error so it kills the coordinator properly.
self._exc_info_to_raise = sys.exc_info()
self._stop_event.set()
def request_stop(self, ex=None):
"""Request that the threads stop.
After this is called, calls to `should_stop()` will return `True`.
Note: If an exception is being passed in, in must be in the context of
handling the exception (i.e. `try: ... except Exception as ex: ...`) and not
a newly created one.
Args:
ex: Optional `Exception`, or Python `exc_info` tuple as returned by
`sys.exc_info()`. If this is the first call to `request_stop()` the
corresponding exception is recorded and re-raised from `join()`.
"""
ex = self._filter_exception(ex)
with self._lock:
if not self._stop_event.is_set():
if ex and self._exc_info_to_raise is None:
if isinstance(ex, tuple):
logging.info("Error reported to Coordinator: %s, %s",
type(ex[1]),
compat.as_str_any(ex[1]))
self._exc_info_to_raise = ex
else:
logging.info("Error reported to Coordinator: %s, %s",
type(ex),
compat.as_str_any(ex))
self._exc_info_to_raise = sys.exc_info()
# self._exc_info_to_raise should contain a tuple containing exception
# (type, value, traceback)
if (len(self._exc_info_to_raise) != 3 or
not self._exc_info_to_raise[0] or
not self._exc_info_to_raise[1]):
# Raise, catch and record the exception here so that error happens
# where expected.
try:
raise ValueError(
"ex must be a tuple or sys.exc_info must return the current "
"exception: %s"
% self._exc_info_to_raise)
except ValueError:
# Record this error so it kills the coordinator properly.
self._exc_info_to_raise = sys.exc_info()
self._stop_event.set()
def product(op, context):
input_name = compat.as_str_any(op.inputs[0].name)
output_name = compat.as_str_any(op.outputs[0].name)
start_ind = context.consts[op.inputs[1].name]
assert start_ind == 0, 'Prod: only start index = 0 case supported'
input_shape = context.shape_dict[input_name]
if len(input_shape) == 1:
axis = 'C'
else:
assert False, 'Reduce Sum axis case not handled currently'
mode = 'prod'
context.translated[output_name] = True
context.builder.add_reduce(output_name, input_name, output_name, axis, mode)
def mirror_pad(op, context):
input_name = compat.as_str_any(op.inputs[0].name)
output_name = compat.as_str_any(op.outputs[0].name)
paddings = context.consts[op.inputs[1].name]
top = paddings[1][0]
bottom = paddings[1][1]
left = paddings[2][0]
right = paddings[2][1]
assert compat.as_str_any(op.get_attr('mode')) != 'SYMMETRIC', \
'symmetric mode is not supported by Core ML'
context.translated[output_name] = True
context.builder.add_padding(
output_name, left, right, top, bottom,
input_name=input_name, output_name=output_name,
padding_type='reflection')
def testMixedStrTypes(self):
temp_dir = compat.as_bytes(test.get_temp_dir())
for sub_dir in ['str', b'bytes', u'unicode']:
base_dir = os.path.join((temp_dir if isinstance(sub_dir, bytes)
else temp_dir.decode()), sub_dir)
self.assertFalse(gfile.Exists(base_dir))
gfile.MakeDirs(os.path.join(compat.as_str_any(base_dir), "42"))
gc.get_paths(base_dir, _create_parser(base_dir))
def argmax(op, context):
input_name = compat.as_str_any(op.inputs[0].name)
output_name = compat.as_str_any(op.outputs[0].name)
input_shape = context.shape_dict[input_name]
axis_tensor = compat.as_str_any(op.inputs[1].name)
if axis_tensor in context.consts:
axis_tf = context.consts[axis_tensor]
else:
assert False, 'ArgMax: Axis tensor not found in the list of Consts'
if len(input_shape) == 4 and axis_tf == 3:
axis = 'C'
else:
assert False, 'ArgMax: Axis translation case not handled currently'
context.builder.add_reduce(
output_name, input_name, output_name, axis, 'argmax')
context.translated[output_name] = True
def extract_image_patches(op, context):
# use a big convolution layer (that has weights!) for this op
input_name = compat.as_str_any(op.inputs[0].name)
output_name = compat.as_str_any(op.outputs[0].name)
ksizes = op.get_attr('ksizes')
padding_type = compat.as_str_any(op.get_attr('padding'))
if padding_type == 'VALID':
padding_type = 'valid'
elif padding_type == 'SAME':
padding_type = 'same'
else:
raise NotImplementedError('%s not implemented' %(padding_type))
strides = op.get_attr('strides')
rates = op.get_attr('rates')
assert rates == [1] * len(rates), 'Only supports when rates are all 1s'
kh, kw = ksizes[1], ksizes[2]
sh, sw = strides[1], strides[2]
c_in = context.shape_dict[input_name][-1]
n_filters = kh * kw * c_in
W = np.zeros((kh, kw, c_in, n_filters))
for i_h in range(kh):
for i_w in range(kw):
for i_c in range(c_in):
idx = i_c + (i_w * c_in) + (i_h * c_in * kw)
W[i_h, i_w, i_c, idx] = 1
context.builder.add_convolution(name=output_name,
kernel_channels=c_in,
output_channels=n_filters,
height=kh,
width=kw,
stride_height=sh,
stride_width=sw,
border_mode=padding_type,
groups=1,
W=W,
b=None,
has_bias=False,
is_deconv=False,
output_shape=None,
input_name=input_name,
output_name=output_name)
context.translated[output_name] = True
def testMixedStrTypes(self):
temp_dir = compat.as_bytes(test.get_temp_dir())
for sub_dir in ["str", b"bytes", u"unicode"]:
base_dir = os.path.join((temp_dir if isinstance(sub_dir, bytes)
else temp_dir.decode()), sub_dir)
self.assertFalse(gfile.Exists(base_dir))
gfile.MakeDirs(os.path.join(compat.as_str_any(base_dir), "42"))
gc._get_paths(base_dir, _create_parser(base_dir))
gfile.DeleteRecursively(base_dir)
def list_directory(dirname):
"""Returns a list of entries contained within a directory.
The list is in arbitrary order. It does not contain the special entries "."
and "..".
Args:
dirname: string, path to a directory
Returns:
[filename1, filename2, ... filenameN] as strings
Raises:
errors.NotFoundError if directory doesn't exist
"""
if not is_directory(dirname):
raise errors.NotFoundError(None, None, "Could not find directory")
file_list = get_matching_files(os.path.join(compat.as_str_any(dirname), "*"))
return [compat.as_str_any(pywrap_tensorflow.Basename(compat.as_bytes(filename))) for filename in file_list]
def testMixedStrTypes(self):
temp_dir = compat.as_bytes(test.get_temp_dir())
for sub_dir in ["str", b"bytes", u"unicode"]:
base_dir = os.path.join(
(temp_dir
if isinstance(sub_dir, bytes) else temp_dir.decode()), sub_dir)
self.assertFalse(gfile.Exists(base_dir))
gfile.MakeDirs(os.path.join(compat.as_str_any(base_dir), "42"))
gc.get_paths(base_dir, _create_parser(base_dir))
def walk_v2(top, topdown=True, onerror=None):
"""Recursive directory tree generator for directories.
Args:
top: string, a Directory name
topdown: bool, Traverse pre order if True, post order if False.
onerror: optional handler for errors. Should be a function, it will be
called with the error as argument. Rethrowing the error aborts the walk.
Errors that happen while listing directories are ignored.
Yields:
Each yield is a 3-tuple: the pathname of a directory, followed by lists of
all its subdirectories and leaf files. That is, each yield looks like:
`(dirname, [subdirname, subdirname, ...], [filename, filename, ...])`.
Each item is a string.
"""
def _make_full_path(parent, item):
# Since `join` discards paths before one that starts with the path
# separator (https://docs.python.org/3/library/os.path.html#join),
# we have to manually handle that case as `/` is a valid character on GCS.
if item[0] == os.sep:
return "".join([join(parent, ""), item])
return join(parent, item)
top = compat.as_str_any(compat.path_to_str(top))
try:
listing = list_directory(top)
except errors.NotFoundError as err:
if onerror:
onerror(err)
else:
return
files = []
subdirs = []
for item in listing:
full_path = _make_full_path(top, item)
if is_directory(full_path):
subdirs.append(item)
else:
files.append(item)
here = (top, subdirs, files)
if topdown:
yield here
for subdir in subdirs:
for subitem in walk_v2(_make_full_path(top, subdir),
topdown,
onerror=onerror):
yield subitem
if not topdown:
yield here
def request_stop(self, ex=None):
"""Request that the threads stop.
After this is called, calls to `should_stop()` will return `True`.
Args:
ex: Optional `Exception`, or Python `exc_info` tuple as returned by
`sys.exc_info()`. If this is the first call to `request_stop()` the
corresponding exception is recorded and re-raised from `join()`.
"""
with self._lock:
if not self._stop_event.is_set():
if ex and self._exc_info_to_raise is None:
if isinstance(ex, tuple):
logging.info("Error reported to Coordinator: %s", compat.as_str_any(ex[1]))
self._exc_info_to_raise = ex
else:
logging.info("Error reported to Coordinator: %s", compat.as_str_any(ex))
self._exc_info_to_raise = sys.exc_info()
self._stop_event.set()
def relu6(op, context):
input_name = compat.as_str_any(op.inputs[0].name)
output_name = compat.as_str_any(op.outputs[0].name)
relu_output_name = 'relu_' + output_name
context.builder.add_activation(
relu_output_name, 'RELU', input_name, relu_output_name)
neg_output_name = relu_output_name + '_neg'
# negate it
context.builder.add_activation(
neg_output_name, 'LINEAR', relu_output_name, neg_output_name, [-1.0, 0])
# apply threshold
clip_output_name = relu_output_name + '_clip'
context.builder.add_unary(
clip_output_name, neg_output_name, clip_output_name, 'threshold',
alpha=-6.0)
# negate it back
context.builder.add_activation(
output_name, 'LINEAR', clip_output_name, output_name, [-1.0, 0])
context.translated[output_name] = True
def placeholder(op, context):
context.translated[compat.as_str_any(op.outputs[0].name)] = True
try:
inname = op.inputs[0].name
# chain together no-ops here
if inname in context.out_name_to_in_name:
context.out_name_to_in_name[op.outputs[0].name] = (
context.out_name_to_in_name[op.inputs[0].name])
else:
context.out_name_to_in_name[op.outputs[0].name] = op.inputs[0].name
except:
print('Skipping name of placeholder')
def one_hot(op, context):
input_name = compat.as_str_any(op.inputs[0].name)
output_name = compat.as_str_any(op.outputs[0].name)
depth = context.consts[compat.as_str_any(op.inputs[1].name)]
on_value = context.consts[compat.as_str_any(op.inputs[2].name)]
off_value = context.consts[compat.as_str_any(op.inputs[3].name)]
n_dims = depth
W = np.ones((depth, depth)) * off_value
for i in range(depth):
W[i, i] = on_value
context.builder.add_embedding(name=output_name,
W=W,
b=None,
input_dim=n_dims,
output_channels=n_dims,
has_bias=False,
input_name=input_name,
output_name=output_name)
context.translated[output_name] = True
def path_to_str(path):
"""Returns the file system path representation of a `PathLike` object,
else as it is.
Args:
path: An object that can be converted to path representation.
Returns:
A `str` object.
"""
if hasattr(path, "__fspath__"):
path = as_str_any(path.__fspath__())
return path
def testListDirectory(self):
dir_path = os.path.join(self._base_dir, "test_dir")
file_io.create_dir(dir_path)
files = [b"file1.txt", b"file2.txt", b"file3.txt"]
for name in files:
file_path = os.path.join(dir_path, compat.as_str_any(name))
file_io.FileIO(file_path, mode="w").write("testing")
subdir_path = os.path.join(dir_path, "sub_dir")
file_io.create_dir(subdir_path)
subdir_file_path = os.path.join(subdir_path, "file4.txt")
file_io.FileIO(subdir_file_path, mode="w").write("testing")
dir_list = file_io.list_directory(dir_path)
self.assertItemsEqual(files + [b"sub_dir"], dir_list)
def IsTensorFlowEventsFile(path):
"""Check the path name to see if it is probably a TF Events file.
Args:
path: A file path to check if it is an event file.
Raises:
ValueError: If the path is an empty string.
Returns:
If path is formatted like a TensorFlowEventsFile.
"""
if not path:
raise ValueError('Path must be a nonempty string')
return 'tfevents' in compat.as_str_any(os.path.basename(path))
def __init__(self, handle, dtype, session):
"""Constructs a new tensor handle.
A tensor handle for a persistent tensor is a python string
that has the form of "tensor_name;unique_id;device_name".
Args:
handle: A tensor handle.
dtype: The data type of the tensor represented by `handle`.
session: The session in which the tensor is produced.
"""
self._handle = compat.as_str_any(handle)
self._dtype = dtype
self._session = session
self._auto_gc_enabled = True
def walk_v2(top, topdown=True, onerror=None):
"""Recursive directory tree generator for directories.
Args:
top: string, a Directory name
topdown: bool, Traverse pre order if True, post order if False.
onerror: optional handler for errors. Should be a function, it will be
called with the error as argument. Rethrowing the error aborts the walk.
Errors that happen while listing directories are ignored.
Yields:
Each yield is a 3-tuple: the pathname of a directory, followed by lists of
all its subdirectories and leaf files.
(dirname, [subdirname, subdirname, ...], [filename, filename, ...])
as strings
"""
top = compat.as_str_any(top)
try:
listing = list_directory(top)
except errors.NotFoundError as err:
if onerror:
onerror(err)
else:
return
files = []
subdirs = []
for item in listing:
full_path = os.path.join(top, item)
if is_directory(full_path):
subdirs.append(item)
else:
files.append(item)
here = (top, subdirs, files)
if topdown:
yield here
for subdir in subdirs:
for subitem in walk_v2(os.path.join(top, subdir), topdown, onerror=onerror):
yield subitem
if not topdown:
yield here
def get_matching_files(filename):
"""Returns a list of files that match the given pattern.
Args:
filename: string, the pattern
Returns:
Returns a list of strings containing filenames that match the given pattern.
Raises:
errors.OpError: If there are filesystem / directory listing errors.
"""
with errors.raise_exception_on_not_ok_status() as status:
# Convert each element to string, since the return values of the
# vector of string should be interpreted as strings, not bytes.
return [compat.as_str_any(matching_filename)
for matching_filename in pywrap_tensorflow.GetMatchingFiles(
compat.as_bytes(filename), status)]
def _is_removed_mentioned(s, removed_op_names):
"""Determine whether any removed op is mentioned in the given object.
This relies on the string representation of the object. This is used for
proto messages that may mention ops by name in nested fields. The string
representation of the proto includes those field values, so this string
search approach is sufficient.
Args:
s: an object to search for removed op names.
removed_op_names: An iterable of names of ops that were removed.
Returns:
True if any removed op is mentioned in the given object, False otherwise.
"""
for removed_op_name in removed_op_names:
if removed_op_name in compat.as_str_any(s):
return True
return False
def walk(top, in_order=True):
"""Recursive directory tree generator for directories.
Args:
top: string, a Directory name
in_order: bool, Traverse in order if True, post order if False.
Errors that happen while listing directories are ignored.
Yields:
Each yield is a 3-tuple: the pathname of a directory, followed by lists of
all its subdirectories and leaf files.
(dirname, [subdirname, subdirname, ...], [filename, filename, ...])
as strings
"""
top = compat.as_str_any(top)
try:
listing = list_directory(top)
except errors.NotFoundError:
return
files = []
subdirs = []
for item in listing:
full_path = os.path.join(top, item)
if is_directory(full_path):
subdirs.append(item)
else:
files.append(item)
here = (top, subdirs, files)
if in_order:
yield here
for subdir in subdirs:
for subitem in walk(os.path.join(top, subdir), in_order):
yield subitem
if not in_order:
yield here
def get_summary_description(node_def):
"""Given a TensorSummary node_def, retrieve its SummaryDescription.
When a Summary op is instantiated, a SummaryDescription of associated
metadata is stored in its NodeDef. This method retrieves the description.
Args:
node_def: the node_def_pb2.NodeDef of a TensorSummary op
Returns:
a summary_pb2.SummaryDescription
Raises:
ValueError: if the node is not a summary op.
"""
if node_def.op != 'TensorSummary':
raise ValueError("Can't get_summary_description on %s" % node_def.op)
description_str = _compat.as_str_any(node_def.attr['description'].s)
summary_description = SummaryDescription()
_json_format.Parse(description_str, summary_description)
return summary_description
def readline(self): r"""Reads the next line from the file. Leaves the '\n' at the end.""" self._prereadline_check() return compat.as_str_any(self._read_buf.ReadLineAsString())
def _get_device_name(handle):
"""The device name encoded in the handle."""
handle_str = compat.as_str_any(handle)
return pydev.canonical_name(handle_str.split(";")[-1])
