def _check_health_pills_in_events_file(self, events_file_path,
debug_key_to_tensors):
reader = tf.compat.v1.python_io.tf_record_iterator(events_file_path)
event_read = tf.Event()
# The first event in the file should contain the events version, which is
# important because without it, TensorBoard may purge health pill events.
event_read.ParseFromString(next(reader))
self.assertEqual("brain.Event:2", event_read.file_version)
health_pills = {}
while True:
next_event = next(reader, None)
if not next_event:
break
event_read.ParseFromString(next_event)
values = event_read.summary.value
if values:
if (values[0].metadata.plugin_data.plugin_name ==
constants.DEBUGGER_PLUGIN_NAME):
debug_key = values[0].node_name
if debug_key not in health_pills:
health_pills[debug_key] = [
tf_debug.load_tensor_from_event(event_read)
]
else:
health_pills[debug_key].append(
tf_debug.load_tensor_from_event(event_read))
for debug_key in debug_key_to_tensors:
tensors = debug_key_to_tensors[debug_key]
for i, tensor in enumerate(tensors):
self.assertAllClose(self._compute_health_pill(tensor),
health_pills[debug_key][i])
def _check_health_pills_in_events_file(self,
events_file_path,
debug_key_to_tensors):
reader = tf.python_io.tf_record_iterator(events_file_path)
event_read = tf.Event()
# The first event in the file should contain the events version, which is
# important because without it, TensorBoard may purge health pill events.
event_read.ParseFromString(next(reader))
self.assertEqual("brain.Event:2", event_read.file_version)
health_pills = {}
while True:
next_event = next(reader, None)
if not next_event:
break
event_read.ParseFromString(next_event)
values = event_read.summary.value
if values:
if (values[0].metadata.plugin_data.plugin_name ==
constants.DEBUGGER_PLUGIN_NAME):
debug_key = values[0].node_name
if debug_key not in health_pills:
health_pills[debug_key] = [
tf_debug.load_tensor_from_event(event_read)]
else:
health_pills[debug_key].append(
tf_debug.load_tensor_from_event(event_read))
for debug_key in debug_key_to_tensors:
tensors = debug_key_to_tensors[debug_key]
for i, tensor in enumerate(tensors):
self.assertAllClose(
self._compute_health_pill(tensor),
health_pills[debug_key][i])
def extract_numerics_alert(event):
"""Determines whether a health pill event contains bad values.
A bad value is one of NaN, -Inf, or +Inf.
Args:
event: (`Event`) A `tensorflow.Event` proto from `DebugNumericSummary`
ops.
Returns:
An instance of `NumericsAlert`, if bad values are found.
`None`, if no bad values are found.
Raises:
ValueError: if the event does not have the expected tag prefix or the
debug op name is not the expected debug op name suffix.
"""
value = event.summary.value[0]
debugger_plugin_metadata_content = None
if value.HasField("metadata"):
plugin_data = value.metadata.plugin_data
if plugin_data.plugin_name == constants.DEBUGGER_PLUGIN_NAME:
debugger_plugin_metadata_content = plugin_data.content
if not debugger_plugin_metadata_content:
raise ValueError("Event proto input lacks debugger plugin SummaryMetadata.")
debugger_plugin_metadata_content = tf.compat.as_text(
debugger_plugin_metadata_content)
try:
content_object = json.loads(debugger_plugin_metadata_content)
device_name = content_object["device"]
except (KeyError, ValueError) as e:
raise ValueError("Could not determine device from JSON string %r, %r" %
(debugger_plugin_metadata_content, e))
debug_op_suffix = ":DebugNumericSummary"
if not value.node_name.endswith(debug_op_suffix):
raise ValueError(
"Event proto input does not have the expected debug op suffix %s" %
debug_op_suffix)
tensor_name = value.node_name[:-len(debug_op_suffix)]
elements = tf_debug.load_tensor_from_event(event)
nan_count = elements[constants.NAN_NUMERIC_SUMMARY_OP_INDEX]
neg_inf_count = elements[constants.NEG_INF_NUMERIC_SUMMARY_OP_INDEX]
pos_inf_count = elements[constants.POS_INF_NUMERIC_SUMMARY_OP_INDEX]
if nan_count > 0 or neg_inf_count > 0 or pos_inf_count > 0:
return NumericsAlert(
device_name, tensor_name, event.wall_time, nan_count, neg_inf_count,
pos_inf_count)
return None
def extract_numerics_alert(event):
"""Determines whether a health pill event contains bad values.
A bad value is one of NaN, -Inf, or +Inf.
Args:
event: (`Event`) A `tensorflow.Event` proto from `DebugNumericSummary`
ops.
Returns:
An instance of `NumericsAlert`, if bad values are found.
`None`, if no bad values are found.
Raises:
ValueError: if the event does not have the expected tag prefix or the
debug op name is not the expected debug op name suffix.
"""
value = event.summary.value[0]
debugger_plugin_metadata_content = None
if value.HasField("metadata"):
plugin_data = value.metadata.plugin_data
if plugin_data.plugin_name == constants.DEBUGGER_PLUGIN_NAME:
debugger_plugin_metadata_content = plugin_data.content
if not debugger_plugin_metadata_content:
raise ValueError(
"Event proto input lacks debugger plugin SummaryMetadata.")
debugger_plugin_metadata_content = tf.compat.as_text(
debugger_plugin_metadata_content)
try:
content_object = json.loads(debugger_plugin_metadata_content)
device_name = content_object["device"]
except (KeyError, ValueError) as e:
raise ValueError("Could not determine device from JSON string %r, %r" %
(debugger_plugin_metadata_content, e))
debug_op_suffix = ":DebugNumericSummary"
if not value.node_name.endswith(debug_op_suffix):
raise ValueError(
"Event proto input does not have the expected debug op suffix %s" %
debug_op_suffix)
tensor_name = value.node_name[:-len(debug_op_suffix)]
elements = tf_debug.load_tensor_from_event(event)
nan_count = elements[constants.NAN_NUMERIC_SUMMARY_OP_INDEX]
neg_inf_count = elements[constants.NEG_INF_NUMERIC_SUMMARY_OP_INDEX]
pos_inf_count = elements[constants.POS_INF_NUMERIC_SUMMARY_OP_INDEX]
if nan_count > 0 or neg_inf_count > 0 or pos_inf_count > 0:
return NumericsAlert(device_name, tensor_name, event.wall_time,
nan_count, neg_inf_count, pos_inf_count)
return None