def read_data(input_path, do_finegrainedness_analysis, do_imbalance_analysis):
"""Reads the data from the evaluation files.
Args:
input_path: The path to the event file to read from.
do_finegrainedness_analysis: Whether to perform analysis of fine- vs coarse-
grained tasks. This affects the tags that are necessary to find in the
event files.
do_imbalance_analysis: Whether to analyze performance for episodes that are
characterized by different degrees of imbalance.
Returns:
ways: A list containing the 'way' of each episode.
shots: A list containing a np.array per episode. The shape of an episode's
array is the [way].
class_ids: A list containing a np.array per episode which contains two class
id's representing the two classes chosen for that binary classification.
test_logits: A list containing a np.array per episode. The shape of an
episode's array is [size of test set, way].
test_targets: A list containing a np.array per episode. The shape of an
episode's array is [size of test set]. This contains integers in the range
from 0 to the way of the episode.
Raises:
ValueError: Finegrainedness analysis is requested but no summaries of
class_ids are found for the provided split, or imbalance analysis is
requested but no summaries of class_props are found.
"""
split = (FLAGS.eval_finegrainedness_split
if FLAGS.eval_finegrainedness else 'test')
logging.info('Reading event file %s for summaries of split %s.',
input_path, split)
(ways, shots, class_props, class_ids, test_logits,
test_targets) = [], [], [], [], [], []
tags = set()
for e in tf.train.summary_iterator(input_path):
for v in e.summary.value:
tags.add(v.tag)
if v.tag == '{}_way'.format(split):
ways.append(v.simple_value)
elif v.tag == '{}_shots'.format(split):
shots.append(tf.make_ndarray(v.tensor))
elif v.tag == '{}_class_props'.format(split):
class_props.append(tf.make_ndarray(v.tensor))
elif v.tag == '{}_class_ids'.format(split):
class_ids.append(tf.make_ndarray(v.tensor))
elif v.tag == '{}_test_logits'.format(split):
test_logits.append(tf.make_ndarray(v.tensor))
elif v.tag == '{}_test_targets'.format(split):
test_targets.append(tf.make_ndarray(v.tensor))
if do_finegrainedness_analysis and not class_ids:
raise ValueError(
'No summaries found with tag: {}_class_ids. The tags that exist in the '
'event file are: {}.'.format(split, list(tags)))
if do_imbalance_analysis and not class_props:
raise ValueError(
'No summaries found with tag: {}_class_props. The tags that exist in '
'the event file are: {}.'.format(split, list(tags)))
return ways, shots, class_props, class_ids, test_logits, test_targets
def parse_events_files(tb_summary_dir, seqio_summaries=False):
"""Parse all TensorBoard events files in tb_summary_dir.
Args:
tb_summary_dir: str, path to look for events files in.
seqio_summaries: boolean, whether event summaries are generated by SeqIO
Evaluator.
Returns:
A dict, where each key is a TensorBoard tag and each value is a list of
Event tuples with step and value attributes.
"""
events = collections.defaultdict(list)
for events_file in tf.io.gfile.glob(
os.path.join(tb_summary_dir, "events.*")):
try:
serialized_events = list(
tfds.as_numpy(tf.data.TFRecordDataset(events_file)))[1:]
for idx, e in enumerate(tf.train.summary_iterator(events_file)):
for v in e.summary.value:
if seqio_summaries:
event = tf.compat.v1.Event.FromString(
serialized_events[idx - 1]).summary.value[0]
# Need to check if event has a tensor or scalar since we need to
# handle both cases.
if event.HasField("tensor"):
metric_value = tf.make_ndarray(event.tensor)
else:
metric_value = event.simple_value
else:
metric_value = v.simple_value
events[v.tag].append(Event(e.step, metric_value))
except tf.errors.DataLossError:
logging.info("Skipping %s due to truncated record.", events_file)
return events
def parse_predict_response(response):
results = {}
for key in response.outputs:
tensor_proto = response.outputs[key]
nd_array = tf.make_ndarray(tensor_proto)
results[key] = nd_array
return results
def _transmit_handler(self, request):
assert self._connected, "Cannot transmit before connect"
if request.seq_num >= self._next_receive_seq_num:
assert request.seq_num == self._next_receive_seq_num, \
"Invalid request"
self._next_receive_seq_num += 1
if request.HasField('start'):
with self._condition:
self._received_data[request.start.iter_id] = {}
elif request.HasField('commit'):
pass
elif request.HasField('data'):
with self._condition:
assert request.data.iter_id in self._received_data
self._received_data[
request.data.iter_id][
request.data.name] = \
tf.make_ndarray(request.data.tensor)
self._condition.notifyAll()
elif request.HasField('prefetch'):
for func in self._prefetch_handlers:
func(request.prefetch)
else:
return tws_pb.TrainerWorkerResponse(
status=common_pb.Status(
code=common_pb.STATUS_INVALID_REQUEST),
next_seq_num=self._next_receive_seq_num)
return tws_pb.TrainerWorkerResponse(
next_seq_num=self._next_receive_seq_num)
def _make_grpc_request(examples):
"""Builds and sends request to TensorFlow model server."""
request = predict_pb2.PredictRequest()
request.model_spec.name = servable_name
request.inputs["input"].CopyFrom(
tf.make_tensor_proto([ex.SerializeToString() for ex in examples],
shape=[len(examples)]))
response = stub.Predict(request, timeout_secs)
outputs = tf.make_ndarray(response.outputs["outputs"])
scores = tf.make_ndarray(response.outputs["scores"])
assert len(outputs) == len(scores)
return [
{ # pylint: disable=g-complex-comprehension
"outputs": output,
"scores": score
} for output, score in zip(outputs, scores)
]
def _get_scalar_value(value):
"""Extract the value from a summary_pb2.Value proto."""
if value.HasField('simple_value'):
return float(value.simple_value)
elif value.HasField('tensor'):
return float(tf.make_ndarray(value.tensor).item())
else:
raise ValueError(
'Can\'t extract scalar value from field: {}'.format(value))
def receive(self, iter_id, name):
logging.debug('Data: Waiting to receive %s for iter %d.', name,
iter_id)
with self._condition:
while (iter_id not in self._received_data) \
or (name not in self._received_data[iter_id]):
self._condition.wait()
data = self._received_data[iter_id][name]
logging.debug('Data: received %s for iter %d.', name, iter_id)
return tf.make_ndarray(data.tensor)
def change_T(new_node, *attr):
for a in attr:
if a in ("T", 'DstT', 'Tindices', 'TI'):
new_node.attr[a].type = types_pb2.DT_INT32
elif a in ("dtype", ):
new_node.attr['dtype'].CopyFrom(
attr_value_pb2.AttrValue(type=types_pb2.DT_INT32))
elif a in ("value", ):
values = tf.make_ndarray(new_node.attr['value'].tensor).astype(
np.int32)
new_node.attr['value'].tensor.CopyFrom(
tf.make_tensor_proto(values, dtype=tf.int32))
def receive(self, iter_id, name):
logging.debug('Data: Waiting to receive %s for iter %d.', name,
iter_id)
start_time = time.time()
with self._condition:
while (iter_id not in self._received_data) \
or (name not in self._received_data[iter_id]):
self._condition.wait()
data = self._received_data[iter_id][name]
duration = time.time() - start_time
metrics.emit_timer('receive_timer', duration)
logging.debug('Data: received %s for iter %d after %f sec.', name,
iter_id, duration)
return tf.make_ndarray(data.tensor)
def check_summarized_tensor(dir_, tensor_name, value, steps):
report = {'ok': True}
event_files = [
e for e in os.listdir(dir_) if EVENT_FILE_PATTERN.match(e) is not None
]
# steps on which tensor was summarized
summarized_steps = []
# all tuples (step, tensor_value) in which tensor_value != value
wrong_results = []
for ef in event_files:
for e in tf.train.summary_iterator(os.path.join(dir_, ef)):
# `step` is added to `summarized_steps` after confirmation
# of presence of the required tensor in an event file
step = e.step
for v in e.summary.value:
if v.tag == tensor_name:
summarized_steps.append(step)
summarized = tf.make_ndarray(v.tensor)
if np.any(value != summarized
) or value.shape != summarized.shape:
wrong_results.append(
(step, summarized.shape, summarized))
report['ok'] = False
missing_steps = [s for s in steps if s not in summarized_steps]
unwanted_steps = [s for s in summarized_steps if s not in steps]
if missing_steps:
report['ok'] = False
report['missing_steps'] = missing_steps
else:
report['missing_steps'] = []
if unwanted_steps:
report['ok'] = False
report['unwanted_steps'] = unwanted_steps
else:
report['unwanted_steps'] = []
if wrong_results:
report['wrong_results'] = wrong_results
else:
report['wrong_results'] = []
report['expected_value'] = value
report['expected_shape'] = value.shape
return report
def attr_value_to_python_type(
attr_value, # type: tf.AttrValue
attr_name # type: String
):
# type (...) -> Any
"""
Inverse of python_type_to_attr_value().
Args:
attr_value: Protocol buffer version of a node's attribute value
Returns:
A Python object or built-in type corresponding to the field in
`attr_value` that is in use.
"""
# TODO(frreiss): Handle AttrValues that are lists
if attr_value.HasField("s"): # str
# TODO(frreiss): Should we return the binary value here?
return tf.compat.as_str(attr_value.s)
elif attr_value.HasField("i"): # int
return attr_value.i
elif attr_value.HasField("f"): # float
return attr_value.f
elif attr_value.HasField("b"): # bool
return attr_value.b
elif attr_value.HasField("type"): # DType
return tf.DType(attr_value.type)
elif attr_value.HasField("shape"): # TensorShape
# Undocumented behavior of public API: tf.TensorShape constructor accepts
# a TensorShapeProto.
return tf.TensorShape(attr_value.shape)
elif attr_value.HasField("tensor"): # TensorProto
return tf.make_ndarray(attr_value.tensor)
elif attr_value.HasField("list"): # list
return attr_value.list
elif attr_value.HasField("func"): # func
return attr_value.func
# TODO(frreiss): Convert the "placeholder" fields of the union here
else:
raise ValueError("Don't know how to convert AttrValue {} to "
"a Python object for attribute {}".format(
attr_value, attr_name))
def receive(self, name):
return tf.make_ndarray(self._receive(name).tensor)
def receive(self, iter_id, name):
return tf.make_ndarray(self._receive(iter_id, name).tensor)
def embed_texts(self, texts: List[str]) -> List[List[float]]:
vectors = tf.make_ndarray(tf.make_tensor_proto(
self.embed(texts))).tolist()
return vectors
