def read_data(input_path, do_finegrainedness_analysis, do_imbalance_analysis): """Reads the data from the evaluation files. Args: input_path: The path to the event file to read from. do_finegrainedness_analysis: Whether to perform analysis of fine- vs coarse- grained tasks. This affects the tags that are necessary to find in the event files. do_imbalance_analysis: Whether to analyze performance for episodes that are characterized by different degrees of imbalance. Returns: ways: A list containing the 'way' of each episode. shots: A list containing a np.array per episode. The shape of an episode's array is the [way]. class_ids: A list containing a np.array per episode which contains two class id's representing the two classes chosen for that binary classification. test_logits: A list containing a np.array per episode. The shape of an episode's array is [size of test set, way]. test_targets: A list containing a np.array per episode. The shape of an episode's array is [size of test set]. This contains integers in the range from 0 to the way of the episode. Raises: ValueError: Finegrainedness analysis is requested but no summaries of class_ids are found for the provided split, or imbalance analysis is requested but no summaries of class_props are found. """ split = (FLAGS.eval_finegrainedness_split if FLAGS.eval_finegrainedness else 'test') logging.info('Reading event file %s for summaries of split %s.', input_path, split) (ways, shots, class_props, class_ids, test_logits, test_targets) = [], [], [], [], [], [] tags = set() for e in tf.train.summary_iterator(input_path): for v in e.summary.value: tags.add(v.tag) if v.tag == '{}_way'.format(split): ways.append(v.simple_value) elif v.tag == '{}_shots'.format(split): shots.append(tf.make_ndarray(v.tensor)) elif v.tag == '{}_class_props'.format(split): class_props.append(tf.make_ndarray(v.tensor)) elif v.tag == '{}_class_ids'.format(split): class_ids.append(tf.make_ndarray(v.tensor)) elif v.tag == '{}_test_logits'.format(split): test_logits.append(tf.make_ndarray(v.tensor)) elif v.tag == '{}_test_targets'.format(split): test_targets.append(tf.make_ndarray(v.tensor)) if do_finegrainedness_analysis and not class_ids: raise ValueError( 'No summaries found with tag: {}_class_ids. The tags that exist in the ' 'event file are: {}.'.format(split, list(tags))) if do_imbalance_analysis and not class_props: raise ValueError( 'No summaries found with tag: {}_class_props. The tags that exist in ' 'the event file are: {}.'.format(split, list(tags))) return ways, shots, class_props, class_ids, test_logits, test_targets
def parse_events_files(tb_summary_dir, seqio_summaries=False): """Parse all TensorBoard events files in tb_summary_dir. Args: tb_summary_dir: str, path to look for events files in. seqio_summaries: boolean, whether event summaries are generated by SeqIO Evaluator. Returns: A dict, where each key is a TensorBoard tag and each value is a list of Event tuples with step and value attributes. """ events = collections.defaultdict(list) for events_file in tf.io.gfile.glob( os.path.join(tb_summary_dir, "events.*")): try: serialized_events = list( tfds.as_numpy(tf.data.TFRecordDataset(events_file)))[1:] for idx, e in enumerate(tf.train.summary_iterator(events_file)): for v in e.summary.value: if seqio_summaries: event = tf.compat.v1.Event.FromString( serialized_events[idx - 1]).summary.value[0] # Need to check if event has a tensor or scalar since we need to # handle both cases. if event.HasField("tensor"): metric_value = tf.make_ndarray(event.tensor) else: metric_value = event.simple_value else: metric_value = v.simple_value events[v.tag].append(Event(e.step, metric_value)) except tf.errors.DataLossError: logging.info("Skipping %s due to truncated record.", events_file) return events
def parse_predict_response(response): results = {} for key in response.outputs: tensor_proto = response.outputs[key] nd_array = tf.make_ndarray(tensor_proto) results[key] = nd_array return results
def _transmit_handler(self, request): assert self._connected, "Cannot transmit before connect" if request.seq_num >= self._next_receive_seq_num: assert request.seq_num == self._next_receive_seq_num, \ "Invalid request" self._next_receive_seq_num += 1 if request.HasField('start'): with self._condition: self._received_data[request.start.iter_id] = {} elif request.HasField('commit'): pass elif request.HasField('data'): with self._condition: assert request.data.iter_id in self._received_data self._received_data[ request.data.iter_id][ request.data.name] = \ tf.make_ndarray(request.data.tensor) self._condition.notifyAll() elif request.HasField('prefetch'): for func in self._prefetch_handlers: func(request.prefetch) else: return tws_pb.TrainerWorkerResponse( status=common_pb.Status( code=common_pb.STATUS_INVALID_REQUEST), next_seq_num=self._next_receive_seq_num) return tws_pb.TrainerWorkerResponse( next_seq_num=self._next_receive_seq_num)
def _make_grpc_request(examples): """Builds and sends request to TensorFlow model server.""" request = predict_pb2.PredictRequest() request.model_spec.name = servable_name request.inputs["input"].CopyFrom( tf.make_tensor_proto([ex.SerializeToString() for ex in examples], shape=[len(examples)])) response = stub.Predict(request, timeout_secs) outputs = tf.make_ndarray(response.outputs["outputs"]) scores = tf.make_ndarray(response.outputs["scores"]) assert len(outputs) == len(scores) return [ { # pylint: disable=g-complex-comprehension "outputs": output, "scores": score } for output, score in zip(outputs, scores) ]
def _get_scalar_value(value): """Extract the value from a summary_pb2.Value proto.""" if value.HasField('simple_value'): return float(value.simple_value) elif value.HasField('tensor'): return float(tf.make_ndarray(value.tensor).item()) else: raise ValueError( 'Can\'t extract scalar value from field: {}'.format(value))
def receive(self, iter_id, name): logging.debug('Data: Waiting to receive %s for iter %d.', name, iter_id) with self._condition: while (iter_id not in self._received_data) \ or (name not in self._received_data[iter_id]): self._condition.wait() data = self._received_data[iter_id][name] logging.debug('Data: received %s for iter %d.', name, iter_id) return tf.make_ndarray(data.tensor)
def change_T(new_node, *attr): for a in attr: if a in ("T", 'DstT', 'Tindices', 'TI'): new_node.attr[a].type = types_pb2.DT_INT32 elif a in ("dtype", ): new_node.attr['dtype'].CopyFrom( attr_value_pb2.AttrValue(type=types_pb2.DT_INT32)) elif a in ("value", ): values = tf.make_ndarray(new_node.attr['value'].tensor).astype( np.int32) new_node.attr['value'].tensor.CopyFrom( tf.make_tensor_proto(values, dtype=tf.int32))
def receive(self, iter_id, name): logging.debug('Data: Waiting to receive %s for iter %d.', name, iter_id) start_time = time.time() with self._condition: while (iter_id not in self._received_data) \ or (name not in self._received_data[iter_id]): self._condition.wait() data = self._received_data[iter_id][name] duration = time.time() - start_time metrics.emit_timer('receive_timer', duration) logging.debug('Data: received %s for iter %d after %f sec.', name, iter_id, duration) return tf.make_ndarray(data.tensor)
def check_summarized_tensor(dir_, tensor_name, value, steps): report = {'ok': True} event_files = [ e for e in os.listdir(dir_) if EVENT_FILE_PATTERN.match(e) is not None ] # steps on which tensor was summarized summarized_steps = [] # all tuples (step, tensor_value) in which tensor_value != value wrong_results = [] for ef in event_files: for e in tf.train.summary_iterator(os.path.join(dir_, ef)): # `step` is added to `summarized_steps` after confirmation # of presence of the required tensor in an event file step = e.step for v in e.summary.value: if v.tag == tensor_name: summarized_steps.append(step) summarized = tf.make_ndarray(v.tensor) if np.any(value != summarized ) or value.shape != summarized.shape: wrong_results.append( (step, summarized.shape, summarized)) report['ok'] = False missing_steps = [s for s in steps if s not in summarized_steps] unwanted_steps = [s for s in summarized_steps if s not in steps] if missing_steps: report['ok'] = False report['missing_steps'] = missing_steps else: report['missing_steps'] = [] if unwanted_steps: report['ok'] = False report['unwanted_steps'] = unwanted_steps else: report['unwanted_steps'] = [] if wrong_results: report['wrong_results'] = wrong_results else: report['wrong_results'] = [] report['expected_value'] = value report['expected_shape'] = value.shape return report
def attr_value_to_python_type( attr_value, # type: tf.AttrValue attr_name # type: String ): # type (...) -> Any """ Inverse of python_type_to_attr_value(). Args: attr_value: Protocol buffer version of a node's attribute value Returns: A Python object or built-in type corresponding to the field in `attr_value` that is in use. """ # TODO(frreiss): Handle AttrValues that are lists if attr_value.HasField("s"): # str # TODO(frreiss): Should we return the binary value here? return tf.compat.as_str(attr_value.s) elif attr_value.HasField("i"): # int return attr_value.i elif attr_value.HasField("f"): # float return attr_value.f elif attr_value.HasField("b"): # bool return attr_value.b elif attr_value.HasField("type"): # DType return tf.DType(attr_value.type) elif attr_value.HasField("shape"): # TensorShape # Undocumented behavior of public API: tf.TensorShape constructor accepts # a TensorShapeProto. return tf.TensorShape(attr_value.shape) elif attr_value.HasField("tensor"): # TensorProto return tf.make_ndarray(attr_value.tensor) elif attr_value.HasField("list"): # list return attr_value.list elif attr_value.HasField("func"): # func return attr_value.func # TODO(frreiss): Convert the "placeholder" fields of the union here else: raise ValueError("Don't know how to convert AttrValue {} to " "a Python object for attribute {}".format( attr_value, attr_name))
def receive(self, name): return tf.make_ndarray(self._receive(name).tensor)
def receive(self, iter_id, name): return tf.make_ndarray(self._receive(iter_id, name).tensor)
def embed_texts(self, texts: List[str]) -> List[List[float]]: vectors = tf.make_ndarray(tf.make_tensor_proto( self.embed(texts))).tolist() return vectors