def _histogram_summary(self, tf_name, value, step=None): """ Args: tf_name (str): name of tensorflow variable value (tuple or list): either a tuple of bin_edges and bincounts or a list of values to summarize in a histogram. References: https://github.com/yunjey/pytorch-tutorial/blob/master/tutorials/04-utils/tensorboard/logger.py#L45 Example: >>> tf_name = 'foo' >>> value = ([0, 1, 2, 3, 4, 5], [1, 20, 10, 22, 11]) >>> self = Logger(None, is_dummy=True) >>> summary = self._histogram_summary(tf_name, value, step=None) >>> assert summary.value[0].histo.max == 5 Example: >>> tf_name = 'foo' >>> value = [0.72, 0.18, 0.34, 0.66, 0.11, 0.70, 0.23] >>> self = Logger(None, is_dummy=True) >>> summary = self._histogram_summary(tf_name, value, step=None) >>> assert summary.value[0].histo.num == 7.0 """ if isinstance(value, tuple): bin_edges, bincounts = value assert len(bin_edges) == len(bincounts) + 1, ( 'must have one more edge than count') hist = summary_pb2.HistogramProto() hist.min = float(min(bin_edges)) hist.max = float(max(bin_edges)) else: values = np.array(value) bincounts, bin_edges = np.histogram(values) hist = summary_pb2.HistogramProto() hist.min = float(np.min(values)) hist.max = float(np.max(values)) hist.num = int(np.prod(values.shape)) hist.sum = float(np.sum(values)) hist.sum_squares = float(np.sum(values**2)) # Add bin edges and counts for edge in bin_edges[1:]: hist.bucket_limit.append(edge) for v in bincounts: hist.bucket.append(v) summary = summary_pb2.Summary() summary.value.add(tag=tf_name, histo=hist) return summary
def AddHistogram(self, tag, wall_time=0, step=0, hmin=1, hmax=2, hnum=3, hsum=4, hsum_squares=5, hbucket_limit=None, hbucket=None): histo = summary_pb2.HistogramProto( min=hmin, max=hmax, num=hnum, sum=hsum, sum_squares=hsum_squares, bucket_limit=hbucket_limit, bucket=hbucket) event = event_pb2.Event( wall_time=wall_time, step=step, summary=summary_pb2.Summary( value=[summary_pb2.Summary.Value( tag=tag, histo=histo)])) self.AddEvent(event)
def value(self): """Retrieves the current distribution of samples. Returns: A HistogramProto describing the distribution of samples. """ with c_api_util.tf_buffer() as buffer_: pywrap_tfe.TFE_MonitoringSamplerCellValue(self._cell, buffer_) proto_data = pywrap_tf_session.TF_GetBuffer(buffer_) histogram_proto = summary_pb2.HistogramProto() histogram_proto.ParseFromString(compat.as_bytes(proto_data)) return histogram_proto
def _MakeHistogram(values): """Convert values into a histogram proto using logic from histogram.cc.""" limits = _MakeHistogramBuckets() counts = [0] * len(limits) for v in values: idx = bisect.bisect_left(limits, v) counts[idx] += 1 limit_counts = [(limits[i], counts[i]) for i in xrange(len(limits)) if counts[i]] bucket_limit = [lc[0] for lc in limit_counts] bucket = [lc[1] for lc in limit_counts] sum_sq = sum(v * v for v in values) return summary_pb2.HistogramProto(min=min(values), max=max(values), num=len(values), sum=sum(values), sum_squares=sum_sq, bucket_limit=bucket_limit, bucket=bucket)
def _WriteScalarSummaries(self, data, subdirs=('', )): # Writes data to a tempfile in subdirs, and returns generator for the data. # If subdirs is given, writes data identically to all subdirectories. for subdir_ in subdirs: subdir = os.path.join(self.logdir, subdir_) self._MakeDirectoryIfNotExists(subdir) sw = summary_lib.summary.FileWriter(subdir) for datum in data: summary = summary_pb2.Summary() if 'simple_value' in datum: summary.value.add(tag=datum['tag'], simple_value=datum['simple_value']) sw.add_summary(summary, global_step=datum['step']) elif 'histo' in datum: summary.value.add(tag=datum['tag'], histo=summary_pb2.HistogramProto()) sw.add_summary(summary, global_step=datum['step']) elif 'session_log' in datum: sw.add_session_log(datum['session_log'], global_step=datum['step']) sw.close()
def _get_histogram_proto(self, proto_bytes): histogram_proto = summary_pb2.HistogramProto() histogram_proto.ParseFromString(proto_bytes) return histogram_proto
def _get_read_histogram_proto(self, api_label): proto_bytes = metrics.GetCheckpointReadDurations(api_label=api_label) histogram_proto = summary_pb2.HistogramProto() histogram_proto.ParseFromString(proto_bytes) return histogram_proto
def _GenerateTestData(self): """Generates the test data directory. The test data has a single run named run1 which contains: - a histogram - an image at timestamp and step 0 - scalar events containing the value i at step 10 * i and wall time 100 * i, for i in [1, _SCALAR_COUNT). - a graph definition Returns: temp_dir: The directory the test data is generated under. """ temp_dir = tempfile.mkdtemp(prefix=self.get_temp_dir()) self.addCleanup(shutil.rmtree, temp_dir) run1_path = os.path.join(temp_dir, 'run1') os.makedirs(run1_path) writer = writer_lib.FileWriter(run1_path) histogram_value = summary_pb2.HistogramProto( min=0, max=2, num=3, sum=6, sum_squares=5, bucket_limit=[0, 1, 2], bucket=[1, 1, 1]) # Add a simple graph event. graph_def = graph_pb2.GraphDef() node1 = graph_def.node.add() node1.name = 'a' node2 = graph_def.node.add() node2.name = 'b' node2.attr['very_large_attr'].s = b'a' * 2048 # 2 KB attribute meta_graph_def = meta_graph_pb2.MetaGraphDef(graph_def=graph_def) if self._only_use_meta_graph: writer.add_meta_graph(meta_graph_def) else: writer.add_graph(graph_def) # Add a simple run metadata event. run_metadata = config_pb2.RunMetadata() device_stats = run_metadata.step_stats.dev_stats.add() device_stats.device = 'test device' writer.add_run_metadata(run_metadata, 'test run') # 1x1 transparent GIF. encoded_image = base64.b64decode( 'R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7') image_value = summary_pb2.Summary.Image( height=1, width=1, colorspace=1, encoded_image_string=encoded_image) audio_value = summary_pb2.Summary.Audio( sample_rate=44100, length_frames=22050, num_channels=2, encoded_audio_string=b'', content_type='audio/wav') writer.add_event( event_pb2.Event( wall_time=0, step=0, summary=summary_pb2.Summary(value=[ summary_pb2.Summary.Value( tag='histogram', histo=histogram_value), summary_pb2.Summary.Value( tag='image', image=image_value), summary_pb2.Summary.Value( tag='audio', audio=audio_value) ]))) # Write 100 simple values. for i in xrange(1, self._SCALAR_COUNT + 1): writer.add_event( event_pb2.Event( # We use different values for wall time, step, and the value so we # can tell them apart. wall_time=100 * i, step=10 * i, summary=summary_pb2.Summary(value=[ summary_pb2.Summary.Value( tag='simple_values', simple_value=i) ]))) writer.flush() writer.close() return temp_dir