def monitoring_infos(self, transform_id): # type: (str) -> Dict[FrozenSet, metrics_pb2.MonitoringInfo] infos = super(DoOperation, self).monitoring_infos(transform_id) if self.tagged_receivers: for tag, receiver in self.tagged_receivers.items(): mi = monitoring_infos.int64_counter( monitoring_infos.ELEMENT_COUNT_URN, receiver.opcounter.element_counter.value(), ptransform=transform_id, tag=str(tag)) infos[monitoring_infos.to_key(mi)] = mi (unused_mean, sum, count, min, max) = (receiver.opcounter.mean_byte_counter.value()) metric = metrics_pb2.Metric( distribution_data=metrics_pb2.DistributionData( int_distribution_data=metrics_pb2.IntDistributionData( count=count, sum=sum, min=min, max=max))) sampled_byte_count = monitoring_infos.int64_distribution( monitoring_infos.SAMPLED_BYTE_SIZE_URN, metric, ptransform=transform_id, tag=str(tag)) infos[monitoring_infos.to_key( sampled_byte_count)] = sampled_byte_count return infos
def pcollection_count_monitoring_infos(self, transform_id): """Returns the element count MonitoringInfo collected by this operation.""" if len(self.receivers) == 1: # If there is exactly one output, we can unambiguously # fix its name later, which we do. # TODO(robertwb): Plumb the actual name here. elem_count_mi = monitoring_infos.int64_counter( monitoring_infos.ELEMENT_COUNT_URN, self.receivers[0].opcounter.element_counter.value(), ptransform=transform_id, tag='ONLY_OUTPUT' if len(self.receivers) == 1 else str(None), ) (unused_mean, sum, count, min, max) = (self.receivers[0].opcounter.mean_byte_counter.value()) metric = metrics_pb2.Metric( distribution_data=metrics_pb2.DistributionData( int_distribution_data=metrics_pb2.IntDistributionData( count=count, sum=sum, min=min, max=max))) sampled_byte_count = monitoring_infos.int64_distribution( monitoring_infos.SAMPLED_BYTE_SIZE_URN, metric, ptransform=transform_id, tag='ONLY_OUTPUT' if len(self.receivers) == 1 else str(None), ) return { monitoring_infos.to_key(elem_count_mi): elem_count_mi, monitoring_infos.to_key(sampled_byte_count): sampled_byte_count } return {}
def pcollection_count_monitoring_infos(self, tag_to_pcollection_id): # type: (Dict[str, str]) -> Dict[FrozenSet, metrics_pb2.MonitoringInfo] """Returns the element count MonitoringInfo collected by this operation.""" infos = super( DoOperation, self).pcollection_count_monitoring_infos(tag_to_pcollection_id) if self.tagged_receivers: for tag, receiver in self.tagged_receivers.items(): if str(tag) not in tag_to_pcollection_id: continue pcollection_id = tag_to_pcollection_id[str(tag)] mi = monitoring_infos.int64_counter( monitoring_infos.ELEMENT_COUNT_URN, receiver.opcounter.element_counter.value(), pcollection=pcollection_id) infos[monitoring_infos.to_key(mi)] = mi (unused_mean, sum, count, min, max) = ( receiver.opcounter.mean_byte_counter.value()) sampled_byte_count = monitoring_infos.int64_distribution( monitoring_infos.SAMPLED_BYTE_SIZE_URN, DistributionData(sum, count, min, max), pcollection=pcollection_id) infos[monitoring_infos.to_key(sampled_byte_count)] = sampled_byte_count return infos
def pcollection_count_monitoring_infos(self, tag_to_pcollection_id): # type: (Dict[str, str]) -> Dict[FrozenSet, metrics_pb2.MonitoringInfo] """Returns the element count MonitoringInfo collected by this operation.""" # Skip producing monitoring infos if there is more then one receiver # since there is no way to provide a mapping from tag to pcollection id # within Operation. if len(self.receivers) != 1 or len(tag_to_pcollection_id) != 1: return {} all_monitoring_infos = {} pcollection_id = next(iter(tag_to_pcollection_id.values())) receiver = self.receivers[0] elem_count_mi = monitoring_infos.int64_counter( monitoring_infos.ELEMENT_COUNT_URN, receiver.opcounter.element_counter.value(), pcollection=pcollection_id, ) (unused_mean, sum, count, min, max) = ( receiver.opcounter.mean_byte_counter.value()) sampled_byte_count = monitoring_infos.int64_distribution( monitoring_infos.SAMPLED_BYTE_SIZE_URN, DistributionData(sum, count, min, max), pcollection=pcollection_id, ) all_monitoring_infos[monitoring_infos.to_key(elem_count_mi)] = elem_count_mi all_monitoring_infos[monitoring_infos.to_key( sampled_byte_count)] = sampled_byte_count return all_monitoring_infos
def to_runner_api_monitoring_infos(self, transform_id): """Returns a list of MonitoringInfos for the metrics in this container.""" all_user_metrics = [] for k, v in self.counters.items(): all_user_metrics.append( monitoring_infos.int64_counter( user_metric_urn(k.namespace, k.name), v.to_runner_api_monitoring_info(), ptransform=transform_id)) for k, v in self.distributions.items(): all_user_metrics.append( monitoring_infos.int64_distribution( user_metric_urn(k.namespace, k.name), v.get_cumulative().to_runner_api_monitoring_info(), ptransform=transform_id)) for k, v in self.gauges.items(): all_user_metrics.append( monitoring_infos.int64_gauge( user_metric_urn(k.namespace, k.name), v.get_cumulative().to_runner_api_monitoring_info(), ptransform=transform_id)) return {monitoring_infos.to_key(mi): mi for mi in all_user_metrics}
def to_runner_api_monitoring_infos(self, transform_id): """Returns a list of MonitoringInfos for the metrics in this container.""" all_user_metrics = [] for k, v in self.counters.items(): all_user_metrics.append(monitoring_infos.int64_counter( user_metric_urn(k.namespace, k.name), v.to_runner_api_monitoring_info(), ptransform=transform_id )) for k, v in self.distributions.items(): all_user_metrics.append(monitoring_infos.int64_distribution( user_distribution_metric_urn(k.namespace, k.name), v.get_cumulative().to_runner_api_monitoring_info(), ptransform=transform_id )) for k, v in self.gauges.items(): all_user_metrics.append(monitoring_infos.int64_gauge( user_metric_urn(k.namespace, k.name), v.get_cumulative().to_runner_api_monitoring_info(), ptransform=transform_id )) return {monitoring_infos.to_key(mi) : mi for mi in all_user_metrics}