def monitoring_infos(self, transform_id, tag_to_pcollection_id):
# type: (str, Dict[str, str]) -> Dict[FrozenSet, metrics_pb2.MonitoringInfo]
def encode_progress(value):
# type: (float) -> bytes
coder = coders.IterableCoder(coders.FloatCoder())
return coder.encode([value])
with self.lock:
infos = super(SdfProcessSizedElements,
self).monitoring_infos(transform_id, tag_to_pcollection_id)
current_element_progress = self.current_element_progress()
if current_element_progress:
if current_element_progress.completed_work:
completed = current_element_progress.completed_work
remaining = current_element_progress.remaining_work
else:
completed = current_element_progress.fraction_completed
remaining = current_element_progress.fraction_remaining
assert completed is not None
assert remaining is not None
completed_mi = metrics_pb2.MonitoringInfo(
urn=monitoring_infos.WORK_COMPLETED_URN,
type=monitoring_infos.PROGRESS_TYPE,
labels=monitoring_infos.create_labels(ptransform=transform_id),
payload=encode_progress(completed))
remaining_mi = metrics_pb2.MonitoringInfo(
urn=monitoring_infos.WORK_REMAINING_URN,
type=monitoring_infos.PROGRESS_TYPE,
labels=monitoring_infos.create_labels(ptransform=transform_id),
payload=encode_progress(remaining))
infos[monitoring_infos.to_key(completed_mi)] = completed_mi
infos[monitoring_infos.to_key(remaining_mi)] = remaining_mi
return infos
def pcollection_count_monitoring_infos(self, tag_to_pcollection_id):
# type: (Dict[str, str]) -> Dict[FrozenSet, metrics_pb2.MonitoringInfo]
"""Returns the element count MonitoringInfo collected by this operation."""
# Skip producing monitoring infos if there is more then one receiver
# since there is no way to provide a mapping from tag to pcollection id
# within Operation.
if len(self.receivers) != 1 or len(tag_to_pcollection_id) != 1:
return {}
all_monitoring_infos = {}
pcollection_id = next(iter(tag_to_pcollection_id.values()))
receiver = self.receivers[0]
elem_count_mi = monitoring_infos.int64_counter(
monitoring_infos.ELEMENT_COUNT_URN,
receiver.opcounter.element_counter.value(),
pcollection=pcollection_id,
)
(unused_mean, sum, count, min, max) = (
receiver.opcounter.mean_byte_counter.value())
sampled_byte_count = monitoring_infos.int64_distribution(
monitoring_infos.SAMPLED_BYTE_SIZE_URN,
DistributionData(sum, count, min, max),
pcollection=pcollection_id,
)
all_monitoring_infos[monitoring_infos.to_key(elem_count_mi)] = elem_count_mi
all_monitoring_infos[monitoring_infos.to_key(
sampled_byte_count)] = sampled_byte_count
return all_monitoring_infos
def monitoring_infos(self, transform_id):
# type: (str) -> Dict[FrozenSet, metrics_pb2.MonitoringInfo]
infos = super(DoOperation, self).monitoring_infos(transform_id)
if self.tagged_receivers:
for tag, receiver in self.tagged_receivers.items():
mi = monitoring_infos.int64_counter(
monitoring_infos.ELEMENT_COUNT_URN,
receiver.opcounter.element_counter.value(),
ptransform=transform_id,
tag=str(tag))
infos[monitoring_infos.to_key(mi)] = mi
(unused_mean, sum, count, min,
max) = (receiver.opcounter.mean_byte_counter.value())
metric = metrics_pb2.Metric(
distribution_data=metrics_pb2.DistributionData(
int_distribution_data=metrics_pb2.IntDistributionData(
count=count, sum=sum, min=min, max=max)))
sampled_byte_count = monitoring_infos.int64_distribution(
monitoring_infos.SAMPLED_BYTE_SIZE_URN,
metric,
ptransform=transform_id,
tag=str(tag))
infos[monitoring_infos.to_key(
sampled_byte_count)] = sampled_byte_count
return infos
def monitoring_infos(self, transform_id):
# type: (str) -> Dict[FrozenSet, metrics_pb2.MonitoringInfo]
with self.lock:
infos = super(SdfProcessSizedElements,
self).monitoring_infos(transform_id)
current_element_progress = self.current_element_progress()
if current_element_progress:
if current_element_progress.completed_work:
completed = current_element_progress.completed_work
remaining = current_element_progress.remaining_work
else:
completed = current_element_progress.fraction_completed
remaining = current_element_progress.fraction_remaining
assert completed is not None
assert remaining is not None
completed_mi = metrics_pb2.MonitoringInfo(
urn=monitoring_infos.WORK_COMPLETED_URN,
type=monitoring_infos.LATEST_DOUBLES_TYPE,
labels=monitoring_infos.create_labels(ptransform=transform_id),
payload=coders.FloatCoder().get_impl().encode_nested(completed),
timestamp=monitoring_infos.to_timestamp_proto(time.time()))
remaining_mi = metrics_pb2.MonitoringInfo(
urn=monitoring_infos.WORK_REMAINING_URN,
type=monitoring_infos.LATEST_DOUBLES_TYPE,
labels=monitoring_infos.create_labels(ptransform=transform_id),
payload=coders.FloatCoder().get_impl().encode_nested(remaining),
timestamp=monitoring_infos.to_timestamp_proto(time.time()))
infos[monitoring_infos.to_key(completed_mi)] = completed_mi
infos[monitoring_infos.to_key(remaining_mi)] = remaining_mi
return infos
def pcollection_count_monitoring_infos(self, tag_to_pcollection_id):
# type: (Dict[str, str]) -> Dict[FrozenSet, metrics_pb2.MonitoringInfo]
"""Returns the element count MonitoringInfo collected by this operation."""
infos = super(
DoOperation,
self).pcollection_count_monitoring_infos(tag_to_pcollection_id)
if self.tagged_receivers:
for tag, receiver in self.tagged_receivers.items():
if str(tag) not in tag_to_pcollection_id:
continue
pcollection_id = tag_to_pcollection_id[str(tag)]
mi = monitoring_infos.int64_counter(
monitoring_infos.ELEMENT_COUNT_URN,
receiver.opcounter.element_counter.value(),
pcollection=pcollection_id)
infos[monitoring_infos.to_key(mi)] = mi
(unused_mean, sum, count, min, max) = (
receiver.opcounter.mean_byte_counter.value())
sampled_byte_count = monitoring_infos.int64_distribution(
monitoring_infos.SAMPLED_BYTE_SIZE_URN,
DistributionData(sum, count, min, max),
pcollection=pcollection_id)
infos[monitoring_infos.to_key(sampled_byte_count)] = sampled_byte_count
return infos
def pcollection_count_monitoring_infos(self, transform_id):
"""Returns the element count MonitoringInfo collected by this operation."""
if len(self.receivers) == 1:
# If there is exactly one output, we can unambiguously
# fix its name later, which we do.
# TODO(robertwb): Plumb the actual name here.
elem_count_mi = monitoring_infos.int64_counter(
monitoring_infos.ELEMENT_COUNT_URN,
self.receivers[0].opcounter.element_counter.value(),
ptransform=transform_id,
tag='ONLY_OUTPUT' if len(self.receivers) == 1 else str(None),
)
(unused_mean, sum, count, min,
max) = (self.receivers[0].opcounter.mean_byte_counter.value())
metric = metrics_pb2.Metric(
distribution_data=metrics_pb2.DistributionData(
int_distribution_data=metrics_pb2.IntDistributionData(
count=count, sum=sum, min=min, max=max)))
sampled_byte_count = monitoring_infos.int64_distribution(
monitoring_infos.SAMPLED_BYTE_SIZE_URN,
metric,
ptransform=transform_id,
tag='ONLY_OUTPUT' if len(self.receivers) == 1 else str(None),
)
return {
monitoring_infos.to_key(elem_count_mi): elem_count_mi,
monitoring_infos.to_key(sampled_byte_count): sampled_byte_count
}
return {}
def execution_time_monitoring_infos(self, transform_id):
total_time_spent_msecs = (
self.scoped_start_state.sampled_msecs_int()
+ self.scoped_process_state.sampled_msecs_int()
+ self.scoped_finish_state.sampled_msecs_int())
mis = [
monitoring_infos.int64_counter(
monitoring_infos.START_BUNDLE_MSECS_URN,
self.scoped_start_state.sampled_msecs_int(),
ptransform=transform_id
),
monitoring_infos.int64_counter(
monitoring_infos.PROCESS_BUNDLE_MSECS_URN,
self.scoped_process_state.sampled_msecs_int(),
ptransform=transform_id
),
monitoring_infos.int64_counter(
monitoring_infos.FINISH_BUNDLE_MSECS_URN,
self.scoped_finish_state.sampled_msecs_int(),
ptransform=transform_id
),
monitoring_infos.int64_counter(
monitoring_infos.TOTAL_MSECS_URN,
total_time_spent_msecs,
ptransform=transform_id
),
]
return {monitoring_infos.to_key(mi) : mi for mi in mis}
def to_runner_api_monitoring_infos(self, transform_id):
"""Returns a list of MonitoringInfos for the metrics in this container."""
all_user_metrics = []
for k, v in self.counters.items():
all_user_metrics.append(
monitoring_infos.int64_user_counter(
k.namespace,
k.name,
v.to_runner_api_monitoring_info(),
ptransform=transform_id))
for k, v in self.distributions.items():
all_user_metrics.append(
monitoring_infos.int64_user_distribution(
k.namespace,
k.name,
v.get_cumulative().to_runner_api_monitoring_info(),
ptransform=transform_id))
for k, v in self.gauges.items():
all_user_metrics.append(
monitoring_infos.int64_user_gauge(
k.namespace,
k.name,
v.get_cumulative().to_runner_api_monitoring_info(),
ptransform=transform_id))
return {monitoring_infos.to_key(mi): mi for mi in all_user_metrics}
def execution_time_monitoring_infos(self, transform_id):
total_time_spent_msecs = (
self.scoped_start_state.sampled_msecs_int()
+ self.scoped_process_state.sampled_msecs_int()
+ self.scoped_finish_state.sampled_msecs_int())
mis = [
monitoring_infos.int64_counter(
monitoring_infos.START_BUNDLE_MSECS_URN,
self.scoped_start_state.sampled_msecs_int(),
ptransform=transform_id
),
monitoring_infos.int64_counter(
monitoring_infos.PROCESS_BUNDLE_MSECS_URN,
self.scoped_process_state.sampled_msecs_int(),
ptransform=transform_id
),
monitoring_infos.int64_counter(
monitoring_infos.FINISH_BUNDLE_MSECS_URN,
self.scoped_finish_state.sampled_msecs_int(),
ptransform=transform_id
),
monitoring_infos.int64_counter(
monitoring_infos.TOTAL_MSECS_URN,
total_time_spent_msecs,
ptransform=transform_id
),
]
return {monitoring_infos.to_key(mi) : mi for mi in mis}
def to_runner_api_monitoring_infos(self, transform_id):
"""Returns a list of MonitoringInfos for the metrics in this container."""
all_user_metrics = [
cell.to_runner_api_monitoring_info(key.metric_name, transform_id)
for key, cell in self.metrics.items()
]
return {monitoring_infos.to_key(mi): mi for mi in all_user_metrics}
def monitoring_infos(self):
"""Returns the list of MonitoringInfos collected processing this bundle."""
# Construct a new dict first to remove duplciates.
all_monitoring_infos_dict = {}
for transform_id, op in self.ops.items():
for mi in op.monitoring_infos(transform_id).values():
fixed_mi = self._fix_output_tags_monitoring_info(transform_id, mi)
all_monitoring_infos_dict[monitoring_infos.to_key(fixed_mi)] = fixed_mi
return list(all_monitoring_infos_dict.values())
def monitoring_infos(self):
"""Returns the list of MonitoringInfos collected processing this bundle."""
# Construct a new dict first to remove duplciates.
all_monitoring_infos_dict = {}
for transform_id, op in self.ops.items():
for mi in op.monitoring_infos(transform_id).values():
fixed_mi = self._fix_output_tags_monitoring_info(transform_id, mi)
all_monitoring_infos_dict[monitoring_infos.to_key(fixed_mi)] = fixed_mi
return list(all_monitoring_infos_dict.values())
def monitoring_infos(self, transform_id):
infos = super(DoOperation, self).monitoring_infos(transform_id)
if self.tagged_receivers:
for tag, receiver in self.tagged_receivers.items():
mi = monitoring_infos.int64_counter(
monitoring_infos.ELEMENT_COUNT_URN,
receiver.opcounter.element_counter.value(),
ptransform=transform_id,
tag=str(tag))
infos[monitoring_infos.to_key(mi)] = mi
return infos
def monitoring_infos(self, transform_id):
infos = super(DoOperation, self).monitoring_infos(transform_id)
if self.tagged_receivers:
for tag, receiver in self.tagged_receivers.items():
mi = monitoring_infos.int64_counter(
monitoring_infos.ELEMENT_COUNT_URN,
receiver.opcounter.element_counter.value(),
ptransform=transform_id,
tag=str(tag)
)
infos[monitoring_infos.to_key(mi)] = mi
return infos
def to_runner_api_monitoring_infos(self, transform_id):
# type: (str) -> Dict[FrozenSet, metrics_pb2.MonitoringInfo]
"""Returns a list of MonitoringInfos for the metrics in this container."""
with self.lock:
items = list(self.metrics.items())
all_metrics = [
cell.to_runner_api_monitoring_info(key.metric_name, transform_id)
for key, cell in items
]
return {
monitoring_infos.to_key(mi): mi
for mi in all_metrics if mi is not None
}
def element_count_monitoring_infos(self, transform_id):
"""Returns the element count MonitoringInfo collected by this operation."""
if len(self.receivers) == 1:
# If there is exactly one output, we can unambiguously
# fix its name later, which we do.
# TODO(robertwb): Plumb the actual name here.
mi = monitoring_infos.int64_counter(
monitoring_infos.ELEMENT_COUNT_URN,
self.receivers[0].opcounter.element_counter.value(),
ptransform=transform_id,
tag='ONLY_OUTPUT' if len(self.receivers) == 1 else str(None),
)
return {monitoring_infos.to_key(mi): mi}
return {}
def element_count_monitoring_infos(self, transform_id):
"""Returns the element count MonitoringInfo collected by this operation."""
if len(self.receivers) == 1:
# If there is exactly one output, we can unambiguously
# fix its name later, which we do.
# TODO(robertwb): Plumb the actual name here.
mi = monitoring_infos.int64_counter(
monitoring_infos.ELEMENT_COUNT_URN,
self.receivers[0].opcounter.element_counter.value(),
ptransform=transform_id,
tag='ONLY_OUTPUT' if len(self.receivers) == 1 else str(None),
)
return {monitoring_infos.to_key(mi) : mi}
return {}
def monitoring_infos(self):
"""Returns the list of MonitoringInfos collected processing this bundle."""
# Construct a new dict first to remove duplciates.
all_monitoring_infos_dict = {}
for transform_id, op in self.ops.items():
for mi in op.monitoring_infos(transform_id).values():
fixed_mi = self._fix_output_tags_monitoring_info(transform_id, mi)
all_monitoring_infos_dict[monitoring_infos.to_key(fixed_mi)] = fixed_mi
infos_list = list(all_monitoring_infos_dict.values())
def inject_pcollection(monitoring_info):
"""
If provided metric is element count metric:
Finds relevant transform output info in current process_bundle_descriptor
and adds tag with PCOLLECTION_LABEL and pcollection_id into monitoring
info.
"""
if monitoring_info.urn in URNS_NEEDING_PCOLLECTIONS:
if not monitoring_infos.PTRANSFORM_LABEL in monitoring_info.labels:
return
ptransform_label = monitoring_info.labels[
monitoring_infos.PTRANSFORM_LABEL]
if not monitoring_infos.TAG_LABEL in monitoring_info.labels:
return
tag_label = monitoring_info.labels[monitoring_infos.TAG_LABEL]
if not ptransform_label in self.process_bundle_descriptor.transforms:
return
if not tag_label in self.process_bundle_descriptor.transforms[
ptransform_label].outputs:
return
pcollection_name = (self.process_bundle_descriptor
.transforms[ptransform_label].outputs[tag_label])
monitoring_info.labels[
monitoring_infos.PCOLLECTION_LABEL] = pcollection_name
# Cleaning up labels that are not in specification.
monitoring_info.labels.pop(monitoring_infos.PTRANSFORM_LABEL)
monitoring_info.labels.pop(monitoring_infos.TAG_LABEL)
for mi in infos_list:
inject_pcollection(mi)
return infos_list
def monitoring_infos(self):
"""Returns the list of MonitoringInfos collected processing this bundle."""
# Construct a new dict first to remove duplciates.
all_monitoring_infos_dict = {}
for transform_id, op in self.ops.items():
for mi in op.monitoring_infos(transform_id).values():
fixed_mi = self._fix_output_tags_monitoring_info(transform_id, mi)
all_monitoring_infos_dict[monitoring_infos.to_key(fixed_mi)] = fixed_mi
infos_list = list(all_monitoring_infos_dict.values())
def inject_pcollection_into_element_count(monitoring_info):
"""
If provided metric is element count metric:
Finds relevant transform output info in current process_bundle_descriptor
and adds tag with PCOLLECTION_LABEL and pcollection_id into monitoring
info.
"""
if monitoring_info.urn == monitoring_infos.ELEMENT_COUNT_URN:
if not monitoring_infos.PTRANSFORM_LABEL in monitoring_info.labels:
return
ptransform_label = monitoring_info.labels[
monitoring_infos.PTRANSFORM_LABEL]
if not monitoring_infos.TAG_LABEL in monitoring_info.labels:
return
tag_label = monitoring_info.labels[monitoring_infos.TAG_LABEL]
if not ptransform_label in self.process_bundle_descriptor.transforms:
return
if not tag_label in self.process_bundle_descriptor.transforms[
ptransform_label].outputs:
return
pcollection_name = (self.process_bundle_descriptor
.transforms[ptransform_label].outputs[tag_label])
monitoring_info.labels[
monitoring_infos.PCOLLECTION_LABEL] = pcollection_name
# Cleaning up labels that are not in specification.
monitoring_info.labels.pop(monitoring_infos.PTRANSFORM_LABEL)
monitoring_info.labels.pop(monitoring_infos.TAG_LABEL)
for mi in infos_list:
inject_pcollection_into_element_count(mi)
return infos_list
def getShortId(self, monitoring_info):
# type: (metrics_pb2.MonitoringInfo) -> str
""" Returns the assigned shortId for a given MonitoringInfo, assigns one if
not assigned already.
"""
key = monitoring_infos.to_key(monitoring_info)
with self._lock:
try:
return self._infoKeyToShortId[key]
except KeyError:
self._lastShortId += 1
# Convert to a hex string (and drop the '0x') for some compression
shortId = hex(self._lastShortId)[2:]
payload_cleared = metrics_pb2.MonitoringInfo()
payload_cleared.CopyFrom(monitoring_info)
payload_cleared.ClearField('payload')
self._infoKeyToShortId[key] = shortId
self._shortIdToInfo[shortId] = payload_cleared
return shortId
def to_runner_api_monitoring_infos(self, transform_id):
"""Returns a list of MonitoringInfos for the metrics in this container."""
all_user_metrics = []
for k, v in self.counters.items():
all_user_metrics.append(monitoring_infos.int64_counter(
user_metric_urn(k.namespace, k.name),
v.to_runner_api_monitoring_info(),
ptransform=transform_id
))
for k, v in self.distributions.items():
all_user_metrics.append(monitoring_infos.int64_distribution(
user_distribution_metric_urn(k.namespace, k.name),
v.get_cumulative().to_runner_api_monitoring_info(),
ptransform=transform_id
))
for k, v in self.gauges.items():
all_user_metrics.append(monitoring_infos.int64_gauge(
user_metric_urn(k.namespace, k.name),
v.get_cumulative().to_runner_api_monitoring_info(),
ptransform=transform_id
))
return {monitoring_infos.to_key(mi) : mi for mi in all_user_metrics}
