def process(self, element, *args, **kwargs):
pipeline_options = kwargs
pipeline_job_id = job_id(pipeline_options)
(metric_key, value) = element
if value is None:
# Due to how the pipeline arrives at this function, this should be impossible.
raise ValueError("No value associated with this metric key.")
# Convert JSON string to dictionary
dict_metric_key = json.loads(metric_key)
metric_type = dict_metric_key.get('metric_type')
if metric_type == MetricType.REFERRAL.value:
dict_metric_key['count'] = value
program_metric = ProgramReferralMetric.build_from_metric_key_group(
dict_metric_key, pipeline_job_id)
else:
logging.error("Unexpected metric of type: %s",
dict_metric_key.get('metric_type'))
return
if program_metric:
yield program_metric
def process(self, element, *args, **kwargs):
"""Converts a program metric key into a ProgramMetric.
The pipeline options are sent in as the **kwargs so that the job_id(pipeline_options) function can be called to
retrieve the job_id.
Args:
element: A tuple containing the dictionary for the program metric, and the value of that metric.
**kwargs: This should be a dictionary with values for the following keys:
- runner: Either 'DirectRunner' or 'DataflowRunner'
- project: GCP project ID
- job_name: Name of the pipeline job
- region: Region where the pipeline job is running
- job_timestamp: Timestamp for the current job, to be used if the job is running locally.
Yields:
The ProgramMetric.
"""
pipeline_options = kwargs
pipeline_job_id = job_id(pipeline_options)
(dict_metric_key, value) = element
if value is None:
# Due to how the pipeline arrives at this function, this should be impossible.
raise ValueError("No value associated with this metric key.")
if not dict_metric_key:
# Due to how the pipeline arrives at this function, this should be impossible.
raise ValueError("Empty dict_metric_key.")
metric_type = dict_metric_key.pop('metric_type')
if dict_metric_key.get('person_id') is not None:
# The count value for all person-level metrics should be 1
value = 1
if metric_type == ProgramMetricType.PROGRAM_REFERRAL:
dict_metric_key['count'] = value
program_metric = ProgramReferralMetric.build_from_metric_key_group(
dict_metric_key, pipeline_job_id)
elif metric_type == ProgramMetricType.PROGRAM_PARTICIPATION:
dict_metric_key['count'] = value
program_metric = ProgramParticipationMetric.build_from_metric_key_group(
dict_metric_key, pipeline_job_id)
else:
logging.error("Unexpected metric of type: %s", metric_type)
return
if program_metric:
yield program_metric