def expand(self, pcol: pvalue.PCollection) -> pvalue.PCollection:
backend = pipeline_dp.BeamBackend()
dp_engine = pipeline_dp.DPEngine(self._budget_accountant, backend)
params = pipeline_dp.AggregateParams(
noise_kind=self._privacy_id_count_params.noise_kind,
metrics=[pipeline_dp.Metrics.PRIVACY_ID_COUNT],
max_partitions_contributed=self._privacy_id_count_params.
max_partitions_contributed,
max_contributions_per_partition=1,
public_partitions=self._privacy_id_count_params.public_partitions)
data_extractors = pipeline_dp.DataExtractors(
partition_extractor=lambda x: self._privacy_id_count_params.
partition_extractor(x[1]),
privacy_id_extractor=lambda x: x[0],
# PrivacyIdCount ignores values.
value_extractor=lambda x: None)
dp_result = dp_engine.aggregate(pcol, params, data_extractors)
# dp_result : (partition_key, [dp_privacy_id_count])
# aggregate() returns a namedtuple of metrics for each partition key.
# Here is only one metric - privacy_id_count. Extract it from the list.
dp_result = backend.map_values(dp_result, lambda v: v.privacy_id_count,
"Extract privacy_id_count")
# dp_result : (partition_key, dp_privacy_id_count)
return dp_result
def expand(self, pcol: pvalue.PCollection) -> pvalue.PCollection:
backend = pipeline_dp.BeamBackend()
dp_engine = pipeline_dp.DPEngine(self._budget_accountant, backend)
params = pipeline_dp.AggregateParams(
noise_kind=self._mean_params.noise_kind,
metrics=[pipeline_dp.Metrics.MEAN],
max_partitions_contributed=self._mean_params.
max_partitions_contributed,
max_contributions_per_partition=self._mean_params.
max_contributions_per_partition,
min_value=self._mean_params.min_value,
max_value=self._mean_params.max_value,
public_partitions=self._mean_params.public_partitions)
data_extractors = pipeline_dp.DataExtractors(
partition_extractor=lambda x: self._mean_params.partition_extractor(
x[1]),
privacy_id_extractor=lambda x: x[0],
value_extractor=lambda x: self._mean_params.value_extractor(x[1]))
dp_result = dp_engine.aggregate(pcol, params, data_extractors)
# dp_result : (partition_key, [dp_sum])
# aggregate() returns a namedtuple of metrics for each partition key.
# Here is only one metric - mean. Extract it from the list.
dp_result = backend.map_values(dp_result, lambda v: v.mean,
"Extract mean")
# dp_result : (partition_key, dp_sum)
return dp_result
def expand(self, pcol: pvalue.PCollection) -> pvalue.PCollection:
backend = pipeline_dp.BeamBackend()
dp_engine = pipeline_dp.DPEngine(self._budget_accountant, backend)
params = pipeline_dp.AggregateParams(
noise_kind=self._count_params.noise_kind,
metrics=[pipeline_dp.Metrics.COUNT],
max_partitions_contributed=self._count_params.
max_partitions_contributed,
max_contributions_per_partition=self._count_params.
max_contributions_per_partition,
public_partitions=self._count_params.public_partitions)
data_extractors = pipeline_dp.DataExtractors(
partition_extractor=lambda x: self._count_params.
partition_extractor(x[1]),
privacy_id_extractor=lambda x: x[0],
# Count calculates the number of elements per partition key and
# doesn't use value extractor.
value_extractor=lambda x: None)
dp_result = dp_engine.aggregate(pcol, params, data_extractors)
# dp_result : (partition_key, [dp_count])
# aggregate() returns a namedtuple of metrics for each partition key.
# Here is only one metric - count. Extract it from the list.
dp_result = backend.map_values(dp_result, lambda v: v.count,
"Extract sum")
# dp_result : (partition_key, dp_count)
return dp_result
def test_run_e2e_beam(self):
with test_pipeline.TestPipeline() as p:
input = p | "Create input" >> beam.Create(list(range(10)))
output = self.run_e2e_private_partition_selection_large_budget(
input, pipeline_dp.BeamBackend())
beam_util.assert_that(output, beam_util.is_not_empty())
def compute_on_beam():
runner = fn_api_runner.FnApiRunner() # local runner
with beam.Pipeline(runner=runner) as pipeline:
movie_views = pipeline | beam.io.ReadFromText(
FLAGS.input_file) | beam.ParDo(ParseFile())
pipeline_backend = pipeline_dp.BeamBackend()
dp_result = calculate_private_result(movie_views, pipeline_backend)
dp_result | beam.io.WriteToText(FLAGS.output_file)
def expand(self, pcol: pvalue.PCollection) -> pvalue.PCollection:
backend = pipeline_dp.BeamBackend()
dp_engine = pipeline_dp.DPEngine(self._budget_accountant, backend)
data_extractors = pipeline_dp.DataExtractors(
partition_extractor=lambda x: self._partition_extractor(x[1]),
privacy_id_extractor=lambda x: x[0])
dp_result = dp_engine.select_partitions(pcol,
self._select_partitions_params,
data_extractors)
return dp_result
def _create_dp_engine(self):
backend = pipeline_dp.BeamBackend()
return backend, pipeline_dp.DPEngine(self._budget_accountant, backend)