def test_export_dashboard_data_to_cloud_storage_update_all_views(
self,
mock_view_exporter,
mock_view_update_manager_deploy,
mock_view_update_manager_rematerialize,
mock_view_builders_by_namespace,
) -> None:
"""Tests that all views in the namespace are updated before the export when the export name is in
export_config.NAMESPACES_REQUIRING_FULL_UPDATE."""
self.mock_export_config.NAMESPACES_REQUIRING_FULL_UPDATE = [
self.mock_big_query_view_namespace
]
mock_view_builders_by_namespace.return_value = {
self.mock_big_query_view_namespace: self.view_buidlers_for_dataset
}
view_export_manager.export_view_data_to_cloud_storage(
self.mock_state_code, mock_view_exporter)
mock_view_update_manager_deploy.assert_called_with(
self.mock_big_query_view_namespace,
mock_view_builders_by_namespace[
self.mock_big_query_view_namespace],
)
mock_view_update_manager_rematerialize.assert_called_once()
def test_export_dashboard_data_to_cloud_storage_update_materialized_views_only(
self,
mock_view_exporter: Mock,
mock_view_update_manager_rematerialize: Mock,
mock_deployed_views: Mock,
) -> None:
"""Tests that only materialized views in the namespace are updated before the export when the export name is not
in export_config.NAMESPACES_REQUIRING_FULL_UPDATE."""
self.mock_export_config.NAMESPACES_REQUIRING_FULL_UPDATE = [
"OTHER_NAMESPACE"
]
mock_deployed_views.DEPLOYED_VIEW_BUILDERS_BY_NAMESPACE = {
self.mock_big_query_view_namespace: self.view_builders_for_dataset
}
view_export_manager.export_view_data_to_cloud_storage(
self.mock_export_name, override_view_exporter=mock_view_exporter)
mock_view_update_manager_rematerialize.assert_called_with(
view_source_table_datasets=VIEW_SOURCE_TABLE_DATASETS,
all_view_builders=DEPLOYED_VIEW_BUILDERS,
views_to_update=[
view.build() for view in self.view_builders_for_dataset
],
)
def test_export_dashboard_data_to_cloud_storage(
self, mock_view_exporter,
mock_view_update_manager_rematerialize) -> None:
"""Tests the table is created from the view and then extracted."""
view_export_manager.export_view_data_to_cloud_storage(
self.mock_state_code, mock_view_exporter)
view = self.mock_view_builder.build()
metric_view = self.mock_metric_view_builder.build()
view_export_configs = [
ExportBigQueryViewConfig(
view=view,
view_filter_clause=" WHERE state_code = 'US_XX'",
intermediate_table_name=f"{view.view_id}_table_US_XX",
output_directory=GcsfsDirectoryPath.from_absolute_path(
"gs://{project_id}-dataset-location/subdirectory/{state_code}"
.format(
project_id=self.mock_project_id,
state_code="US_XX",
)),
export_output_formats=[ExportOutputFormatType.JSON],
),
ExportBigQueryViewConfig(
view=metric_view,
view_filter_clause=" WHERE state_code = 'US_XX'",
intermediate_table_name=f"{view.view_id}_table_US_XX",
output_directory=GcsfsDirectoryPath.from_absolute_path(
"gs://{project_id}-dataset-location/subdirectory/{state_code}"
.format(
project_id=self.mock_project_id,
state_code="US_XX",
)),
export_output_formats=[
ExportOutputFormatType.JSON,
ExportOutputFormatType.METRIC,
],
),
]
mock_view_update_manager_rematerialize.assert_called()
mock_view_exporter.export_and_validate.assert_has_calls(
[
mock.call([]), # CSV export
mock.call([
view_export_configs[1].pointed_to_staging_subdirectory()
]), # JSON export
mock.call([
conf.pointed_to_staging_subdirectory()
for conf in view_export_configs
]), # METRIC export
],
any_order=True,
)
def test_export_dashboard_data_to_cloud_storage_value_error(
self, mock_view_exporter,
mock_view_update_manager_rematerialize) -> None:
"""Tests the table is created from the view and then extracted."""
mock_view_exporter.export_and_validate.side_effect = ValueError
with self.assertRaises(ValueError):
view_export_manager.export_view_data_to_cloud_storage(
self.mock_state_code, mock_view_exporter)
# Just the metric export is attempted and then the raise stops subsequent checks from happening
mock_view_update_manager_rematerialize.assert_called_once()
def test_export_dashboard_data_to_cloud_storage_validation_error(
self, mock_view_exporter: Mock,
mock_view_update_manager_rematerialize: Mock) -> None:
"""Tests the table is created from the view and then extracted."""
mock_view_exporter.export_and_validate.side_effect = ViewExportValidationError
# Should not throw
view_export_manager.export_view_data_to_cloud_storage(
self.mock_export_name, override_view_exporter=mock_view_exporter)
# Just the metric export is attempted and then the raise stops subsequent checks from happening
mock_view_update_manager_rematerialize.assert_called_once()
def test_raise_exception_no_export_matched(
self, mock_view_exporter,
mock_view_update_manager_rematerialize) -> None:
# pylint: disable=unused-argument
"""Tests the table is created from the view and then extracted."""
self.mock_export_config.NAMESPACE_TO_UPDATE_FOR_EXPORT_FILTER = {
"US_YY": "NAMESPACE"
}
with self.assertRaises(ValueError) as e:
view_export_manager.export_view_data_to_cloud_storage(
"US_YY", mock_view_exporter)
self.assertEqual(
str(e.exception),
"Export filter did not match any export configs:",
" US_YY",
)
def export_metrics_from_dataset_to_gcs(
destination_bucket: str,
project_id: str,
export_name: str,
state_code: Optional[str],
sandbox_dataset_prefix: Optional[str],
) -> None:
"""Exports metric files into a sandbox GCS bucket."""
sandbox_dataset_overrides = None
if sandbox_dataset_prefix:
sandbox_dataset_overrides = dataset_overrides_for_deployed_view_datasets(
view_dataset_override_prefix=sandbox_dataset_prefix,
)
if destination_bucket in get_protected_buckets(project_id):
raise ValueError(
f"Must specify a destination_bucket that is not a protected bucket. "
f"Protected buckets are: {get_protected_buckets(project_id)}"
)
product_configs = ProductConfigs.from_file(path=PRODUCTS_CONFIG_PATH)
_ = product_configs.get_export_config(
export_job_name=export_name, state_code=state_code
)
export_view_data_to_cloud_storage(
export_job_name=export_name,
state_code=state_code,
should_materialize_views=False,
destination_override=destination_bucket,
dataset_overrides=sandbox_dataset_overrides,
)
logging.info(
"Done exporting metrics from sandbox with prefix [%s] to GCS bucket [%s].",
sandbox_dataset_prefix,
destination_bucket,
)
def test_export_dashboard_data_to_cloud_storage_update_materialized_views_only(
self,
mock_view_exporter,
mock_view_update_manager_rematerialize,
mock_view_builders_by_namespace,
) -> None:
"""Tests that only materialized views in the namespace are updated before the export when the export name is not
in export_config.NAMESPACES_REQUIRING_FULL_UPDATE."""
self.mock_export_config.NAMESPACES_REQUIRING_FULL_UPDATE = [
"OTHER_NAMESPACE"
]
mock_view_builders_by_namespace.return_value = {
self.mock_big_query_view_namespace: self.view_buidlers_for_dataset
}
view_export_manager.export_view_data_to_cloud_storage(
self.mock_state_code, mock_view_exporter)
mock_view_update_manager_rematerialize.assert_called_with(
bq_view_namespace=self.mock_big_query_view_namespace,
candidate_view_builders=mock_view_builders_by_namespace[
self.mock_big_query_view_namespace],
)
def test_export_dashboard_data_to_cloud_storage_state_agnostic(
self, mock_view_exporter: Mock,
mock_view_update_manager_rematerialize: Mock) -> None:
"""Tests the table is created from the view and then extracted, where the export is not state-specific."""
state_agnostic_dataset_export_configs = {
self.mock_export_name:
ExportViewCollectionConfig(
view_builders_to_export=self.view_builders_for_dataset,
output_directory_uri_template=
"gs://{project_id}-bucket-without-state-codes",
export_name=self.mock_export_name,
bq_view_namespace=self.mock_big_query_view_namespace,
),
}
self.mock_export_config.VIEW_COLLECTION_EXPORT_INDEX = (
state_agnostic_dataset_export_configs)
view_export_manager.export_view_data_to_cloud_storage(
export_job_name=self.mock_export_name,
override_view_exporter=mock_view_exporter,
)
view = self.mock_view_builder.build()
metric_view = self.mock_metric_view_builder.build()
view_export_configs = [
ExportBigQueryViewConfig(
bq_view_namespace=self.mock_big_query_view_namespace,
view=view,
view_filter_clause=None,
intermediate_table_name=f"{view.view_id}_table",
output_directory=GcsfsDirectoryPath.from_absolute_path(
"gs://{project_id}-bucket-without-state-codes".format(
project_id=self.mock_project_id, )),
export_output_formats=[ExportOutputFormatType.JSON],
),
ExportBigQueryViewConfig(
bq_view_namespace=self.mock_big_query_view_namespace,
view=metric_view,
view_filter_clause=None,
intermediate_table_name=f"{view.view_id}_table",
output_directory=GcsfsDirectoryPath.from_absolute_path(
"gs://{project_id}-bucket-without-state-codes".format(
project_id=self.mock_project_id, )),
export_output_formats=[
ExportOutputFormatType.JSON,
ExportOutputFormatType.METRIC,
],
),
]
mock_view_update_manager_rematerialize.assert_called()
mock_view_exporter.export_and_validate.assert_has_calls(
[
mock.call([]), # CSV export
mock.call([
view_export_configs[1].pointed_to_staging_subdirectory()
]), # JSON export
mock.call([
conf.pointed_to_staging_subdirectory()
for conf in view_export_configs
]), # METRIC export
],
any_order=True,
)
