def test_dimension_object_builder_does_build_with_page_level_data_from_grouped_table(): data_source = DataSourceFactory.build( title="DWP Stats", source_url="http://dwp.gov.uk", publisher__name="Department for Work and Pensions" ) measure_version = MeasureVersionWithDimensionFactory( title="Test Measure Page", area_covered=[UKCountry.ENGLAND], data_sources=[data_source], dimensions__dimension_table__table_object=grouped_table(), measure__slug="test-measure-page-slug", ) # given - a table without a category_caption value builder = DimensionObjectBuilder() dimension = measure_version.dimensions[0] # when we process the object dimension_object = builder.build(dimension) # then the measure level info should be brought through assert dimension_object["context"]["measure"] == "Test Measure Page" assert dimension_object["context"]["measure_slug"] == "test-measure-page-slug" assert dimension_object["context"]["location"] == "England" assert dimension_object["context"]["title"] == "DWP Stats" assert dimension_object["context"]["source_url"] == "http://dwp.gov.uk" assert dimension_object["context"]["publisher"] == "Department for Work and Pensions"
def test_table_object_builder_does_build_object_from_grouped_table( stub_page_with_grouped_table): # given - a table without a category_caption value builder = DimensionObjectBuilder() dimension = stub_page_with_grouped_table.dimensions[0] # when we process the object table_object = builder.build(dimension) # then the header for the returned table should match the ones from the simple table assert table_object is not None
def test_table_object_builder_does_build_object_from_simple_table(): measure_version = MeasureVersionWithDimensionFactory(dimensions__dimension_table__table_object=simple_table()) # given - a table without a category_caption value builder = DimensionObjectBuilder() dimension = measure_version.dimensions[0] # when we process the object table_object = builder.build(dimension) # then the header for the returned table should match the ones from the simple table assert table_object is not None assert table_object.get("table").get("title") == "Title of simple table"
def test_table_object_builder_does_build_with_dimension_level_data_from_grouped_table( stub_page_with_grouped_table): # given - a table without a category_caption value builder = DimensionObjectBuilder() dimension = stub_page_with_grouped_table.dimensions[0] # when we process the object dimension_object = builder.build(dimension) # then the dimension level info should be brought through assert dimension_object["context"]["dimension"] == "stub dimension" assert dimension_object["context"]["guid"] == "stub_dimension" assert dimension_object["context"]["time_period"] == "stub_timeperiod"
def dimension_file_table_download(topic_slug, subtopic_slug, measure_slug, version, dimension_guid): try: *_, dimension = page_service.get_measure_version_hierarchy( topic_slug, subtopic_slug, measure_slug, version, dimension_guid=dimension_guid) dimension_obj = DimensionObjectBuilder.build(dimension) data = write_dimension_tabular_csv(dimension=dimension_obj) response = make_response(data) if dimension_obj["context"][ "dimension"] and dimension_obj["context"]["dimension"] != "": filename = "%s-table.csv" % cleanup_filename( dimension_obj["context"]["dimension"].lower()) else: filename = "%s-table.csv" % cleanup_filename( dimension_obj["context"]["guid"]) response.headers["Content-Type"] = "text/csv" response.headers[ "Content-Disposition"] = 'attachment; filename="%s"' % filename return response except DimensionNotFoundException: abort(404)
def process_dimensions(measure_version, slug): if measure_version.dimensions: download_dir = os.path.join(slug, "downloads") os.makedirs(download_dir, exist_ok=True) for dimension in measure_version.dimensions: if ( dimension.dimension_chart and dimension.dimension_chart.chart_object and dimension.dimension_chart.chart_object["type"] != "panel_bar_chart" ): chart_dir = "%s/charts" % slug os.makedirs(chart_dir, exist_ok=True) dimension_obj = DimensionObjectBuilder.build(dimension) output = write_dimension_csv(dimension=dimension_obj) try: file_path = os.path.join(download_dir, dimension.static_file_name) with open(file_path, "w") as dimension_file: dimension_file.write(output) except Exception as e: print(f"Could not write file path {file_path}") print(e) if dimension.dimension_table and dimension.dimension_table.table_object: table_output = write_dimension_tabular_csv(dimension=dimension_obj) table_file_path = os.path.join(download_dir, dimension.static_table_file_name) with open(table_file_path, "w") as dimension_file: dimension_file.write(table_output)
def test_dimension_object_builder_does_build_with_page_level_data_from_grouped_table( stub_page_with_grouped_table): # given - a table without a category_caption value builder = DimensionObjectBuilder() dimension = stub_page_with_grouped_table.dimensions[0] # when we process the object dimension_object = builder.build(dimension) # then the measure level info should be brought through assert dimension_object["context"]["measure"] == "Test Measure Page" assert dimension_object["context"]["measure_guid"] == "test-measure-page" assert dimension_object["context"]["measure_uri"] == "test-measure-page" assert dimension_object["context"]["location"] == "UK" assert dimension_object["context"]["source_text"] == "DWP Stats" assert dimension_object["context"]["source_url"] == "http://dwp.gov.uk" assert dimension_object["context"][ "department"] == "Department for Work and Pensions"
def test_table_object_builder_does_build_with_dimension_level_data_from_grouped_table(): measure_version = MeasureVersionWithDimensionFactory( dimensions__title="Dimension title", dimensions__guid="dimension-guid", dimensions__time_period="dimension-time-period", dimensions__dimension_table__table_object=grouped_table(), ) # given - a table without a category_caption value builder = DimensionObjectBuilder() dimension = measure_version.dimensions[0] # when we process the object dimension_object = builder.build(dimension) # then the dimension level info should be brought through assert dimension_object["context"]["dimension"] == "Dimension title" assert dimension_object["context"]["guid"] == "dimension-guid" assert dimension_object["context"]["time_period"] == "dimension-time-period"
def test_if_dimension_has_chart_and_table_download_table_source_data(logged_in_rdu_user, test_app_client): from tests.test_data.chart_and_table import ( chart, chart_settings_and_source_data, table, table_settings_and_source_data, ) measure_version = MeasureVersionWithDimensionFactory( dimensions__title="Dimension title", # Chart dimensions__dimension_chart__chart_object=chart, dimensions__dimension_chart__settings_and_source_data=chart_settings_and_source_data, dimensions__dimension_chart__classification=ClassificationFactory(id="2A"), dimensions__dimension_chart__includes_parents=False, dimensions__dimension_chart__includes_all=True, dimensions__dimension_chart__includes_unknown=False, # Table dimensions__dimension_table__table_object=table, dimensions__dimension_table__settings_and_source_data=table_settings_and_source_data, dimensions__dimension_table__classification=ClassificationFactory(id="5A"), dimensions__dimension_table__includes_parents=True, dimensions__dimension_table__includes_all=False, dimensions__dimension_table__includes_unknown=True, ) # GIVEN # we have a dimension with table and chart data dimension = measure_version.dimensions[0] resp = test_app_client.get( url_for( "static_site.dimension_file_download", topic_slug=measure_version.measure.subtopic.topic.slug, subtopic_slug=measure_version.measure.subtopic.slug, measure_slug=measure_version.measure.slug, version=measure_version.version, dimension_guid=dimension.guid, ) ) # WHEN # we generate a plain table csv d = DimensionObjectBuilder.build(dimension) expected_csv = write_dimension_csv(dimension=d) # THEN # we get a return assert resp.status_code == 200 assert resp.content_type == "text/csv" assert resp.headers["Content-Disposition"] == 'attachment; filename="dimension-title.csv"' # from the data in the table (not chart) actual_data = resp.data.decode("utf-8") assert actual_data == expected_csv
def test_table_object_builder_does_build_with_dimension_level_data_from_simple_table(): measure_version = MeasureVersionWithDimensionFactory( title="Test Measure Page", area_covered=[UKCountry.ENGLAND], dimensions__title="Dimension title", dimensions__guid="dimension-guid", dimensions__time_period="dimension-time-period", dimensions__dimension_table__table_object=simple_table(), measure__slug="test-measure-page-slug", ) # given - a table without a category_caption value builder = DimensionObjectBuilder() dimension = measure_version.dimensions[0] # when we process the object dimension_object = builder.build(dimension) # then the dimension level info should be brought through assert dimension_object["context"]["dimension"] == "Dimension title" assert dimension_object["context"]["guid"] == "dimension-guid" assert dimension_object["context"]["time_period"] == "dimension-time-period"
def dimension_for_api(dimension): dimension_object = DimensionObjectBuilder.build(dimension) metadata = dimension_object["context"] title = metadata.get("dimension") if "table" in dimension_object: data = dimension_object["table"]["data"] elif "chart" in dimension_object: data = dimension_object["chart"]["data"] else: data = [] return {"_dimension": title, "metadata": metadata, "data": data}
def process_dimensions(page, uri): if page.dimensions: download_dir = os.path.join(uri, "downloads") os.makedirs(download_dir, exist_ok=True) else: return dimensions = [] for d in page.dimensions: if d.chart and d.chart["type"] != "panel_bar_chart": chart_dir = "%s/charts" % uri os.makedirs(chart_dir, exist_ok=True) dimension_obj = DimensionObjectBuilder.build(d) output = write_dimension_csv(dimension=dimension_obj) if d.title: filename = "%s.csv" % cleanup_filename(d.title) table_filename = "%s-table.csv" % cleanup_filename(d.title) else: filename = "%s.csv" % d.guid table_filename = "%s-table.csv" % d.guid try: file_path = os.path.join(download_dir, filename) with open(file_path, "w") as dimension_file: dimension_file.write(output) except Exception as e: print(f"Could not write file path {file_path}") print(e) d_as_dict = d.to_dict() d_as_dict["static_file_name"] = filename if d.table: table_output = write_dimension_tabular_csv(dimension=dimension_obj) table_file_path = os.path.join(download_dir, table_filename) with open(table_file_path, "w") as dimension_file: dimension_file.write(table_output) d_as_dict["static_table_file_name"] = table_filename dimensions.append(d_as_dict) return dimensions
def test_if_dimension_has_chart_and_table_download_table_source_data( app, mock_rdu_user, test_app_client, stub_topic_page, stub_subtopic_page, stub_page_with_dimension_and_chart_and_table, ): # GIVEN # we have a dimension with table and chart data dimension = stub_page_with_dimension_and_chart_and_table.dimensions[0] with test_app_client: test_app_client.post(url_for("security.login"), data={ "email": mock_rdu_user.email, "password": "******" }) resp = test_app_client.get( url_for( "static_site.dimension_file_download", topic_uri=stub_topic_page.uri, subtopic_uri=stub_subtopic_page.uri, measure_uri=stub_page_with_dimension_and_chart_and_table.uri, version=stub_page_with_dimension_and_chart_and_table.version, dimension_guid=dimension.guid, )) # WHEN # we generate a plain table csv d = DimensionObjectBuilder.build(dimension) expected_csv = write_dimension_csv(dimension=d) # THEN # we get a return assert resp.status_code == 200 assert resp.content_type == "text/csv" assert resp.headers[ "Content-Disposition"] == 'attachment; filename="stub-dimension.csv"' # from the data in the table (not chart) actual_data = resp.data.decode("utf-8") assert actual_data == expected_csv
def dimension_file_download(topic_uri, subtopic_uri, measure_uri, version, dimension_guid): try: page = page_service.get_page_by_uri_and_type(measure_uri, "measure", version) dimension_obj = DimensionObjectBuilder.build(page.get_dimension(dimension_guid)) data = write_dimension_csv(dimension=dimension_obj) response = make_response(data) if dimension_obj["context"]["dimension"] and dimension_obj["context"]["dimension"] != "": filename = "%s.csv" % cleanup_filename(dimension_obj["context"]["dimension"]) else: filename = "%s.csv" % cleanup_filename(dimension_obj["context"]["guid"]) response.headers["Content-Type"] = "text/csv" response.headers["Content-Disposition"] = 'attachment; filename="%s"' % filename return response except DimensionNotFoundException as e: abort(404)