def dimension_file_table_download(topic_slug, subtopic_slug, measure_slug,
version, dimension_guid):
try:
*_, dimension = page_service.get_measure_version_hierarchy(
topic_slug,
subtopic_slug,
measure_slug,
version,
dimension_guid=dimension_guid)
dimension_obj = DimensionObjectBuilder.build(dimension)
data = write_dimension_tabular_csv(dimension=dimension_obj)
response = make_response(data)
if dimension_obj["context"][
"dimension"] and dimension_obj["context"]["dimension"] != "":
filename = "%s-table.csv" % cleanup_filename(
dimension_obj["context"]["dimension"].lower())
else:
filename = "%s-table.csv" % cleanup_filename(
dimension_obj["context"]["guid"])
response.headers["Content-Type"] = "text/csv"
response.headers[
"Content-Disposition"] = 'attachment; filename="%s"' % filename
return response
except DimensionNotFoundException:
abort(404)
def test_dimension_object_builder_does_build_with_page_level_data_from_grouped_table():
data_source = DataSourceFactory.build(
title="DWP Stats", source_url="http://dwp.gov.uk", publisher__name="Department for Work and Pensions"
)
measure_version = MeasureVersionWithDimensionFactory(
title="Test Measure Page",
area_covered=[UKCountry.ENGLAND],
data_sources=[data_source],
dimensions__dimension_table__table_object=grouped_table(),
measure__slug="test-measure-page-slug",
)
# given - a table without a category_caption value
builder = DimensionObjectBuilder()
dimension = measure_version.dimensions[0]
# when we process the object
dimension_object = builder.build(dimension)
# then the measure level info should be brought through
assert dimension_object["context"]["measure"] == "Test Measure Page"
assert dimension_object["context"]["measure_slug"] == "test-measure-page-slug"
assert dimension_object["context"]["location"] == "England"
assert dimension_object["context"]["title"] == "DWP Stats"
assert dimension_object["context"]["source_url"] == "http://dwp.gov.uk"
assert dimension_object["context"]["publisher"] == "Department for Work and Pensions"
def process_dimensions(measure_version, slug):
if measure_version.dimensions:
download_dir = os.path.join(slug, "downloads")
os.makedirs(download_dir, exist_ok=True)
for dimension in measure_version.dimensions:
if (
dimension.dimension_chart
and dimension.dimension_chart.chart_object
and dimension.dimension_chart.chart_object["type"] != "panel_bar_chart"
):
chart_dir = "%s/charts" % slug
os.makedirs(chart_dir, exist_ok=True)
dimension_obj = DimensionObjectBuilder.build(dimension)
output = write_dimension_csv(dimension=dimension_obj)
try:
file_path = os.path.join(download_dir, dimension.static_file_name)
with open(file_path, "w") as dimension_file:
dimension_file.write(output)
except Exception as e:
print(f"Could not write file path {file_path}")
print(e)
if dimension.dimension_table and dimension.dimension_table.table_object:
table_output = write_dimension_tabular_csv(dimension=dimension_obj)
table_file_path = os.path.join(download_dir, dimension.static_table_file_name)
with open(table_file_path, "w") as dimension_file:
dimension_file.write(table_output)
def test_if_dimension_has_chart_and_table_download_table_source_data(logged_in_rdu_user, test_app_client):
from tests.test_data.chart_and_table import (
chart,
chart_settings_and_source_data,
table,
table_settings_and_source_data,
)
measure_version = MeasureVersionWithDimensionFactory(
dimensions__title="Dimension title",
# Chart
dimensions__dimension_chart__chart_object=chart,
dimensions__dimension_chart__settings_and_source_data=chart_settings_and_source_data,
dimensions__dimension_chart__classification=ClassificationFactory(id="2A"),
dimensions__dimension_chart__includes_parents=False,
dimensions__dimension_chart__includes_all=True,
dimensions__dimension_chart__includes_unknown=False,
# Table
dimensions__dimension_table__table_object=table,
dimensions__dimension_table__settings_and_source_data=table_settings_and_source_data,
dimensions__dimension_table__classification=ClassificationFactory(id="5A"),
dimensions__dimension_table__includes_parents=True,
dimensions__dimension_table__includes_all=False,
dimensions__dimension_table__includes_unknown=True,
)
# GIVEN
# we have a dimension with table and chart data
dimension = measure_version.dimensions[0]
resp = test_app_client.get(
url_for(
"static_site.dimension_file_download",
topic_slug=measure_version.measure.subtopic.topic.slug,
subtopic_slug=measure_version.measure.subtopic.slug,
measure_slug=measure_version.measure.slug,
version=measure_version.version,
dimension_guid=dimension.guid,
)
)
# WHEN
# we generate a plain table csv
d = DimensionObjectBuilder.build(dimension)
expected_csv = write_dimension_csv(dimension=d)
# THEN
# we get a return
assert resp.status_code == 200
assert resp.content_type == "text/csv"
assert resp.headers["Content-Disposition"] == 'attachment; filename="dimension-title.csv"'
# from the data in the table (not chart)
actual_data = resp.data.decode("utf-8")
assert actual_data == expected_csv
def test_table_object_builder_does_build_object_from_grouped_table(
stub_page_with_grouped_table):
# given - a table without a category_caption value
builder = DimensionObjectBuilder()
dimension = stub_page_with_grouped_table.dimensions[0]
# when we process the object
table_object = builder.build(dimension)
# then the header for the returned table should match the ones from the simple table
assert table_object is not None
def test_table_object_builder_does_build_object_from_simple_table():
measure_version = MeasureVersionWithDimensionFactory(dimensions__dimension_table__table_object=simple_table())
# given - a table without a category_caption value
builder = DimensionObjectBuilder()
dimension = measure_version.dimensions[0]
# when we process the object
table_object = builder.build(dimension)
# then the header for the returned table should match the ones from the simple table
assert table_object is not None
assert table_object.get("table").get("title") == "Title of simple table"
def test_table_object_builder_does_build_with_dimension_level_data_from_grouped_table(
stub_page_with_grouped_table):
# given - a table without a category_caption value
builder = DimensionObjectBuilder()
dimension = stub_page_with_grouped_table.dimensions[0]
# when we process the object
dimension_object = builder.build(dimension)
# then the dimension level info should be brought through
assert dimension_object["context"]["dimension"] == "stub dimension"
assert dimension_object["context"]["guid"] == "stub_dimension"
assert dimension_object["context"]["time_period"] == "stub_timeperiod"
def dimension_for_api(dimension):
dimension_object = DimensionObjectBuilder.build(dimension)
metadata = dimension_object["context"]
title = metadata.get("dimension")
if "table" in dimension_object:
data = dimension_object["table"]["data"]
elif "chart" in dimension_object:
data = dimension_object["chart"]["data"]
else:
data = []
return {"_dimension": title, "metadata": metadata, "data": data}
def process_dimensions(page, uri):
if page.dimensions:
download_dir = os.path.join(uri, "downloads")
os.makedirs(download_dir, exist_ok=True)
else:
return
dimensions = []
for d in page.dimensions:
if d.chart and d.chart["type"] != "panel_bar_chart":
chart_dir = "%s/charts" % uri
os.makedirs(chart_dir, exist_ok=True)
dimension_obj = DimensionObjectBuilder.build(d)
output = write_dimension_csv(dimension=dimension_obj)
if d.title:
filename = "%s.csv" % cleanup_filename(d.title)
table_filename = "%s-table.csv" % cleanup_filename(d.title)
else:
filename = "%s.csv" % d.guid
table_filename = "%s-table.csv" % d.guid
try:
file_path = os.path.join(download_dir, filename)
with open(file_path, "w") as dimension_file:
dimension_file.write(output)
except Exception as e:
print(f"Could not write file path {file_path}")
print(e)
d_as_dict = d.to_dict()
d_as_dict["static_file_name"] = filename
if d.table:
table_output = write_dimension_tabular_csv(dimension=dimension_obj)
table_file_path = os.path.join(download_dir, table_filename)
with open(table_file_path, "w") as dimension_file:
dimension_file.write(table_output)
d_as_dict["static_table_file_name"] = table_filename
dimensions.append(d_as_dict)
return dimensions
def test_if_dimension_has_chart_and_table_download_table_source_data(
app,
mock_rdu_user,
test_app_client,
stub_topic_page,
stub_subtopic_page,
stub_page_with_dimension_and_chart_and_table,
):
# GIVEN
# we have a dimension with table and chart data
dimension = stub_page_with_dimension_and_chart_and_table.dimensions[0]
with test_app_client:
test_app_client.post(url_for("security.login"),
data={
"email": mock_rdu_user.email,
"password": "******"
})
resp = test_app_client.get(
url_for(
"static_site.dimension_file_download",
topic_uri=stub_topic_page.uri,
subtopic_uri=stub_subtopic_page.uri,
measure_uri=stub_page_with_dimension_and_chart_and_table.uri,
version=stub_page_with_dimension_and_chart_and_table.version,
dimension_guid=dimension.guid,
))
# WHEN
# we generate a plain table csv
d = DimensionObjectBuilder.build(dimension)
expected_csv = write_dimension_csv(dimension=d)
# THEN
# we get a return
assert resp.status_code == 200
assert resp.content_type == "text/csv"
assert resp.headers[
"Content-Disposition"] == 'attachment; filename="stub-dimension.csv"'
# from the data in the table (not chart)
actual_data = resp.data.decode("utf-8")
assert actual_data == expected_csv
def test_dimension_object_builder_does_build_with_page_level_data_from_grouped_table(
stub_page_with_grouped_table):
# given - a table without a category_caption value
builder = DimensionObjectBuilder()
dimension = stub_page_with_grouped_table.dimensions[0]
# when we process the object
dimension_object = builder.build(dimension)
# then the measure level info should be brought through
assert dimension_object["context"]["measure"] == "Test Measure Page"
assert dimension_object["context"]["measure_guid"] == "test-measure-page"
assert dimension_object["context"]["measure_uri"] == "test-measure-page"
assert dimension_object["context"]["location"] == "UK"
assert dimension_object["context"]["source_text"] == "DWP Stats"
assert dimension_object["context"]["source_url"] == "http://dwp.gov.uk"
assert dimension_object["context"][
"department"] == "Department for Work and Pensions"
def test_table_object_builder_does_build_with_dimension_level_data_from_grouped_table():
measure_version = MeasureVersionWithDimensionFactory(
dimensions__title="Dimension title",
dimensions__guid="dimension-guid",
dimensions__time_period="dimension-time-period",
dimensions__dimension_table__table_object=grouped_table(),
)
# given - a table without a category_caption value
builder = DimensionObjectBuilder()
dimension = measure_version.dimensions[0]
# when we process the object
dimension_object = builder.build(dimension)
# then the dimension level info should be brought through
assert dimension_object["context"]["dimension"] == "Dimension title"
assert dimension_object["context"]["guid"] == "dimension-guid"
assert dimension_object["context"]["time_period"] == "dimension-time-period"
def dimension_file_download(topic_uri, subtopic_uri, measure_uri, version, dimension_guid):
try:
page = page_service.get_page_by_uri_and_type(measure_uri, "measure", version)
dimension_obj = DimensionObjectBuilder.build(page.get_dimension(dimension_guid))
data = write_dimension_csv(dimension=dimension_obj)
response = make_response(data)
if dimension_obj["context"]["dimension"] and dimension_obj["context"]["dimension"] != "":
filename = "%s.csv" % cleanup_filename(dimension_obj["context"]["dimension"])
else:
filename = "%s.csv" % cleanup_filename(dimension_obj["context"]["guid"])
response.headers["Content-Type"] = "text/csv"
response.headers["Content-Disposition"] = 'attachment; filename="%s"' % filename
return response
except DimensionNotFoundException as e:
abort(404)
def test_table_object_builder_does_build_with_dimension_level_data_from_simple_table():
measure_version = MeasureVersionWithDimensionFactory(
title="Test Measure Page",
area_covered=[UKCountry.ENGLAND],
dimensions__title="Dimension title",
dimensions__guid="dimension-guid",
dimensions__time_period="dimension-time-period",
dimensions__dimension_table__table_object=simple_table(),
measure__slug="test-measure-page-slug",
)
# given - a table without a category_caption value
builder = DimensionObjectBuilder()
dimension = measure_version.dimensions[0]
# when we process the object
dimension_object = builder.build(dimension)
# then the dimension level info should be brought through
assert dimension_object["context"]["dimension"] == "Dimension title"
assert dimension_object["context"]["guid"] == "dimension-guid"
assert dimension_object["context"]["time_period"] == "dimension-time-period"
