def test_merge_from_two_other_data_sources(self, db_session):
data_source_1 = DataSourceFactory.create()
data_source_2 = DataSourceFactory.create()
data_source_3 = DataSourceFactory.create()
measure_version_associated_with_data_source_2 = MeasureVersionFactory.create(
data_sources=[data_source_2])
second_measure_version_associated_with_data_source_2 = MeasureVersionFactory.create(
data_sources=[data_source_2])
measure_version_associated_with_data_source_3 = MeasureVersionFactory.create(
data_sources=[data_source_3])
data_source_1.merge(
data_source_ids=[data_source_2.id, data_source_3.id])
db_session.session.commit()
assert DataSource.query.get(data_source_2.id) is None
assert DataSource.query.get(data_source_3.id) is None
assert measure_version_associated_with_data_source_2.data_sources == [
data_source_1
]
assert second_measure_version_associated_with_data_source_2.data_sources == [
data_source_1
]
assert measure_version_associated_with_data_source_3.data_sources == [
data_source_1
]
def test_data_sources_ordered_lexicographically_by_title(self, test_app_client, logged_in_admin_user):
DataSourceFactory.create(title="Police statistics 2019")
DataSourceFactory.create(title="2011 Census of England and Wales")
response = test_app_client.get(self.__path)
page = BeautifulSoup(response.data.decode("utf-8"), "html.parser")
main = page.find("main")
assert main.text.index("2011 Census of England and Wales") < main.text.index("Police statistics 2019")
def test_search_should_match_full_phrase_case_insensitve(self):
data_source = DataSourceFactory.create(
title="Annual Population Survey",
source_url=None,
publisher_id=None)
DataSourceFactory.create(title="2011 Census of England and Wales",
source_url=None,
publisher_id=None)
assert DataSource.search("annual population survey") == [data_source]
def test_data_source_associated_with_published_measure_versions(self):
data_source_1 = DataSourceFactory.create()
MeasureVersionFactory.create(data_sources=[data_source_1],
status="APPROVED")
data_source_2 = DataSourceFactory.create()
MeasureVersionFactory.create(data_sources=[data_source_2],
status="DRAFT")
assert data_source_1.associated_with_published_measure_versions is True
assert data_source_2.associated_with_published_measure_versions is False
def test_form_has_checkbox_for_each_data_source(self, test_app_client, logged_in_admin_user):
ds1 = DataSourceFactory.create(title="Police statistics 2019")
ds2 = DataSourceFactory.create(title="2011 Census of England and Wales")
response = test_app_client.get(self.__path)
page = BeautifulSoup(response.data.decode("utf-8"), "html.parser")
form = page.find("form", action=url_for("admin.merge_data_sources"))
assert form
assert find_input_for_label_with_text(form, "Police statistics 2019").get("value") == str(ds1.id)
assert find_input_for_label_with_text(form, "2011 Census of England and Wales").get("value") == str(ds2.id)
def test_search_should_match_publisher_name(self):
home_office = OrganisationFactory.create(name="Home Office")
foreign_office = OrganisationFactory.create(name="Foreign Office")
data_source = DataSourceFactory.create(
title="Annual Population Survey",
source_url=None,
publisher=home_office)
DataSourceFactory.create(title="Workforce Survey",
source_url=None,
publisher=foreign_office)
assert DataSource.search("home office") == [data_source]
def test_admin_user_can_see_all_data_sources(self, test_app_client, logged_in_admin_user):
DataSourceFactory.create(title="Police statistics 2019")
DataSourceFactory.create(title="2011 Census of England and Wales")
response = test_app_client.get(self.__path)
page = BeautifulSoup(response.data.decode("utf-8"), "html.parser")
assert response.status_code == 200
assert "Data sources" == page.find("h1").text
assert "Data sources" == page.find("title").text
assert "Police statistics 2019" in page.find("main").text
assert "2011 Census of England and Wales" in page.find("main").text
def test_failing_to_select_one_to_keep(self, test_app_client, logged_in_admin_user):
DataSourceFactory.create()
data_source_1 = DataSourceFactory.create(title="Police Statistics 2019")
data_source_2 = DataSourceFactory.create(title="Police Stats 2019")
response = test_app_client.post(
self.__path_with_data_source_ids(data_source_ids=[data_source_1.id, data_source_2.id]),
data=ImmutableMultiDict((("ids", data_source_1.id), ("ids", data_source_2.id))),
follow_redirects=False,
)
assert response.status_code == 200
def test_search_should_only_return_results_matching_all_keywords(self):
data_source = DataSourceFactory.create(
title="Annual Population Survey",
source_url=None,
publisher_id=None)
DataSourceFactory.create(title="Annual Workforce Survey",
source_url=None,
publisher_id=None)
DataSourceFactory.create(title="Estimated Population Statistics",
source_url=None,
publisher_id=None)
assert DataSource.search("survey population") == [data_source]
def test_search_should_match_publisher_abbreviation(self):
dwp = OrganisationFactory.create(
name="Department for Work and Pensions", abbreviations=["DWP"])
dfe = OrganisationFactory.create(name="Department for Education",
abbreviations=["FO"])
data_source = DataSourceFactory.create(title="Workforce Survey",
source_url=None,
publisher=dwp)
DataSourceFactory.create(title="Workforce Survey",
source_url=None,
publisher=dfe)
assert DataSource.search("dwp") == [data_source]
def test_post_with_an_updated_title_redirects_back_edit_data_source(
self, test_app_client, logged_in_rdu_user):
data_source = DataSourceFactory.create(title="Police stats 2019")
measure_version = MeasureVersionFactory.create(
data_sources=[data_source])
url = self.__update_data_source_url(data_source, measure_version)
response = test_app_client.post(
url,
data={
"title": "Police statistics 2019",
"type_of_data": "ADMINISTRATIVE",
"type_of_statistic_id": data_source.type_of_statistic_id,
"publisher_id": data_source.publisher_id,
"source_url": data_source.source_url,
"frequency_of_release_id": data_source.frequency_of_release_id,
"purpose": data_source.purpose,
},
)
# assert response.data.decode("utf-8") == ""
assert response.status_code == 302
redirected_to_location = response.headers["Location"]
match = re.search(f"{url}$", redirected_to_location)
assert match, f"Expected {redirected_to_location} to match {url}"
# Re-fetch the model from the database to be sure that it has been saved.
data_source = DataSource.query.get(data_source.id)
assert data_source.title == "Police statistics 2019", "Expected title to have been updated"
def test_admin_user_search_for_data_sources(self, test_app_client, logged_in_admin_user):
DataSourceFactory.create(title="Police statistics 2019")
DataSourceFactory.create(title="2011 Census of England and Wales")
response = test_app_client.get(self.__path_with_query("police"))
page = BeautifulSoup(response.data.decode("utf-8"), "html.parser")
assert response.status_code == 200
assert "Data sources" == page.find("h1").text
assert "police - Search data sources" == page.find("title").text
assert "Police statistics 2019" in page.find("main").text
assert "2011 Census of England and Wales" not in page.find("main").text
input_field = find_input_for_label_with_text(page, "Search data sources")
assert input_field["value"] == "police"
def test_search_param_specified(self, test_app_client, logged_in_rdu_user):
DataSourceFactory.create(title="Annual population survey", publication_date="25 October 2018")
measure_version = MeasureVersionFactory.create()
response = test_app_client.get(self.__search_url(measure_version) + "?q=population")
page = BeautifulSoup(response.data.decode("utf-8"), "html.parser")
assert response.status_code == 200
assert "Search for an existing data source" == page.find("title").text
assert "Search for an existing data source" == page.find("h1").text.strip()
input_field = find_input_for_label_with_text(page, "Search for an existing data\u00A0source")
assert input_field["value"] == "population"
assert "Annual population survey" in page.text
assert "25 October 2018" in page.text
def test_returns_200(self, test_app_client, logged_in_rdu_user):
data_source = DataSourceFactory.create()
measure_version = MeasureVersionFactory.create(
data_sources=[data_source])
response, _ = self.__get_page(test_app_client, data_source,
measure_version)
assert response.status_code == 200
def test_edit_page_if_data_source_not_associated_with_measure_version(
self, test_app_client, logged_in_rdu_user):
data_source = DataSourceFactory.create(title="Police statistics 2019")
measure_version = MeasureVersionFactory.create(data_sources=[])
response, _ = self.__get_page(test_app_client, data_source,
measure_version)
assert response.status_code == 404
def test_page_title_is_set(self, test_app_client, logged_in_rdu_user):
data_source = DataSourceFactory.create()
measure_version = MeasureVersionFactory.create(
data_sources=[data_source])
response, page = self.__get_page(test_app_client, data_source,
measure_version)
assert "Edit data source" == page.find("h1").text
assert "Edit data source" == page.find("title").text
def test_edit_page_if_dept_user_cannot_access_measure(
self, test_app_client, logged_in_dept_user):
data_source = DataSourceFactory.create()
measure_version = MeasureVersionFactory.create(
data_sources=[data_source], measure__shared_with=[])
response, _ = self.__get_page(test_app_client, data_source,
measure_version)
assert response.status_code == 403
def test_adding_an_existing_data_source(driver, app, live_server):
rdu_user = UserFactory(user_type=TypeOfUser.RDU_USER, active=True)
topic = TopicFactory.create(title="Police and crime")
subtopic = SubtopicFactory.create(title="Policing", topic=topic)
DataSourceFactory.create(title="Police statistics 2019")
existing_measure = MeasureFactory.create(subtopics=[subtopic])
MeasureVersionFactory.create(status="APPROVED", measure=existing_measure)
driver_login(driver, live_server, rdu_user)
home_page = HomePage(driver, live_server)
home_page.click_topic_link(topic)
topic_page = TopicPage(driver, live_server, topic)
topic_page.expand_accordion_for_subtopic(subtopic)
topic_page.click_add_measure(subtopic)
create_measure_page = MeasureEditPage(driver)
create_measure_page.set_title("Arrests")
create_measure_page.click_save()
create_measure_page.click_add_primary_data_source()
fill_in(driver,
label_text="Search for an existing data source",
with_text="Police statistics")
click_button_with_text(driver, "Search")
label_for_existing_data_source = driver.find_element_by_xpath(
"//label[text()='Police statistics 2019']")
label_for_existing_data_source.click()
click_button_with_text(driver, "Select")
assert "Successfully added the data source ‘Police statistics 2019’" in driver.page_source
click_link_with_text(driver, "Preview this version")
assert "Police statistics 2019" in driver.page_source
def test_post_when_dept_user_cannot_access_measure(self, test_app_client,
logged_in_dept_user):
data_source = DataSourceFactory.create(title="Police stats 2019")
measure_version = MeasureVersionFactory.create(
data_sources=[data_source], measure__shared_with=[])
url = self.__update_data_source_url(data_source, measure_version)
response = test_app_client.post(
url, data={"title": "Police statistics 2019"})
assert response.status_code == 403
def test_admin_user_attempting_to_merge_non_existent_ids(self, test_app_client, logged_in_admin_user):
organisation = OrganisationFactory.create(name="Home Office")
data_source_1 = DataSourceFactory.create(
title="2019 police statistics",
source_url="https://www.gov.uk/statistics/police/2019",
publisher=organisation,
)
response = test_app_client.get(self.__path_with_data_source_ids([data_source_1.id, 9999]))
assert response.status_code == 400
def test_post_with_no_title_redisplays_form_with_error(
self, test_app_client, logged_in_rdu_user):
data_source = DataSourceFactory.create(title="Police stats 2019")
measure_version = MeasureVersionFactory.create(
data_sources=[data_source])
url = self.__update_data_source_url(data_source, measure_version)
response = test_app_client.post(url, data={"title": ""})
page = BeautifulSoup(response.data.decode("utf-8"), "html.parser")
assert response.status_code == 200
assert "Error: Edit data source" == page.find("title").text
def test_edit_form_fields_populate_from_existing_data(
self, test_app_client, logged_in_rdu_user):
data_source = DataSourceFactory.create(title="Police statistics 2019")
measure_version = MeasureVersionFactory.create(
data_sources=[data_source])
response, page = self.__get_page(test_app_client, data_source,
measure_version)
title_input = find_input_for_label_with_text(page,
"Title of data source")
assert "Police statistics 2019" == title_input["value"]
def test_search_should_match_exact_source_url(self):
data_source = DataSourceFactory.create(
title="Annual Population Survey",
source_url=
"https://assets.publishing.service.gov.uk/government/uploads/" +
"system/uploads/attachment_data/file/4364643/2018-annual-population"
+ "-survey.pdf",
publisher_id=None,
)
assert DataSource.search(
"https://assets.publishing.service.gov.uk/government/uploads/system/uploads/"
+ "attachment_data/file/4364643/2018-annual-population-survey.pdf"
) == [data_source]
def test_merging_two_data_sources(self, test_app_client, logged_in_admin_user):
DataSourceFactory.create()
data_source_1 = DataSourceFactory.create(title="Police Statistics 2019")
data_source_2 = DataSourceFactory.create(title="Police Stats 2019")
response = test_app_client.post(
self.__path_with_data_source_ids(data_source_ids=[data_source_1.id, data_source_2.id]),
data=ImmutableMultiDict((("ids", data_source_1.id), ("ids", data_source_2.id), ("keep", data_source_1.id))),
follow_redirects=False,
)
assert response.status_code == 302
assert response.location == url_for("admin.data_sources", _external=True)
# Now follow the redirect
response_2 = test_app_client.get(response.location)
assert response_2.status_code == 200
page = BeautifulSoup(response_2.data.decode("utf-8"), "html.parser")
assert "Police Statistics 2019" in page.find("main").text
assert "Police Stats 2019" not in page.find("main").text
def test_admin_user_viewing_two_data_source_to_merge(self, test_app_client, logged_in_admin_user):
organisation = OrganisationFactory.create(name="Home Office")
data_source_1 = DataSourceFactory.create(
title="2019 police statistics",
source_url="https://www.gov.uk/statistics/police/2019",
publisher=organisation,
)
data_source_2 = DataSourceFactory.create(
title="Police statistics 2019", source_url="https://statistics.gov.uk/police/2019", publisher=organisation
)
response = test_app_client.get(self.__path_with_data_source_ids([data_source_1.id, data_source_2.id]))
assert response.status_code == 200
page = BeautifulSoup(response.data.decode("utf-8"), "html.parser")
assert "Merge 2 data sources" == page.find("h1").text
assert "Merge 2 data sources" == page.find("title").text
assert "2019 police statistics" in page.text
assert "Police statistics 2019" in page.text
def test_search_should_be_ordered_by_number_of_associated_measure_versions(
self):
data_source_associated_with_1_measure_version = DataSourceFactory.create(
title="Survey")
data_source_associated_with_2_measure_versions = DataSourceFactory.create(
title="Survey")
MeasureVersionFactory.create(
data_sources=[data_source_associated_with_1_measure_version])
for _ in range(2):
MeasureVersionFactory.create(
data_sources=[data_source_associated_with_2_measure_versions])
search = DataSource.search("survey")
assert len(search) == 2
assert (
data_source_associated_with_2_measure_versions == search[0]
), "Expected data source associated with 2 measure versions to be first"
assert (
data_source_associated_with_1_measure_version == search[1]
), "Expected data source associated with 1 measure version to be second"
def test_pending_build_is_added_to_database(self, test_app_client,
logged_in_rdu_user):
data_source = DataSourceFactory.create(title="Police stats 2019")
measure_version = MeasureVersionFactory.create(
status="DRAFT", data_sources=[data_source])
assert Build.query.count() == 0
test_app_client.post(
self.__update_data_source_url(data_source, measure_version),
data={
"title": "Police statistics 2019",
"type_of_data": "ADMINISTRATIVE",
"type_of_statistic_id": data_source.type_of_statistic_id,
"publisher_id": data_source.publisher_id,
"source_url": data_source.source_url,
"frequency_of_release_id": data_source.frequency_of_release_id,
"purpose": data_source.purpose,
},
)
# No build is created if there isn't a published (approved) measure version associated with the data source
assert Build.query.count() == 0
# Toggle the measure version to published (approved)
measure_version.status = "APPROVED"
test_app_client.post(
self.__update_data_source_url(data_source, measure_version),
data={
"title": "Police statistics 2019",
"type_of_data": "ADMINISTRATIVE",
"type_of_statistic_id": data_source.type_of_statistic_id,
"publisher_id": data_source.publisher_id,
"source_url": data_source.source_url,
"frequency_of_release_id": data_source.frequency_of_release_id,
"purpose": data_source.purpose,
},
)
# There's an approved measure version, so we should log a build request.
assert Build.query.count() == 1
def test_post_when_dept_user_has_access_to_measure(self, test_app_client,
logged_in_dept_user):
data_source = DataSourceFactory.create(title="Police stats 2019")
measure_version = MeasureVersionFactory.create(
data_sources=[data_source],
measure__shared_with=[logged_in_dept_user])
url = self.__update_data_source_url(data_source, measure_version)
response = test_app_client.post(
url,
data={
"title": "Police statistics 2019",
"type_of_data": "ADMINISTRATIVE",
"type_of_statistic_id": data_source.type_of_statistic_id,
"publisher_id": data_source.publisher_id,
"source_url": data_source.source_url,
"frequency_of_release_id": data_source.frequency_of_release_id,
"purpose": data_source.purpose,
},
)
assert response.status_code == 302
def test_merge_with_self(self):
data_source_1 = DataSourceFactory.create()
with pytest.raises(ValueError):
data_source_1.merge(data_source_ids=[data_source_1.id])
def test_search_be_limited_by_param(self):
for _ in range(5):
DataSourceFactory.create(title="Workforce Survey")
assert len(DataSource.search("survey", limit=2)) == 2