def process_schema(in_schema, out_dir, spec_file, require_path=".."):
try:
with open(in_schema, encoding="utf-8") as schema:
data = json_loads(schema.read())
except json.JSONDecodeError:
logger.exception("error reading %s", in_schema)
return
try:
os.stat(out_dir)
except IndexError:
os.mkdir(out_dir)
process_questionnaire_flow(data, require_path, out_dir, spec_file)
if data.get("post_submission", {}).get("view_response"):
process_view_submitted_response(data, require_path, out_dir, spec_file)
for section in data["sections"]:
if "summary" in section:
process_section_summary(section["id"], out_dir, section, spec_file,
require_path)
for group in section["groups"]:
for block in group["blocks"]:
process_block(block, out_dir, data, spec_file, require_path)
def _deserialize(self, data: str) -> None:
json_data = json_loads(data)
self.progress_store = ProgressStore(json_data.get("PROGRESS"))
self.set_metadata(json_data.get("METADATA", {}))
self.answer_store = AnswerStore(json_data.get("ANSWERS"))
self.list_store = ListStore.deserialize(json_data.get("LISTS"))
self.response_metadata = json_data.get("RESPONSE_METADATA", {})
def test_same_name_items(self):
self.launchSurvey("test_list_collector_same_name_items",
roles=["dumper"])
self.post({"you-live-here": "Yes"})
self.post({
"first-name": "James",
"middle-names": "Brian",
"last-name": "May"
})
self.post({"anyone-else": "Yes"})
self.post({
"first-name": "James",
"middle-names": "Roger",
"last-name": "May"
})
self.get("/dump/debug")
actual = json_loads(self.getResponseData())
item_id_a = actual["LISTS"][0]["items"][0]
item_id_b = actual["LISTS"][0]["items"][1]
assert item_id_a in actual["LISTS"][0]["same_name_items"]
assert item_id_b in actual["LISTS"][0]["same_name_items"]
def test_get_schema_json(self):
self.get("/schemas/test_textfield")
response = self.getResponseData()
parsed_json = json_loads(response)
self.assertIn("title", parsed_json)
self.assertEqual(parsed_json["title"], "Other input fields")
def _load(self) -> None:
logger.debug("finding eq_session_id in database",
eq_session_id=self.eq_session_id)
self._eq_session: Optional[EQSession] = current_app.eq["storage"].get(
EQSession, self.eq_session_id) # type: ignore
if self._eq_session and self._eq_session.session_data:
self.user_id = self._eq_session.user_id
encrypted_session_data = self._eq_session.session_data
session_data_as_bytes = StorageEncryption(
self.user_id, self.user_ik,
self.pepper).decrypt_data(encrypted_session_data)
session_data_as_str = session_data_as_bytes.decode()
# for backwards compatibility
# session data used to be base64 encoded before encryption
try:
session_data_as_str = base64url_decode(
session_data_as_str).decode()
except ValueError:
pass
self.session_data = json_loads(
session_data_as_str, object_hook=lambda d: SessionData(**d))
logger.debug(
"found matching eq_session for eq_session_id in database",
session_id=self._eq_session.eq_session_id,
user_id=self._eq_session.user_id,
)
else:
logger.debug("eq_session_id not found in database",
eq_session_id=self.eq_session_id)
def test_list_item_conversion_empty_list(fake_questionnaire_store):
"""Test that the list store is populated with an empty list for lists which
do not have answers yet."""
routing_path = [
RoutingPath(
["list-collector", "next-interstitial", "another-list-collector-block"],
section_id="section-1",
)
]
answer_objects = [
{"answer_id": "last-name", "value": "2", "list_item_id": "RfAGDc"},
{"answer_id": "anyone-else", "value": "No"},
{"answer_id": "another-anyone-else", "value": "No"},
{"answer_id": "extraneous-answer", "value": "Bad", "list_item_id": "123"},
]
fake_questionnaire_store.answer_store = AnswerStore(answer_objects)
fake_questionnaire_store.list_store = ListStore()
schema = load_schema_from_name("test_list_collector")
output = convert_answers(
schema, fake_questionnaire_store, routing_path, SUBMITTED_AT
)
# Answers not on the routing path
del answer_objects[0]
del answer_objects[-1]
data_dict = json_loads(json_dumps(output["data"]["answers"]))
assert sorted(answer_objects, key=lambda x: x["answer_id"]) == sorted(
data_dict, key=lambda x: x["answer_id"]
)
def test_primary_person_not_in_payload_when_not_answered(fake_questionnaire_store):
routing_path = [
RoutingPath(
["list-collector", "next-interstitial", "another-list-collector-block"],
section_id="section-1",
)
]
answer_objects = [
{"answer_id": "first-name", "value": "1", "list_item_id": "xJlKBy"},
{"answer_id": "last-name", "value": "1", "list_item_id": "xJlKBy"},
{"answer_id": "first-name", "value": "2", "list_item_id": "RfAGDc"},
{"answer_id": "last-name", "value": "2", "list_item_id": "RfAGDc"},
{"answer_id": "anyone-else", "value": "No"},
{"answer_id": "another-anyone-else", "value": "No"},
{"answer_id": "extraneous-answer", "value": "Bad", "list_item_id": "123"},
]
answers = AnswerStore(answer_objects)
list_store = ListStore(
existing_items=[{"name": "people", "items": ["xJlKBy", "RfAGDc"]}]
)
fake_questionnaire_store.answer_store = answers
fake_questionnaire_store.list_store = list_store
schema = load_schema_from_name("test_list_collector")
output = convert_answers(schema, fake_questionnaire_store, routing_path)
data_dict = json_loads(json_dumps(output["data"]["lists"]))
assert "primary_person" not in data_dict[0]
def getCookie(self):
"""
Returns the last received response cookie session
"""
cookie = self.last_response.headers["Set-Cookie"]
cookie_session = cookie.split("session=.")[1].split(";")[0]
decoded_cookie_session = decode_flask_cookie(cookie_session)
return json_loads(decoded_cookie_session)
def test_list_schemas(self):
self.get("/schemas")
response = self.getResponseData()
parsed_json = json_loads(response)
self.assertIsInstance(parsed_json, dict)
self.assertIsInstance(parsed_json["test"][0], str)
self.assertIn("test_textfield", parsed_json["test"])
def test_encryption_decryption(self):
data = {"data1": "Test Data One", "data2": "Test Data Two"}
encrypted_data = self.encrypter.encrypt_data(data)
self.assertNotEqual(encrypted_data, data)
self.assertIsInstance(encrypted_data, str)
decrypted_data = self.encrypter.decrypt_data(encrypted_data)
decrypted_data = json_loads(decrypted_data)
self.assertEqual(data, decrypted_data)
def dumpSubmission(self):
self.get("/dump/submission")
# Then I get a 200 OK response
self.assertStatusOK()
# And the JSON response contains the data I submitted
dump_submission = json_loads(self.getResponseData())
return dump_submission
def dumpAnswers(self):
self.get("/dump/answers")
# Then I get a 200 OK response
self.assertStatusOK()
# And the JSON response contains the data I submitted
dump_answers = json_loads(self.getResponseData())
return dump_answers
def dumpSubmission(self, client):
cache = self.cache[client]
self.get(client, "/dump/submission")
self.assertEqual(cache["last_response"].status_code, 200)
# And the JSON response contains the data I submitted
dump_submission = json_loads(cache.get("last_response").get_data(True))
return dump_submission
def test_individual_case_id_not_present_when_case_type_ce():
metadata = {
"region_code": "GB-ENG",
"case_id": str(uuid4()),
"case_type": "CE"
}
fulfilment_request = IndividualResponseFulfilmentRequest(metadata)
json_message = json_loads(fulfilment_request.message)
assert "individualCaseId" not in json_message["payload"][
"fulfilmentRequest"]
def test_fulfilment_code_for_postal(region_code, expected_fulfilment_code):
metadata = {
"region_code": region_code,
"case_id": str(uuid4()),
"case_type": "SPG"
}
fulfilment_request = IndividualResponseFulfilmentRequest(metadata)
json_message = json_loads(fulfilment_request.message)
assert (json_message["payload"]["fulfilmentRequest"]["fulfilmentCode"] ==
expected_fulfilment_code)
def test_data_is_deleted_on_submission(self):
# Given we submit a survey
self.launchSurvey("test_percentage", roles=["dumper"])
self.post({"answer": "99"})
self.post()
# When we start the survey again
self.launchSurvey("test_percentage", roles=["dumper"])
# Then no answers should have persisted
self.get("/dump/debug")
answers = json_loads(self.getResponseData())
self.assertEqual(0, len(answers["ANSWERS"]))
def test_get_thank_you_data_not_deleted_when_questionnaire_is_not_complete(
self):
# Given we start a survey
self.launchSurvey("test_percentage", roles=["dumper"])
self.post({"answer": "99"})
# When we request the thank you page (without submitting the survey)
self.get("submitted/thank-you")
# Then the answers are not deleted
self.get("/dump/debug")
answers = json_loads(self.getResponseData())
self.assertEqual(1, len(answers["ANSWERS"]))
def test_questionnaire_store_updates_storage(questionnaire_store, basic_input):
# Given
store = QuestionnaireStore(questionnaire_store.storage)
store.set_metadata(basic_input["METADATA"])
store.answer_store = AnswerStore(basic_input["ANSWERS"])
store.response_metadata = basic_input["RESPONSE_METADATA"]
store.progress_store = ProgressStore(basic_input["PROGRESS"])
# When
store.save()
# Then
assert basic_input == json_loads(questionnaire_store.output_data)
def get(self, model_type, key_value):
storage_model = StorageModel(model_type=model_type)
try:
item = self.client.get(key_value)
except RedisConnectionError:
self.log_retry("get")
item = self.client.get(key_value)
if item:
item_dict = json_loads(item.decode("utf-8"))
item_dict[storage_model.key_field] = key_value
return storage_model.deserialize(item_dict)
def test_dump_submission_authenticated_with_role_with_answers(self):
# Given I am an authenticated user who has launched a survey
# and does have the 'dumper' role in my metadata
self.launchSurvey("test_radio_mandatory", roles=["dumper"])
# When I submit an answer
self.post(post_data={"radio-mandatory-answer": "Coffee"})
# And I attempt to dump the submission payload
self.get("/dump/submission")
# Then I get a 200 OK response
self.assertStatusOK()
# And the JSON response contains the data I submitted
actual = json_loads(self.getResponseData())
# tx_id and submitted_at are dynamic; so copy them over
expected = {
"submission": {
"version": "0.0.3",
"survey_id": "0",
"flushed": False,
"origin": "uk.gov.ons.edc.eq",
"type": "uk.gov.ons.edc.eq:surveyresponse",
"tx_id": actual["submission"]["tx_id"],
"started_at": actual["submission"]["started_at"],
"submitted_at": actual["submission"]["submitted_at"],
"case_id": actual["submission"]["case_id"],
"collection": {
"period": "201604",
"exercise_sid": "789",
"schema_name": "test_radio_mandatory",
},
"data": {
"answers": [{
"answer_id": "radio-mandatory-answer",
"value": "Coffee"
}],
"lists": [],
},
"metadata": {
"ru_ref": "123456789012A",
"user_id": "integration-test"
},
"launch_language_code": "en",
"submission_language_code": "en",
}
}
assert actual == expected
def test_questionnaire_store_updates_storage(self):
# Given
expected = get_basic_input()
store = QuestionnaireStore(self.storage)
store.set_metadata(expected["METADATA"])
store.answer_store = AnswerStore(expected["ANSWERS"])
store.response_metadata = expected["RESPONSE_METADATA"]
store.progress_store = ProgressStore(expected["PROGRESS"])
# When
store.save() # See setUp - populates self.output_data
# Then
self.assertEqual(expected, json_loads(self.output_data))
def test_given_csrf_attack_when_refresh_then_on_question(self):
# Given
self.launchSurvey("test_interstitial_page", roles=["dumper"])
self.post()
self.last_csrf_token = "made-up-token"
self.post({"favourite-breakfast": "Pancakes"})
# When
self.get(self.last_url)
# Then
self.assertEqual(self.last_response.status_code, 200)
self.get("/dump/debug")
answers = json_loads(self.getResponseData())
self.assertEqual(0, len(answers["ANSWERS"]))
def test_get_session_expiry_doesnt_extend_session(self):
self.launchSurvey()
# Advance time by 20 mins...
with freeze_time(TIME_TO_FREEZE + timedelta(minutes=20)):
self.get("/session-expiry")
response = self.getResponseData()
parsed_json = json_loads(response)
# ... check that the session expiry time is not affected by
# the request, and is still 45mins from the start time
expected_expires_at = (
TIME_TO_FREEZE +
timedelta(seconds=EQ_SESSION_TIMEOUT_SECONDS)).isoformat()
self.assertIn("expires_at", parsed_json)
self.assertEqual(parsed_json["expires_at"], expected_expires_at)
def test_postal_fulfilment_request_message():
metadata = {"region_code": "GB-ENG", "case_id": str(uuid4())}
fulfilment_request = IndividualResponseFulfilmentRequest(metadata)
postal_json_message = json_loads(fulfilment_request.message)
payload = postal_json_message["payload"]
validate_uuids_in_payload(payload)
expected_sms_payload = {
"fulfilmentRequest": {
"fulfilmentCode": "P_UAC_UACIPA1",
"contact": {},
}
}
assert payload == expected_sms_payload
def test_patch_session_expiry_extends_session(self):
self.launchSurvey()
# Advance time by 20 mins...
request_time = TIME_TO_FREEZE + timedelta(minutes=20)
with freeze_time(request_time):
self.patch(None, "/session-expiry")
response = self.getResponseData()
parsed_json = json_loads(response)
# ... check that the session expiry time is reset by the request
# and is now 45 mins from the request time
expected_expires_at = (
request_time +
timedelta(seconds=EQ_SESSION_TIMEOUT_SECONDS)).isoformat()
self.assertIn("expires_at", parsed_json)
self.assertEqual(parsed_json["expires_at"], expected_expires_at)
def test_given_csrf_attack_when_submit_new_answers_then_answers_saved(self):
# Given
self.launchSurvey("test_interstitial_page", roles=["dumper"])
self.post()
self.last_csrf_token = "made-up-token"
self.post({"favourite-breakfast": "Muesli"})
# When
self.get(self.last_url)
self.post({"favourite-breakfast": "Pancakes"})
# Then
self.assertStatusOK()
self.get("/dump/debug")
answers = json_loads(self.getResponseData())
self.assertEqual("Pancakes", answers["ANSWERS"][0]["value"])
def load_schema_from_url(survey_url, language_code):
language_code = language_code or DEFAULT_LANGUAGE_CODE
logger.info("loading schema from URL",
survey_url=survey_url,
language_code=language_code)
constructed_survey_url = f"{survey_url}?language={language_code}"
req = requests.get(constructed_survey_url)
schema_response = req.content.decode()
if req.status_code == 404:
logger.error("no schema exists", survey_url=constructed_survey_url)
raise NotFound
return QuestionnaireSchema(json_loads(schema_response), language_code)
def test_given_answered_question_when_change_answer_with_invalid_csrf_token_then_answers_not_saved(
self,
):
# Given
self.launchSurvey("test_interstitial_page", roles=["dumper"])
self.post()
self.post({"favourite-breakfast": "Muesli"})
# When
self.last_csrf_token = "made-up-token"
self.post({"favourite-breakfast": "Pancakes"})
# Then
self.assertStatusCode(401)
self.get("/dump/debug")
answers = json_loads(self.getResponseData())
self.assertEqual("Muesli", answers["ANSWERS"][0]["value"])
def test_no_relationships_in_payload(fake_questionnaire_store):
routing_path = [
RoutingPath(
["list-collector", "relationships"],
section_id="section",
)
]
answer_objects = [
{"answer_id": "first-name", "value": "1", "list_item_id": "person1"},
{"answer_id": "last-name", "value": "1", "list_item_id": "person1"},
{"answer_id": "first-name", "value": "2", "list_item_id": "person2"},
{"answer_id": "last-name", "value": "2", "list_item_id": "person2"},
{"answer_id": "first-name", "value": "3", "list_item_id": "person3"},
{"answer_id": "last-name", "value": "3", "list_item_id": "person3"},
{"answer_id": "anyone-else", "value": "No"},
]
answers = AnswerStore(answer_objects)
list_store = ListStore(
existing_items=[
{
"name": "people",
"items": [
"person1",
"person2",
"person3",
],
}
]
)
fake_questionnaire_store.answer_store = answers
fake_questionnaire_store.list_store = list_store
schema = load_schema_from_name("test_relationships_unrelated")
output = convert_answers(
schema, fake_questionnaire_store, routing_path, SUBMITTED_AT
)
data = json_loads(json_dumps(output["data"]["answers"]))
answers = {answer["answer_id"]: answer for answer in data}
assert "relationship-answer" not in answers
def test_redis_does_not_store_key_field_in_value(self):
# Given
eq_session = EQSession(
eq_session_id="sessionid",
user_id="someuser",
session_data="somedata",
expires_at=EXPIRES_AT,
)
stored_data = self.redis.get(EQSession, eq_session.eq_session_id)
self.assertIsNone(stored_data)
self.redis.put(eq_session)
# When
stored_data = self.mock_client.get(eq_session.eq_session_id)
storage_model = StorageModel(model_type=EQSession)
assert storage_model.key_field not in json_loads(stored_data)
