def process_schema(in_schema, out_dir, spec_file, require_path=".."): try: with open(in_schema, encoding="utf-8") as schema: data = json_loads(schema.read()) except json.JSONDecodeError: logger.exception("error reading %s", in_schema) return try: os.stat(out_dir) except IndexError: os.mkdir(out_dir) process_questionnaire_flow(data, require_path, out_dir, spec_file) if data.get("post_submission", {}).get("view_response"): process_view_submitted_response(data, require_path, out_dir, spec_file) for section in data["sections"]: if "summary" in section: process_section_summary(section["id"], out_dir, section, spec_file, require_path) for group in section["groups"]: for block in group["blocks"]: process_block(block, out_dir, data, spec_file, require_path)
def _deserialize(self, data: str) -> None: json_data = json_loads(data) self.progress_store = ProgressStore(json_data.get("PROGRESS")) self.set_metadata(json_data.get("METADATA", {})) self.answer_store = AnswerStore(json_data.get("ANSWERS")) self.list_store = ListStore.deserialize(json_data.get("LISTS")) self.response_metadata = json_data.get("RESPONSE_METADATA", {})
def test_same_name_items(self): self.launchSurvey("test_list_collector_same_name_items", roles=["dumper"]) self.post({"you-live-here": "Yes"}) self.post({ "first-name": "James", "middle-names": "Brian", "last-name": "May" }) self.post({"anyone-else": "Yes"}) self.post({ "first-name": "James", "middle-names": "Roger", "last-name": "May" }) self.get("/dump/debug") actual = json_loads(self.getResponseData()) item_id_a = actual["LISTS"][0]["items"][0] item_id_b = actual["LISTS"][0]["items"][1] assert item_id_a in actual["LISTS"][0]["same_name_items"] assert item_id_b in actual["LISTS"][0]["same_name_items"]
def test_get_schema_json(self): self.get("/schemas/test_textfield") response = self.getResponseData() parsed_json = json_loads(response) self.assertIn("title", parsed_json) self.assertEqual(parsed_json["title"], "Other input fields")
def _load(self) -> None: logger.debug("finding eq_session_id in database", eq_session_id=self.eq_session_id) self._eq_session: Optional[EQSession] = current_app.eq["storage"].get( EQSession, self.eq_session_id) # type: ignore if self._eq_session and self._eq_session.session_data: self.user_id = self._eq_session.user_id encrypted_session_data = self._eq_session.session_data session_data_as_bytes = StorageEncryption( self.user_id, self.user_ik, self.pepper).decrypt_data(encrypted_session_data) session_data_as_str = session_data_as_bytes.decode() # for backwards compatibility # session data used to be base64 encoded before encryption try: session_data_as_str = base64url_decode( session_data_as_str).decode() except ValueError: pass self.session_data = json_loads( session_data_as_str, object_hook=lambda d: SessionData(**d)) logger.debug( "found matching eq_session for eq_session_id in database", session_id=self._eq_session.eq_session_id, user_id=self._eq_session.user_id, ) else: logger.debug("eq_session_id not found in database", eq_session_id=self.eq_session_id)
def test_list_item_conversion_empty_list(fake_questionnaire_store): """Test that the list store is populated with an empty list for lists which do not have answers yet.""" routing_path = [ RoutingPath( ["list-collector", "next-interstitial", "another-list-collector-block"], section_id="section-1", ) ] answer_objects = [ {"answer_id": "last-name", "value": "2", "list_item_id": "RfAGDc"}, {"answer_id": "anyone-else", "value": "No"}, {"answer_id": "another-anyone-else", "value": "No"}, {"answer_id": "extraneous-answer", "value": "Bad", "list_item_id": "123"}, ] fake_questionnaire_store.answer_store = AnswerStore(answer_objects) fake_questionnaire_store.list_store = ListStore() schema = load_schema_from_name("test_list_collector") output = convert_answers( schema, fake_questionnaire_store, routing_path, SUBMITTED_AT ) # Answers not on the routing path del answer_objects[0] del answer_objects[-1] data_dict = json_loads(json_dumps(output["data"]["answers"])) assert sorted(answer_objects, key=lambda x: x["answer_id"]) == sorted( data_dict, key=lambda x: x["answer_id"] )
def test_primary_person_not_in_payload_when_not_answered(fake_questionnaire_store): routing_path = [ RoutingPath( ["list-collector", "next-interstitial", "another-list-collector-block"], section_id="section-1", ) ] answer_objects = [ {"answer_id": "first-name", "value": "1", "list_item_id": "xJlKBy"}, {"answer_id": "last-name", "value": "1", "list_item_id": "xJlKBy"}, {"answer_id": "first-name", "value": "2", "list_item_id": "RfAGDc"}, {"answer_id": "last-name", "value": "2", "list_item_id": "RfAGDc"}, {"answer_id": "anyone-else", "value": "No"}, {"answer_id": "another-anyone-else", "value": "No"}, {"answer_id": "extraneous-answer", "value": "Bad", "list_item_id": "123"}, ] answers = AnswerStore(answer_objects) list_store = ListStore( existing_items=[{"name": "people", "items": ["xJlKBy", "RfAGDc"]}] ) fake_questionnaire_store.answer_store = answers fake_questionnaire_store.list_store = list_store schema = load_schema_from_name("test_list_collector") output = convert_answers(schema, fake_questionnaire_store, routing_path) data_dict = json_loads(json_dumps(output["data"]["lists"])) assert "primary_person" not in data_dict[0]
def getCookie(self): """ Returns the last received response cookie session """ cookie = self.last_response.headers["Set-Cookie"] cookie_session = cookie.split("session=.")[1].split(";")[0] decoded_cookie_session = decode_flask_cookie(cookie_session) return json_loads(decoded_cookie_session)
def test_list_schemas(self): self.get("/schemas") response = self.getResponseData() parsed_json = json_loads(response) self.assertIsInstance(parsed_json, dict) self.assertIsInstance(parsed_json["test"][0], str) self.assertIn("test_textfield", parsed_json["test"])
def test_encryption_decryption(self): data = {"data1": "Test Data One", "data2": "Test Data Two"} encrypted_data = self.encrypter.encrypt_data(data) self.assertNotEqual(encrypted_data, data) self.assertIsInstance(encrypted_data, str) decrypted_data = self.encrypter.decrypt_data(encrypted_data) decrypted_data = json_loads(decrypted_data) self.assertEqual(data, decrypted_data)
def dumpSubmission(self): self.get("/dump/submission") # Then I get a 200 OK response self.assertStatusOK() # And the JSON response contains the data I submitted dump_submission = json_loads(self.getResponseData()) return dump_submission
def dumpAnswers(self): self.get("/dump/answers") # Then I get a 200 OK response self.assertStatusOK() # And the JSON response contains the data I submitted dump_answers = json_loads(self.getResponseData()) return dump_answers
def dumpSubmission(self, client): cache = self.cache[client] self.get(client, "/dump/submission") self.assertEqual(cache["last_response"].status_code, 200) # And the JSON response contains the data I submitted dump_submission = json_loads(cache.get("last_response").get_data(True)) return dump_submission
def test_individual_case_id_not_present_when_case_type_ce(): metadata = { "region_code": "GB-ENG", "case_id": str(uuid4()), "case_type": "CE" } fulfilment_request = IndividualResponseFulfilmentRequest(metadata) json_message = json_loads(fulfilment_request.message) assert "individualCaseId" not in json_message["payload"][ "fulfilmentRequest"]
def test_fulfilment_code_for_postal(region_code, expected_fulfilment_code): metadata = { "region_code": region_code, "case_id": str(uuid4()), "case_type": "SPG" } fulfilment_request = IndividualResponseFulfilmentRequest(metadata) json_message = json_loads(fulfilment_request.message) assert (json_message["payload"]["fulfilmentRequest"]["fulfilmentCode"] == expected_fulfilment_code)
def test_data_is_deleted_on_submission(self): # Given we submit a survey self.launchSurvey("test_percentage", roles=["dumper"]) self.post({"answer": "99"}) self.post() # When we start the survey again self.launchSurvey("test_percentage", roles=["dumper"]) # Then no answers should have persisted self.get("/dump/debug") answers = json_loads(self.getResponseData()) self.assertEqual(0, len(answers["ANSWERS"]))
def test_get_thank_you_data_not_deleted_when_questionnaire_is_not_complete( self): # Given we start a survey self.launchSurvey("test_percentage", roles=["dumper"]) self.post({"answer": "99"}) # When we request the thank you page (without submitting the survey) self.get("submitted/thank-you") # Then the answers are not deleted self.get("/dump/debug") answers = json_loads(self.getResponseData()) self.assertEqual(1, len(answers["ANSWERS"]))
def test_questionnaire_store_updates_storage(questionnaire_store, basic_input): # Given store = QuestionnaireStore(questionnaire_store.storage) store.set_metadata(basic_input["METADATA"]) store.answer_store = AnswerStore(basic_input["ANSWERS"]) store.response_metadata = basic_input["RESPONSE_METADATA"] store.progress_store = ProgressStore(basic_input["PROGRESS"]) # When store.save() # Then assert basic_input == json_loads(questionnaire_store.output_data)
def get(self, model_type, key_value): storage_model = StorageModel(model_type=model_type) try: item = self.client.get(key_value) except RedisConnectionError: self.log_retry("get") item = self.client.get(key_value) if item: item_dict = json_loads(item.decode("utf-8")) item_dict[storage_model.key_field] = key_value return storage_model.deserialize(item_dict)
def test_dump_submission_authenticated_with_role_with_answers(self): # Given I am an authenticated user who has launched a survey # and does have the 'dumper' role in my metadata self.launchSurvey("test_radio_mandatory", roles=["dumper"]) # When I submit an answer self.post(post_data={"radio-mandatory-answer": "Coffee"}) # And I attempt to dump the submission payload self.get("/dump/submission") # Then I get a 200 OK response self.assertStatusOK() # And the JSON response contains the data I submitted actual = json_loads(self.getResponseData()) # tx_id and submitted_at are dynamic; so copy them over expected = { "submission": { "version": "0.0.3", "survey_id": "0", "flushed": False, "origin": "uk.gov.ons.edc.eq", "type": "uk.gov.ons.edc.eq:surveyresponse", "tx_id": actual["submission"]["tx_id"], "started_at": actual["submission"]["started_at"], "submitted_at": actual["submission"]["submitted_at"], "case_id": actual["submission"]["case_id"], "collection": { "period": "201604", "exercise_sid": "789", "schema_name": "test_radio_mandatory", }, "data": { "answers": [{ "answer_id": "radio-mandatory-answer", "value": "Coffee" }], "lists": [], }, "metadata": { "ru_ref": "123456789012A", "user_id": "integration-test" }, "launch_language_code": "en", "submission_language_code": "en", } } assert actual == expected
def test_questionnaire_store_updates_storage(self): # Given expected = get_basic_input() store = QuestionnaireStore(self.storage) store.set_metadata(expected["METADATA"]) store.answer_store = AnswerStore(expected["ANSWERS"]) store.response_metadata = expected["RESPONSE_METADATA"] store.progress_store = ProgressStore(expected["PROGRESS"]) # When store.save() # See setUp - populates self.output_data # Then self.assertEqual(expected, json_loads(self.output_data))
def test_given_csrf_attack_when_refresh_then_on_question(self): # Given self.launchSurvey("test_interstitial_page", roles=["dumper"]) self.post() self.last_csrf_token = "made-up-token" self.post({"favourite-breakfast": "Pancakes"}) # When self.get(self.last_url) # Then self.assertEqual(self.last_response.status_code, 200) self.get("/dump/debug") answers = json_loads(self.getResponseData()) self.assertEqual(0, len(answers["ANSWERS"]))
def test_get_session_expiry_doesnt_extend_session(self): self.launchSurvey() # Advance time by 20 mins... with freeze_time(TIME_TO_FREEZE + timedelta(minutes=20)): self.get("/session-expiry") response = self.getResponseData() parsed_json = json_loads(response) # ... check that the session expiry time is not affected by # the request, and is still 45mins from the start time expected_expires_at = ( TIME_TO_FREEZE + timedelta(seconds=EQ_SESSION_TIMEOUT_SECONDS)).isoformat() self.assertIn("expires_at", parsed_json) self.assertEqual(parsed_json["expires_at"], expected_expires_at)
def test_postal_fulfilment_request_message(): metadata = {"region_code": "GB-ENG", "case_id": str(uuid4())} fulfilment_request = IndividualResponseFulfilmentRequest(metadata) postal_json_message = json_loads(fulfilment_request.message) payload = postal_json_message["payload"] validate_uuids_in_payload(payload) expected_sms_payload = { "fulfilmentRequest": { "fulfilmentCode": "P_UAC_UACIPA1", "contact": {}, } } assert payload == expected_sms_payload
def test_patch_session_expiry_extends_session(self): self.launchSurvey() # Advance time by 20 mins... request_time = TIME_TO_FREEZE + timedelta(minutes=20) with freeze_time(request_time): self.patch(None, "/session-expiry") response = self.getResponseData() parsed_json = json_loads(response) # ... check that the session expiry time is reset by the request # and is now 45 mins from the request time expected_expires_at = ( request_time + timedelta(seconds=EQ_SESSION_TIMEOUT_SECONDS)).isoformat() self.assertIn("expires_at", parsed_json) self.assertEqual(parsed_json["expires_at"], expected_expires_at)
def test_given_csrf_attack_when_submit_new_answers_then_answers_saved(self): # Given self.launchSurvey("test_interstitial_page", roles=["dumper"]) self.post() self.last_csrf_token = "made-up-token" self.post({"favourite-breakfast": "Muesli"}) # When self.get(self.last_url) self.post({"favourite-breakfast": "Pancakes"}) # Then self.assertStatusOK() self.get("/dump/debug") answers = json_loads(self.getResponseData()) self.assertEqual("Pancakes", answers["ANSWERS"][0]["value"])
def load_schema_from_url(survey_url, language_code): language_code = language_code or DEFAULT_LANGUAGE_CODE logger.info("loading schema from URL", survey_url=survey_url, language_code=language_code) constructed_survey_url = f"{survey_url}?language={language_code}" req = requests.get(constructed_survey_url) schema_response = req.content.decode() if req.status_code == 404: logger.error("no schema exists", survey_url=constructed_survey_url) raise NotFound return QuestionnaireSchema(json_loads(schema_response), language_code)
def test_given_answered_question_when_change_answer_with_invalid_csrf_token_then_answers_not_saved( self, ): # Given self.launchSurvey("test_interstitial_page", roles=["dumper"]) self.post() self.post({"favourite-breakfast": "Muesli"}) # When self.last_csrf_token = "made-up-token" self.post({"favourite-breakfast": "Pancakes"}) # Then self.assertStatusCode(401) self.get("/dump/debug") answers = json_loads(self.getResponseData()) self.assertEqual("Muesli", answers["ANSWERS"][0]["value"])
def test_no_relationships_in_payload(fake_questionnaire_store): routing_path = [ RoutingPath( ["list-collector", "relationships"], section_id="section", ) ] answer_objects = [ {"answer_id": "first-name", "value": "1", "list_item_id": "person1"}, {"answer_id": "last-name", "value": "1", "list_item_id": "person1"}, {"answer_id": "first-name", "value": "2", "list_item_id": "person2"}, {"answer_id": "last-name", "value": "2", "list_item_id": "person2"}, {"answer_id": "first-name", "value": "3", "list_item_id": "person3"}, {"answer_id": "last-name", "value": "3", "list_item_id": "person3"}, {"answer_id": "anyone-else", "value": "No"}, ] answers = AnswerStore(answer_objects) list_store = ListStore( existing_items=[ { "name": "people", "items": [ "person1", "person2", "person3", ], } ] ) fake_questionnaire_store.answer_store = answers fake_questionnaire_store.list_store = list_store schema = load_schema_from_name("test_relationships_unrelated") output = convert_answers( schema, fake_questionnaire_store, routing_path, SUBMITTED_AT ) data = json_loads(json_dumps(output["data"]["answers"])) answers = {answer["answer_id"]: answer for answer in data} assert "relationship-answer" not in answers
def test_redis_does_not_store_key_field_in_value(self): # Given eq_session = EQSession( eq_session_id="sessionid", user_id="someuser", session_data="somedata", expires_at=EXPIRES_AT, ) stored_data = self.redis.get(EQSession, eq_session.eq_session_id) self.assertIsNone(stored_data) self.redis.put(eq_session) # When stored_data = self.mock_client.get(eq_session.eq_session_id) storage_model = StorageModel(model_type=EQSession) assert storage_model.key_field not in json_loads(stored_data)