def test_questionnaire_store_json_loads(self):
# Given
expected = get_basic_input()
self.input_data = json_dumps(expected)
# When
store = QuestionnaireStore(self.storage)
# Then
self.assertEqual(store.metadata.copy(), expected["METADATA"])
self.assertEqual(store.response_metadata,
expected["RESPONSE_METADATA"])
self.assertEqual(store.answer_store, AnswerStore(expected["ANSWERS"]))
expected_completed_block_ids = expected["PROGRESS"][0]["block_ids"][0]
self.assertEqual(
len(
store.progress_store.get_completed_block_ids(
"a-test-section", "abc123")),
1,
)
self.assertEqual(
store.progress_store.get_completed_block_ids(
"a-test-section", "abc123")[0],
expected_completed_block_ids,
)
def put(self, model, overwrite=True):
storage_model = StorageModel(model_type=type(model))
serialized_item = storage_model.serialize(model)
serialized_item.pop(storage_model.key_field)
if len(serialized_item
) == 1 and storage_model.expiry_field in serialized_item:
# Don't store a value if the only key that is not the key_field is the expiry_field
value = ""
else:
value = json_dumps(serialized_item)
key_value = getattr(model, storage_model.key_field)
expires_in = None
if storage_model.expiry_field:
expiry_at = getattr(model, storage_model.expiry_field)
expires_in = expiry_at - datetime.now(tz=tzutc())
try:
record_created = self.client.set(name=key_value,
value=value,
ex=expires_in,
nx=not overwrite)
except RedisConnectionError:
self.log_retry("set")
record_created = self.client.set(name=key_value,
value=value,
ex=expires_in,
nx=not overwrite)
if not record_created:
raise ItemAlreadyExistsError()
def test_questionnaire_store_ignores_extra_json(self):
# Given
expected = get_basic_input()
expected[
"NOT_A_LEGAL_TOP_LEVEL_KEY"] = "woop_woop_thats_the_sound_of_the_police"
self.input_data = json_dumps(expected)
# When
store = QuestionnaireStore(self.storage)
# Then
self.assertEqual(store.metadata.copy(), expected["METADATA"])
self.assertEqual(store.response_metadata,
expected["RESPONSE_METADATA"])
self.assertEqual(store.answer_store, AnswerStore(expected["ANSWERS"]))
expected_completed_block_ids = expected["PROGRESS"][0]["block_ids"][0]
self.assertEqual(
len(
store.progress_store.get_completed_block_ids(
"a-test-section", "abc123")),
1,
)
self.assertEqual(
store.progress_store.get_completed_block_ids(
"a-test-section", "abc123")[0],
expected_completed_block_ids,
)
def test_primary_person_not_in_payload_when_not_answered(fake_questionnaire_store):
routing_path = [
RoutingPath(
["list-collector", "next-interstitial", "another-list-collector-block"],
section_id="section-1",
)
]
answer_objects = [
{"answer_id": "first-name", "value": "1", "list_item_id": "xJlKBy"},
{"answer_id": "last-name", "value": "1", "list_item_id": "xJlKBy"},
{"answer_id": "first-name", "value": "2", "list_item_id": "RfAGDc"},
{"answer_id": "last-name", "value": "2", "list_item_id": "RfAGDc"},
{"answer_id": "anyone-else", "value": "No"},
{"answer_id": "another-anyone-else", "value": "No"},
{"answer_id": "extraneous-answer", "value": "Bad", "list_item_id": "123"},
]
answers = AnswerStore(answer_objects)
list_store = ListStore(
existing_items=[{"name": "people", "items": ["xJlKBy", "RfAGDc"]}]
)
fake_questionnaire_store.answer_store = answers
fake_questionnaire_store.list_store = list_store
schema = load_schema_from_name("test_list_collector")
output = convert_answers(schema, fake_questionnaire_store, routing_path)
data_dict = json_loads(json_dumps(output["data"]["lists"]))
assert "primary_person" not in data_dict[0]
def test_list_item_conversion_empty_list(fake_questionnaire_store):
"""Test that the list store is populated with an empty list for lists which
do not have answers yet."""
routing_path = [
RoutingPath(
["list-collector", "next-interstitial", "another-list-collector-block"],
section_id="section-1",
)
]
answer_objects = [
{"answer_id": "last-name", "value": "2", "list_item_id": "RfAGDc"},
{"answer_id": "anyone-else", "value": "No"},
{"answer_id": "another-anyone-else", "value": "No"},
{"answer_id": "extraneous-answer", "value": "Bad", "list_item_id": "123"},
]
fake_questionnaire_store.answer_store = AnswerStore(answer_objects)
fake_questionnaire_store.list_store = ListStore()
schema = load_schema_from_name("test_list_collector")
output = convert_answers(
schema, fake_questionnaire_store, routing_path, SUBMITTED_AT
)
# Answers not on the routing path
del answer_objects[0]
del answer_objects[-1]
data_dict = json_loads(json_dumps(output["data"]["answers"]))
assert sorted(answer_objects, key=lambda x: x["answer_id"]) == sorted(
data_dict, key=lambda x: x["answer_id"]
)
def upload(self, metadata, payload):
payload.update(metadata)
blob = self.bucket.blob(str(uuid4()))
blob.metadata = metadata
blob.upload_from_string(json_dumps(payload).encode("utf8"),
content_type="application/json")
return True
def serialize(self) -> str:
data = {
"METADATA": self._metadata,
"ANSWERS": list(self.answer_store),
"LISTS": self.list_store.serialize(),
"PROGRESS": self.progress_store.serialize(),
"RESPONSE_METADATA": self.response_metadata,
}
return json_dumps(data)
def encrypt_data(self, data):
if isinstance(data, dict):
data = json_dumps(data)
protected_header = {"alg": "dir", "enc": "A256GCM", "kid": "1,1"}
jwe_token = jwe.JWE(plaintext=data,
protected=protected_header,
recipient=self.key)
return jwe_token.serialize(compact=True)
def test_questionnaire_store_missing_keys(questionnaire_store, basic_input):
# Given
del basic_input["PROGRESS"]
questionnaire_store.input_data = json_dumps(basic_input)
# When
store = QuestionnaireStore(questionnaire_store.storage)
# Then
assert store.metadata.copy() == basic_input["METADATA"]
assert store.response_metadata == basic_input["RESPONSE_METADATA"]
assert store.answer_store == AnswerStore(basic_input["ANSWERS"])
assert not store.progress_store.serialize()
def message(self) -> bytes:
message = {
"event": {
"type": "FULFILMENT_REQUESTED",
"source": "QUESTIONNAIRE_RUNNER",
"channel": "EQ",
"dateTime": datetime.now(tz=timezone.utc).isoformat(),
"transactionId": self.transaction_id,
},
"payload": self._payload(),
}
return json_dumps(message).encode("utf-8")
def test_questionnaire_store_missing_keys(self):
# Given
expected = get_basic_input()
del expected["PROGRESS"]
self.input_data = json_dumps(expected)
# When
store = QuestionnaireStore(self.storage)
# Then
self.assertEqual(store.metadata.copy(), expected["METADATA"])
self.assertEqual(store.response_metadata,
expected["RESPONSE_METADATA"])
self.assertEqual(store.answer_store, AnswerStore(expected["ANSWERS"]))
self.assertEqual(store.progress_store.serialize(), [])
def _submit_data(user):
answer_store = get_answer_store(user)
if answer_store:
questionnaire_store = get_questionnaire_store(user.user_id,
user.user_ik)
answer_store = questionnaire_store.answer_store
metadata = questionnaire_store.metadata
response_metadata = questionnaire_store.response_metadata
progress_store = questionnaire_store.progress_store
list_store = questionnaire_store.list_store
submitted_at = datetime.now(timezone.utc)
schema = load_schema_from_metadata(metadata)
router = Router(
schema,
answer_store,
list_store,
progress_store,
metadata,
response_metadata,
)
full_routing_path = router.full_routing_path()
message = json_dumps(
convert_answers(
schema,
questionnaire_store,
full_routing_path,
submitted_at,
flushed=True,
))
encrypted_message = encrypt(message, current_app.eq["key_store"],
KEY_PURPOSE_SUBMISSION)
sent = current_app.eq["submitter"].send_message(
encrypted_message,
tx_id=metadata.get("tx_id"),
case_id=metadata["case_id"],
)
if not sent:
raise SubmissionFailedException()
get_questionnaire_store(user.user_id, user.user_ik).delete()
logger.info("successfully flushed answers")
return True
logger.info("no answers found to flush")
return False
def dump_routing(schema, questionnaire_store):
router = Router(
schema,
questionnaire_store.answer_store,
questionnaire_store.list_store,
questionnaire_store.progress_store,
questionnaire_store.metadata,
)
response = [{
"section_id": routing_path.section_id,
"list_item_id": routing_path.list_item_id,
"routing_path": routing_path.block_ids,
} for routing_path in router.full_routing_path()]
return json_dumps(response), 200
def test_no_relationships_in_payload(fake_questionnaire_store):
routing_path = [
RoutingPath(
["list-collector", "relationships"],
section_id="section",
)
]
answer_objects = [
{"answer_id": "first-name", "value": "1", "list_item_id": "person1"},
{"answer_id": "last-name", "value": "1", "list_item_id": "person1"},
{"answer_id": "first-name", "value": "2", "list_item_id": "person2"},
{"answer_id": "last-name", "value": "2", "list_item_id": "person2"},
{"answer_id": "first-name", "value": "3", "list_item_id": "person3"},
{"answer_id": "last-name", "value": "3", "list_item_id": "person3"},
{"answer_id": "anyone-else", "value": "No"},
]
answers = AnswerStore(answer_objects)
list_store = ListStore(
existing_items=[
{
"name": "people",
"items": [
"person1",
"person2",
"person3",
],
}
]
)
fake_questionnaire_store.answer_store = answers
fake_questionnaire_store.list_store = list_store
schema = load_schema_from_name("test_relationships_unrelated")
output = convert_answers(
schema, fake_questionnaire_store, routing_path, SUBMITTED_AT
)
data = json_loads(json_dumps(output["data"]["answers"]))
answers = {answer["answer_id"]: answer for answer in data}
assert "relationship-answer" not in answers
def dump_submission(schema, questionnaire_store):
router = Router(
schema,
questionnaire_store.answer_store,
questionnaire_store.list_store,
questionnaire_store.progress_store,
questionnaire_store.metadata,
)
routing_path = router.full_routing_path()
questionnaire_store = get_questionnaire_store(current_user.user_id,
current_user.user_ik)
submission_handler = SubmissionHandler(schema, questionnaire_store,
routing_path)
response = {"submission": submission_handler.get_payload()}
return json_dumps(response), 200
def test_upload_feedback(client):
# Given
feedback = GCSFeedbackSubmitter(bucket_name="feedback")
metadata = {
"feedback_count": 1,
"feedback_submission_date": "2021-03-23",
"form_type": "H",
"language_code": "cy",
"region_code": "GB-ENG",
"tx_id": "12345",
}
payload = {
"feedback-type": "Feedback type",
"feedback-text": "Feedback text",
}
payload.update(metadata)
# When
feedback_upload = feedback.upload(metadata, json_dumps(payload))
# Then
bucket = client.return_value.get_bucket.return_value
blob = bucket.blob.return_value
assert blob.metadata["feedback_count"] == 1
assert blob.metadata["feedback_submission_date"] == "2021-03-23"
assert blob.metadata["form_type"] == "H"
assert blob.metadata["language_code"] == "cy"
assert blob.metadata["tx_id"] == "12345"
assert blob.metadata["region_code"] == "GB-ENG"
blob_contents = blob.upload_from_string.call_args[0][0]
assert (
blob_contents ==
b'{"feedback-type": "Feedback type", "feedback-text": "Feedback text", '
b'"feedback_count": 1, "feedback_submission_date": "2021-03-23", '
b'"form_type": "H", "language_code": "cy", "region_code": "GB-ENG", "tx_id": "12345"}'
)
assert feedback_upload is True
def test_list_structure_in_payload_is_as_expected(fake_questionnaire_store):
routing_path = [
RoutingPath(
["primary-person-list-collector", "list-collector"], section_id="section-1"
)
]
answer_objects = [
{"answer_id": "you-live-here", "value": "Yes"},
{"answer_id": "first-name", "value": "1", "list_item_id": "xJlKBy"},
{"answer_id": "last-name", "value": "1", "list_item_id": "xJlKBy"},
{"answer_id": "first-name", "value": "2", "list_item_id": "RfAGDc"},
{"answer_id": "last-name", "value": "2", "list_item_id": "RfAGDc"},
{"answer_id": "anyone-else", "value": "No"},
]
answers = AnswerStore(answer_objects)
list_store = ListStore(
existing_items=[
{
"name": "people",
"items": ["xJlKBy", "RfAGDc"],
"primary_person": "xJlKBy",
}
]
)
fake_questionnaire_store.answer_store = answers
fake_questionnaire_store.list_store = list_store
schema = load_schema_from_name("test_list_collector_primary_person")
output = convert_answers(
schema, fake_questionnaire_store, routing_path, SUBMITTED_AT
)
data_dict = json_loads(json_dumps(output["data"]["lists"]))
assert data_dict[0]["name"] == "people"
assert "xJlKBy" in data_dict[0]["items"]
assert data_dict[0]["primary_person"] == "xJlKBy"
def submit_questionnaire(self):
payload = self.get_payload()
message = json_dumps(payload)
encrypted_message = encrypt(message, current_app.eq["key_store"],
KEY_PURPOSE_SUBMISSION)
submitted = current_app.eq["submitter"].send_message(
encrypted_message,
case_id=self._metadata["case_id"],
tx_id=self._metadata.get("tx_id"),
)
if not submitted:
raise SubmissionFailedException()
cookie_session["submitted"] = True
self._store_submitted_time_and_display_address_in_session()
self._questionnaire_store.delete()
def _save_session(app_session, session_id, user_id, data, legacy=False):
raw_data = json_dumps(vars(data))
protected_header = {"alg": "dir", "enc": "A256GCM", "kid": "1,1"}
if legacy:
plaintext = base64url_encode(raw_data)
else:
plaintext = raw_data
jwe_token = jwe.JWE(plaintext=plaintext,
protected=protected_header,
recipient=app_session.key)
session_model = EQSession(
eq_session_id=session_id,
user_id=user_id,
session_data=jwe_token.serialize(compact=True),
expires_at=app_session.expires_at,
)
current_app.eq["storage"].put(session_model)
def test_questionnaire_store_json_loads(questionnaire_store, basic_input,
extra_basic_input):
basic_input.update(extra_basic_input)
# Given
questionnaire_store.input_data = json_dumps(basic_input)
# When
store = QuestionnaireStore(questionnaire_store.storage)
# Then
assert store.metadata.copy() == basic_input["METADATA"]
assert store.response_metadata == basic_input["RESPONSE_METADATA"]
assert store.answer_store == AnswerStore(basic_input["ANSWERS"])
assert not hasattr(store, "NOT_A_LEGAL_TOP_LEVEL_KEY")
assert not hasattr(store, "not_a_legal_top_level_key")
expected_completed_block_ids = basic_input["PROGRESS"][0]["block_ids"][0]
assert (len(
store.progress_store.get_completed_block_ids("a-test-section",
"abc123")) == 1)
assert (store.progress_store.get_completed_block_ids(
"a-test-section", "abc123")[0] == expected_completed_block_ids)
def test_default_answers_not_present_when_not_answered(fake_questionnaire_store):
"""Test that default values aren't submitted downstream when an answer with
a default value is not present in the answer store."""
schema = load_schema_from_name("test_default")
answer_objects = [{"answer_id": "number-question-two", "value": "12"}]
fake_questionnaire_store.answer_store = AnswerStore(answer_objects)
fake_questionnaire_store.list_store = ListStore()
routing_path = [
RoutingPath(
["number-question-one", "number-question-two"], section_id="default-section"
)
]
output = convert_answers(schema, fake_questionnaire_store, routing_path)
data = json_loads(json_dumps(output["data"]["answers"]))
answer_ids = {answer["answer_id"] for answer in data}
assert "answer-one" not in answer_ids
def test_list_item_conversion(fake_questionnaire_store):
routing_path = [
RoutingPath(
["list-collector", "next-interstitial", "another-list-collector-block"],
section_id="section-1",
)
]
answer_objects = [
{"answer_id": "first-name", "value": "1", "list_item_id": "xJlKBy"},
{"answer_id": "last-name", "value": "1", "list_item_id": "xJlKBy"},
{"answer_id": "first-name", "value": "2", "list_item_id": "RfAGDc"},
{"answer_id": "last-name", "value": "2", "list_item_id": "RfAGDc"},
{"answer_id": "anyone-else", "value": "No"},
{"answer_id": "another-anyone-else", "value": "No"},
{"answer_id": "extraneous-answer", "value": "Bad", "list_item_id": "123"},
]
answers = AnswerStore(answer_objects)
list_store = ListStore(
existing_items=[{"name": "people", "items": ["xJlKBy", "RfAGDc"]}]
)
fake_questionnaire_store.answer_store = answers
fake_questionnaire_store.list_store = list_store
schema = load_schema_from_name("test_list_collector")
output = convert_answers(
schema, fake_questionnaire_store, routing_path, SUBMITTED_AT
)
del answer_objects[-1]
data_dict = json_loads(json_dumps(output["data"]["answers"]))
assert sorted(answer_objects, key=lambda x: x["answer_id"]) == sorted(
data_dict, key=lambda x: x["answer_id"]
)
def test_primary_person_list_item_conversion(fake_questionnaire_store):
routing_path = [
RoutingPath(
["primary-person-list-collector", "list-collector"], section_id="section-1"
)
]
answer_objects = [
{"answer_id": "you-live-here", "value": "Yes"},
{"answer_id": "first-name", "value": "1", "list_item_id": "xJlKBy"},
{"answer_id": "last-name", "value": "1", "list_item_id": "xJlKBy"},
{"answer_id": "first-name", "value": "2", "list_item_id": "RfAGDc"},
{"answer_id": "last-name", "value": "2", "list_item_id": "RfAGDc"},
{"answer_id": "anyone-else", "value": "No"},
]
answers = AnswerStore(answer_objects)
list_store = ListStore(
existing_items=[
{
"name": "people",
"items": ["xJlKBy", "RfAGDc"],
"primary_person": "xJlKBy",
}
]
)
fake_questionnaire_store.answer_store = answers
fake_questionnaire_store.list_store = list_store
schema = load_schema_from_name("test_list_collector_primary_person")
output = convert_answers(schema, fake_questionnaire_store, routing_path)
data_dict = json_loads(json_dumps(output["data"]["answers"]))
assert sorted(answer_objects, key=lambda x: x["answer_id"]) == sorted(
data_dict, key=lambda x: x["answer_id"]
)
def test_serialize_and_deserialize(basic_answer_store):
json_serialized = json_dumps(basic_answer_store.serialize())
deserialized = AnswerStore(json_loads(json_serialized))
assert deserialized == basic_answer_store
def test_relationship_answers_not_on_path_in_payload(fake_questionnaire_store):
routing_path = [
RoutingPath(
["list-collector", "relationships"],
section_id="section",
)
]
answer_objects = [
{"answer_id": "first-name", "value": "1", "list_item_id": "person1"},
{"answer_id": "last-name", "value": "1", "list_item_id": "person1"},
{"answer_id": "first-name", "value": "2", "list_item_id": "person2"},
{"answer_id": "last-name", "value": "2", "list_item_id": "person2"},
{"answer_id": "first-name", "value": "3", "list_item_id": "person3"},
{"answer_id": "last-name", "value": "3", "list_item_id": "person3"},
{"answer_id": "first-name", "value": "4", "list_item_id": "person4"},
{"answer_id": "last-name", "value": "4", "list_item_id": "person4"},
{"answer_id": "first-name", "value": "5", "list_item_id": "person5"},
{"answer_id": "last-name", "value": "5", "list_item_id": "person5"},
{"answer_id": "anyone-else", "value": "No"},
{
"answer_id": "relationship-answer",
"value": [
{
"list_item_id": "person1",
"to_list_item_id": "person2",
"relationship": "Unrelated",
},
{
"list_item_id": "person1",
"to_list_item_id": "person3",
"relationship": "Unrelated",
},
{
"list_item_id": "person1",
"to_list_item_id": "person4",
"relationship": "Unrelated",
},
{
"list_item_id": "person1",
"to_list_item_id": "person5",
"relationship": "Unrelated",
},
],
},
{
"answer_id": "related-to-anyone-else-answer",
"value": "No",
"list_item_id": "person1",
},
]
answers = AnswerStore(answer_objects)
list_store = ListStore(
existing_items=[
{
"name": "people",
"items": [
"person1",
"person2",
"person3",
"person4",
"person5",
],
}
]
)
fake_questionnaire_store.answer_store = answers
fake_questionnaire_store.list_store = list_store
schema = load_schema_from_name("test_relationships_unrelated")
output = convert_answers(
schema, fake_questionnaire_store, routing_path, SUBMITTED_AT
)
data = json_loads(json_dumps(output["data"]["answers"]))
answers = {
(answer["answer_id"], answer.get("list_item_id")): answer for answer in data
}
expected_relationships_answer = [
{
"list_item_id": "person1",
"relationship": "Unrelated",
"to_list_item_id": "person2",
},
{
"list_item_id": "person1",
"relationship": "Unrelated",
"to_list_item_id": "person3",
},
{
"list_item_id": "person1",
"relationship": "Unrelated",
"to_list_item_id": "person4",
},
{
"list_item_id": "person1",
"relationship": "Unrelated",
"to_list_item_id": "person5",
},
]
assert ("related-to-anyone-else-answer", "person1") in answers
relationships_answer = answers[("relationship-answer", None)]
assert expected_relationships_answer == relationships_answer["value"]
def safe_health_check(): # pylint: disable=unused-variable
data = {
"status": "OK",
"version": application.config["EQ_APPLICATION_VERSION"]
}
return json_dumps(data)
def test_relationships_in_payload(fake_questionnaire_store):
routing_path = [
RoutingPath(
["list-collector", "relationships"],
section_id="section",
)
]
answer_objects = [
{"answer_id": "first-name", "value": "1", "list_item_id": "person1"},
{"answer_id": "last-name", "value": "1", "list_item_id": "person1"},
{"answer_id": "first-name", "value": "2", "list_item_id": "person2"},
{"answer_id": "last-name", "value": "2", "list_item_id": "person2"},
{"answer_id": "first-name", "value": "3", "list_item_id": "person3"},
{"answer_id": "last-name", "value": "3", "list_item_id": "person3"},
{"answer_id": "anyone-else", "value": "No"},
{
"answer_id": "relationship-answer",
"value": [
{
"list_item_id": "person1",
"to_list_item_id": "person2",
"relationship": "Husband or Wife",
},
{
"list_item_id": "person1",
"to_list_item_id": "person3",
"relationship": "Son or daughter",
},
],
},
]
answers = AnswerStore(answer_objects)
list_store = ListStore(
existing_items=[
{
"name": "people",
"items": [
"person1",
"person2",
"person3",
],
}
]
)
fake_questionnaire_store.answer_store = answers
fake_questionnaire_store.list_store = list_store
schema = load_schema_from_name("test_relationships")
output = convert_answers(schema, fake_questionnaire_store, routing_path)
data = json_loads(json_dumps(output["data"]["answers"]))
answers = {answer["answer_id"]: answer for answer in data}
expected_relationships_answer = [
{
"list_item_id": "person1",
"relationship": "Husband or Wife",
"to_list_item_id": "person2",
},
{
"list_item_id": "person1",
"relationship": "Son or daughter",
"to_list_item_id": "person3",
},
]
relationships_answer = answers["relationship-answer"]
assert expected_relationships_answer == relationships_answer["value"]
