def handle_post(self) -> None:
session_data: SessionData = self._session_store.session_data # type: ignore
session_data.feedback_count += 1
feedback_metadata = FeedbackMetadata(
session_data.case_id, session_data.tx_id) # type: ignore
# pylint: disable=no-member
# wtforms Form parents are not discoverable in the 2.3.3 implementation
feedback_message = FeedbackPayload(
metadata=self._questionnaire_store.metadata,
response_metadata=self._questionnaire_store.response_metadata,
schema=self._schema,
case_id=session_data.case_id,
submission_language_code=session_data.language_code,
feedback_count=session_data.feedback_count,
feedback_text=self.form.data.get("feedback-text"),
feedback_type=self.form.data.get("feedback-type"),
)
message = feedback_message()
metadata = feedback_metadata()
message.update(metadata)
encrypted_message = encrypt(
message,
current_app.eq["key_store"],
KEY_PURPOSE_SUBMISSION # type: ignore
)
if not current_app.eq["feedback_submitter"].upload( # type: ignore
metadata, encrypted_message):
raise FeedbackUploadFailed()
self._session_store.save()
def submit():
if request.method == 'POST':
data = request.get_data().decode('UTF8')
logger.info("Encrypting data")
unencrypted_json = json.loads(data)
no_of_submissions = int(unencrypted_json['quantity'])
with open("./keys.yml") as file:
secrets_from_file = yaml.safe_load(file)
key_store = KeyStore(secrets_from_file)
tx_id = unencrypted_json['survey']['tx_id']
for _ in range(0, no_of_submissions):
# If submitting more than one then randomise the tx_id
if tx_id is None:
tx_id = str(uuid.uuid4())
unencrypted_json['survey']['tx_id'] = tx_id
logger.info("Auto setting tx_id", tx_id=tx_id)
payload = encrypt(unencrypted_json['survey'], key_store,
'submission')
send_payload(payload, tx_id,
1) # let the loop handle the submission
return data
else:
return render_template('submit.html')
def test_encrypt_transfer_decrypt(self):
files = [
f for f in listdir(TEST_FILES_PATH)
if isfile(join(TEST_FILES_PATH, f))
]
for file in files:
with open(TEST_FILES_PATH + file, "rb") as fb:
contents = fb.read()
encoded_contents = base64.b64encode(contents)
payload = '{"file":"' + encoded_contents.decode() + '"}'
payload_as_json = json.loads(payload)
jwt = encrypt(payload_as_json, self.ras_key_store,
KEY_PURPOSE_CONSUMER)
decrypted_payload = decrypt(jwt, self.sdx_key_store,
KEY_PURPOSE_CONSUMER)
file_contents = decrypted_payload.get("file")
decoded_contents = base64.b64decode(file_contents)
self.assertEqual(contents, decoded_contents)
with open(TEST_FILES_RECOVERED_PATH + file, "wb") as fb:
fb.write(decoded_contents)
self.assertTrue(
filecmp.cmp(TEST_FILES_PATH + file,
TEST_FILES_RECOVERED_PATH + file))
def send_feedback():
if 'action[sign_out]' in request.form:
return redirect(url_for('session.get_sign_out'))
metadata = get_metadata(current_user)
if not metadata:
raise NoTokenException(401)
form = FeedbackForm()
if form.validate():
message = convert_feedback(
escape(request.form.get('message')),
escape(request.form.get('name')),
escape(request.form.get('email')),
request.referrer or '',
metadata,
g.schema.json['survey_id'],
)
encrypted_message = encrypt(message, current_app.eq['key_store'], key_purpose=KEY_PURPOSE_SUBMISSION)
sent = current_app.eq['submitter'].send_message(encrypted_message,
current_app.config['EQ_RABBITMQ_QUEUE_NAME'],
metadata['tx_id'])
if not sent:
raise SubmissionFailedException()
if request.form.get('redirect', 'true') == 'true':
return redirect(url_for('feedback.thank_you'))
return '', 200
def _submit_data(user):
answer_store = get_answer_store(user)
if answer_store.answers:
metadata = get_metadata(user)
collection_metadata = get_collection_metadata(user)
schema = load_schema_from_metadata(metadata)
completed_blocks = get_completed_blocks(user)
routing_path = PathFinder(schema, answer_store, metadata,
completed_blocks).get_full_routing_path()
message = convert_answers(metadata,
collection_metadata,
schema,
answer_store,
routing_path,
flushed=True)
encrypted_message = encrypt(message, current_app.eq['key_store'],
KEY_PURPOSE_SUBMISSION)
sent = current_app.eq['submitter'].send_message(
encrypted_message, current_app.config['EQ_RABBITMQ_QUEUE_NAME'],
metadata['tx_id'])
if not sent:
raise SubmissionFailedException()
get_questionnaire_store(user.user_id, user.user_ik).delete()
return True
return False
def submit_answers(schema, questionnaire_store, full_routing_path):
answer_store = questionnaire_store.answer_store
list_store = questionnaire_store.list_store
metadata = questionnaire_store.metadata
message = json.dumps(
convert_answers(schema, questionnaire_store, full_routing_path), for_json=True
)
encrypted_message = encrypt(
message, current_app.eq["key_store"], KEY_PURPOSE_SUBMISSION
)
sent = current_app.eq["submitter"].send_message(
encrypted_message,
questionnaire_id=metadata.get("questionnaire_id"),
case_id=metadata.get("case_id"),
tx_id=metadata.get("tx_id"),
)
if not sent:
raise SubmissionFailedException()
submitted_time = datetime.utcnow()
_store_submitted_time_in_session(submitted_time)
if is_view_submitted_response_enabled(schema.json):
_store_viewable_submission(
answer_store.serialise(), list_store.serialise(), metadata, submitted_time
)
get_questionnaire_store(current_user.user_id, current_user.user_ik).delete()
return redirect(url_for("post_submission.get_thank_you"))
def submit():
if request.method == 'POST':
data = request.get_data().decode('UTF8')
logger.debug(" [x] Encrypting data")
unencrypted_json = json.loads(data)
no_of_submissions = int(unencrypted_json['quantity'])
with open("./keys.yml") as file:
secrets_from_file = yaml.safe_load(file)
key_store = KeyStore(secrets_from_file)
tx_id = unencrypted_json['survey']['tx_id']
for _ in range(0, no_of_submissions):
# If submitting more than one then randomise the tx_id
if tx_id is None:
tx_id = str(uuid.uuid4())
unencrypted_json['survey']['tx_id'] = tx_id
logger.info("Auto setting tx_id", tx_id=tx_id)
payload = encrypt(unencrypted_json['survey'], key_store, 'submission')
send_payload(payload, tx_id, 1) # let the loop handle the submission
return data
else:
return render_template('submit.html')
def test_on_message_fails_with_missing_file_contents(self):
payload = '{"filename":"test", "case_id": "601c4ee4-83ed-11e7-bb31-be2e44b06b34", "survey_id": "221"}'
payload_as_json = json.loads(payload)
encrypted_jwt = encrypt(payload_as_json, self.ras_key_store,
KEY_PURPOSE_CONSUMER)
with self.assertRaises(QuarantinableError):
self.consumer.process(encrypted_jwt, uuid.uuid4())
def encrypter(data,
secrets_file='app/tests/encrypter_secrets.json',
key_purpose='ci_uploads'):
with open(secrets_file) as fp:
secrets = json.loads(fp.read())
secret_store = SecretStore(secrets)
print(secret_store.__dict__)
return encrypt(data, secret_store=secret_store, key_purpose=key_purpose)
def encrypt(self, payload):
"""
Encrypts the payload using the keystore values
:param payload: the value to encrypt
:return: string of encrypted data
"""
encrypted_data = encrypt(payload,
key_store=self.key_store,
key_purpose=KEY_PURPOSE)
return encrypted_data
def get_seft():
messages = []
if request.method == 'POST':
seft_file = request.files['file']
if seft_file.filename == '':
messages.append('No selected file')
logger.info("No selected file")
_, file_extension = os.path.splitext(seft_file.filename)
if file_extension not in ['.xls', '.xlsx']:
messages.append('Incorrect file extension')
logger.info("Incorrect file extension")
if messages:
return render_template('SEFT.html', messages=messages)
tx_id = str(uuid.uuid4())
case_id = str(uuid.uuid4())
ru_ref = "12345678901A"
# The submission will be put into a folder of the same name in the FTP server
survey_ref = "survey_ref"
_, file_extension = os.path.splitext(seft_file.filename)
file_as_string = convert_file_object_to_string_base64(
seft_file.stream.read())
time_date_stamp = time.strftime("%Y%m%d%H%M%S")
file_name = "{ru_ref}_{exercise_ref}_" \
"{survey_ref}_{time_date_stamp}{file_format}".format(ru_ref=ru_ref,
exercise_ref="exercise_ref",
survey_ref=survey_ref,
time_date_stamp=time_date_stamp,
file_format=file_extension)
logger.info("Generated filename for file going to FTP",
file_name=file_name,
case_id=case_id,
tx_id=tx_id)
message_json = {
'filename': file_name,
'file': file_as_string,
'case_id': case_id,
'survey_id': survey_ref
}
with open("./seft-keys.yml") as file:
secrets_from_file = yaml.safe_load(file)
key_store = KeyStore(secrets_from_file)
payload = encrypt(message_json, key_store, 'inbound')
send_payload(payload, tx_id)
messages.append('File queued successfully')
return render_template('SEFT.html', messages=messages)
def generate_token(json_secret_keys, payload):
"""
Generates the token by encrypting the payload with sdc-cryptography.
"""
print("Generating token")
keys = json.loads(json_secret_keys)
validate_required_keys(keys, KEY_PURPOSE)
key_store = KeyStore(keys)
encrypted_data = encrypt(payload,
key_store=key_store,
key_purpose=KEY_PURPOSE)
return encrypted_data
def encrypt(self, payload, service):
"""
Encrypts the payload using the keystore values
:param payload: the value to encrypt
:param service: The target service, needed to know which key in the keystore to use.
:return: string of encrypted data
"""
logger.info("About to encrypt payload", encryption_for_service=service)
encrypted_data = encrypt(payload,
key_store=self.key_store,
key_purpose=KEY_PURPOSE,
encryption_for_service=service)
return encrypted_data
def test_on_message_fails_with_missing_survey_id(self):
with open(join(TEST_FILES_PATH, "test1.xls"), "rb") as fb:
contents = fb.read()
encoded_contents = base64.b64encode(contents)
payload = '{"filename":"test1.xls", "file":"' + encoded_contents.decode() + \
'", "case_id": "601c4ee4-83ed-11e7-bb31-be2e44b06b34"}'
payload_as_json = json.loads(payload)
encrypted_jwt = encrypt(payload_as_json, self.ras_key_store,
KEY_PURPOSE_CONSUMER)
with self.assertRaises(QuarantinableError):
self.consumer.process(encrypted_jwt, uuid.uuid4())
def _submit_data(user):
answer_store = get_answer_store(user)
if answer_store:
questionnaire_store = get_questionnaire_store(user.user_id,
user.user_ik)
answer_store = questionnaire_store.answer_store
metadata = questionnaire_store.metadata
response_metadata = questionnaire_store.response_metadata
progress_store = questionnaire_store.progress_store
list_store = questionnaire_store.list_store
submitted_at = datetime.now(timezone.utc)
schema = load_schema_from_metadata(metadata)
router = Router(
schema,
answer_store,
list_store,
progress_store,
metadata,
response_metadata,
)
full_routing_path = router.full_routing_path()
message = json_dumps(
convert_answers(
schema,
questionnaire_store,
full_routing_path,
submitted_at,
flushed=True,
))
encrypted_message = encrypt(message, current_app.eq["key_store"],
KEY_PURPOSE_SUBMISSION)
sent = current_app.eq["submitter"].send_message(
encrypted_message,
tx_id=metadata.get("tx_id"),
case_id=metadata["case_id"],
)
if not sent:
raise SubmissionFailedException()
get_questionnaire_store(user.user_id, user.user_ik).delete()
logger.info("successfully flushed answers")
return True
logger.info("no answers found to flush")
return False
def test_end_to_end(self):
'''
End to end test - spins up a consumer and FTP server. Encrypts a message including a encoded spread sheet and takes the
decrypted and reassemble file are the same files.
This test requires a rabbit mq server to be running locally with the default settings
It also requires a valid OPSWAT API
NOTE this test should only be run manually due to it duration
'''
consumer_thread = ConsumerThread(self.sdx_keys)
consumer_thread.start()
ftp_thread = FTPThread()
ftp_thread.start()
files = [
f for f in listdir(TEST_FILES_PATH)
if isfile(join(TEST_FILES_PATH, f)) and (
f.endswith(".xls") or f.endswith(".xlsx"))
]
for file in files:
with open(join(TEST_FILES_PATH, file), "rb") as fb:
contents = fb.read()
encoded_contents = base64.b64encode(contents)
payload = '{"filename":"' + file + '", "file":"' + encoded_contents.decode() + \
'", "case_id": "601c4ee4-83ed-11e7-bb31-be2e44b06b34","survey_id": "221"}'
payload_as_json = json.loads(payload)
jwt = encrypt(payload_as_json, self.ras_key_store,
KEY_PURPOSE_CONSUMER)
with open("./encrypted_files/" + file, "w") as encrypted_file:
encrypted_file.write(jwt)
queue_publisher = QueuePublisher(settings.RABBIT_URLS,
settings.RABBIT_QUEUE)
headers = {'tx_id': str(uuid.uuid4())}
queue_publisher.publish_message(jwt, headers=headers)
# wait 30 seconds for opswat to process
time.sleep(30)
time.sleep(1)
consumer_thread.stop()
ftp_thread.stop()
for file in files:
if 'infected' not in file:
self.assertTrue(
filecmp.cmp(join(TEST_FILES_PATH, file),
join(EndToEndTest.TARGET_PATH, file)))
def get_seft():
messages = []
if request.method == 'POST':
seft_file = request.files['file']
if seft_file.filename == '':
messages.append('No selected file')
logger.info("No selected file")
_, file_extension = os.path.splitext(seft_file.filename)
if file_extension not in ['.xls', '.xlsx']:
messages.append('Incorrect file extension')
logger.info("Incorrect file extension")
if messages:
return render_template('SEFT.html', messages=messages)
tx_id = str(uuid.uuid4())
case_id = str(uuid.uuid4())
ru_ref = "12345678901A"
# The submission will be put into a folder of the same name in the FTP server
survey_ref = "survey_ref"
_, file_extension = os.path.splitext(seft_file.filename)
file_as_string = convert_file_object_to_string_base64(seft_file.stream.read())
time_date_stamp = time.strftime("%Y%m%d%H%M%S")
file_name = "{ru_ref}_{exercise_ref}_" \
"{survey_ref}_{time_date_stamp}{file_format}".format(ru_ref=ru_ref,
exercise_ref="exercise_ref",
survey_ref=survey_ref,
time_date_stamp=time_date_stamp,
file_format=file_extension)
logger.info("Generated filename for file going to FTP", file_name=file_name, case_id=case_id, tx_id=tx_id)
message_json = {
'filename': file_name,
'file': file_as_string,
'case_id': case_id,
'survey_id': survey_ref
}
with open("./seft-keys.yml") as file:
secrets_from_file = yaml.safe_load(file)
key_store = KeyStore(secrets_from_file)
payload = encrypt(message_json, key_store, 'inbound')
send_payload(payload, tx_id)
messages.append('File queued successfully')
return render_template('SEFT.html', messages=messages)
def test_decrypt_exception(self):
with open(join(TEST_FILES_PATH, "test1.xls"), "rb") as fb:
contents = fb.read()
encoded_contents = base64.b64encode(contents)
payload = '{"filename":"test1", "file":"' + encoded_contents.decode() + \
'", "case_id": "601c4ee4-83ed-11e7-bb31-be2e44b06b34", "survey_id": "221"}'
payload_as_json = json.loads(payload)
encrypted_jwt = encrypt(payload_as_json, self.ras_key_store,
KEY_PURPOSE_CONSUMER)
with unittest.mock.patch('app.main.decrypt', side_effect=Exception):
with self.assertRaises(QuarantinableError):
self.consumer.process(encrypted_jwt, uuid.uuid4())
def test_file_sent_to_av_and_ftp(self, mock_deliver_binary,
mock_send_for_av_scan):
"""Validate the message gets sent to the av and ftp"""
tx_id = uuid.uuid4()
with open(join(TEST_FILES_PATH, "test1.xls"), "rb") as fb:
contents = fb.read()
encoded_contents = base64.b64encode(contents)
payload = '{"filename":"test1.xls", "file":"' + encoded_contents.decode() + \
'", "case_id": "601c4ee4-83ed-11e7-bb31-be2e44b06b34", "survey_id": "SomeSurveyId"}'
payload_as_json = json.loads(payload)
encrypted_jwt = encrypt(payload_as_json, self.ras_key_store,
KEY_PURPOSE_CONSUMER)
self.consumer.process(encrypted_jwt, tx_id)
self.assertTrue(mock_deliver_binary.called)
self.assertTrue(mock_send_for_av_scan.called)
def test_send_ftp_IO_error(self, mock_send_for_av_scan):
with open(join(TEST_FILES_PATH, "test1.xls"), "rb") as fb:
contents = fb.read()
encoded_contents = base64.b64encode(contents)
payload = '{"filename":"test1", "file":"' + encoded_contents.decode() + \
'", "case_id": "601c4ee4-83ed-11e7-bb31-be2e44b06b34","survey_id": "221"}'
payload_as_json = json.loads(payload)
encrypted_jwt = encrypt(payload_as_json, self.ras_key_store,
KEY_PURPOSE_CONSUMER)
with unittest.mock.patch.object(SDXFTP,
'deliver_binary') as mock_method:
mock_method.side_effect = IOError
with self.assertRaises(RetryableError):
self.consumer.process(encrypted_jwt, uuid.uuid4())
self.assertTrue(mock_send_for_av_scan.called)
def submit_answers(routing_path, eq_id, form_type, schema):
metadata = get_metadata(current_user)
collection_metadata = get_collection_metadata(current_user)
answer_store = get_answer_store(current_user)
message = json.dumps(
convert_answers(
metadata,
collection_metadata,
schema,
answer_store,
routing_path,
))
encrypted_message = encrypt(message, current_app.eq['key_store'],
KEY_PURPOSE_SUBMISSION)
sent = current_app.eq['submitter'].send_message(
encrypted_message,
current_app.config['EQ_RABBITMQ_QUEUE_NAME'],
metadata['tx_id'],
)
if current_app.config['EQ_PUBSUB_ENABLED']:
current_app.eq['pubsub_submitter'].send_message(
encrypted_message,
current_app.config['EQ_PUBSUB_TOPIC_ID'],
metadata['tx_id'],
)
if not sent:
raise SubmissionFailedException()
submitted_time = datetime.utcnow()
_store_submitted_time_in_session(submitted_time)
if is_view_submitted_response_enabled(schema.json):
_store_viewable_submission(list(answer_store), metadata,
submitted_time)
get_questionnaire_store(current_user.user_id,
current_user.user_ik).delete()
return redirect(
url_for('post_submission.get_thank_you',
eq_id=eq_id,
form_type=form_type))
def test_valid_message_writes_to_log_after_ftp(self, mock_deliver_binary,
mock_send_for_av_scan):
"""Validate that the log entry is written after a successful ftp write"""
with open(join(TEST_FILES_PATH, "test1.xls"), "rb") as fb:
contents = fb.read()
encoded_contents = base64.b64encode(contents)
payload = '{"filename":"test1.xls", "file":"' + encoded_contents.decode() + \
'", "case_id": "601c4ee4-83ed-11e7-bb31-be2e44b06b34", "survey_id": "221"}'
with self.assertLogs(level="DEBUG") as cm:
payload_as_json = json.loads(payload)
encrypted_jwt = encrypt(payload_as_json, self.ras_key_store,
KEY_PURPOSE_CONSUMER)
self.consumer.process(encrypted_jwt, uuid.uuid4())
self.assertTrue(
ConsumerTests._contains_statement_in_log_file(
"Delivered to FTP server", cm.output))
self.assertTrue(mock_deliver_binary.called)
self.assertTrue(mock_send_for_av_scan.called)
def submit_questionnaire(self):
payload = self.get_payload()
message = json_dumps(payload)
encrypted_message = encrypt(message, current_app.eq["key_store"],
KEY_PURPOSE_SUBMISSION)
submitted = current_app.eq["submitter"].send_message(
encrypted_message,
case_id=self._metadata["case_id"],
tx_id=self._metadata.get("tx_id"),
)
if not submitted:
raise SubmissionFailedException()
cookie_session["submitted"] = True
self._store_submitted_time_and_display_address_in_session()
self._questionnaire_store.delete()
def test_valid_message_ftp_path_includes_survey_id(self,
mock_deliver_binary,
mock_send_for_av_scan):
"""Validates that the correct path and filename are used to deliver the ftp i.e that the survey_id is part
of the path
..note:: Pycharm will pass this test even if the url is manually changed to the wrong string. It appears to be
a bug in pycharm with multiple patches. The test fails under the same circumstances when run via make .
"""
self.consumer._ftp.deliver_binary = mock_deliver_binary
with open(join(TEST_FILES_PATH, "test1.xls"), "rb") as fb:
contents = fb.read()
encoded_contents = base64.b64encode(contents)
payload = '{"filename":"test1.xls", "file":"' + encoded_contents.decode() + \
'", "case_id": "601c4ee4-83ed-11e7-bb31-be2e44b06b34", "survey_id": "SomeSurveyId"}'
payload_as_json = json.loads(payload)
encrypted_jwt = encrypt(payload_as_json, self.ras_key_store,
KEY_PURPOSE_CONSUMER)
self.consumer.process(encrypted_jwt, uuid.uuid4())
mock_deliver_binary.assert_called_with("./SomeSurveyId", 'test1.xls',
unittest.mock.ANY)
self.assertTrue(mock_send_for_av_scan.called)
def test_fully_encrypted(self):
key_store = KeyStore({
"keys": {
SR_USER_AUTHENTICATION_PUBLIC_KEY_KID: {
"purpose": KEY_PURPOSE_AUTHENTICATION,
"type": "public",
"value": TEST_DO_NOT_USE_SR_PUBLIC_KEY,
},
EQ_USER_AUTHENTICATION_RRM_PRIVATE_KEY_KID: {
"purpose": KEY_PURPOSE_AUTHENTICATION,
"type": "private",
"value": TEST_DO_NOT_USE_UPSTREAM_PRIVATE_KEY,
},
}
})
payload = self.create_payload()
encrypted_token = encrypt(payload, key_store,
KEY_PURPOSE_AUTHENTICATION)
response = self.client.get("/session?token=" + encrypted_token)
self.assertEqual(302, response.status_code)
def test_fully_encrypted(self):
key_store = KeyStore({
'keys': {
SR_USER_AUTHENTICATION_PUBLIC_KEY_KID: {
'purpose': KEY_PURPOSE_AUTHENTICATION,
'type': 'public',
'value': TEST_DO_NOT_USE_SR_PUBLIC_KEY
},
EQ_USER_AUTHENTICATION_RRM_PRIVATE_KEY_KID: {
'purpose': KEY_PURPOSE_AUTHENTICATION,
'type': 'private',
'value': TEST_DO_NOT_USE_UPSTREAM_PRIVATE_KEY
},
}
})
payload = self.create_payload()
encrypted_token = encrypt(payload, key_store,
KEY_PURPOSE_AUTHENTICATION)
response = self.client.get('/session?token=' + encrypted_token)
self.assertEqual(302, response.status_code)
def transfer_files(self):
if not self.publisher.publishing:
logger.warning("Publisher is not ready.")
return
if self.transfer:
logger.warning("Cancelling overlapped task.")
return
else:
self.transfer = True
try:
logger.info("Looking for files...")
worker = FTPWorker(**self.ftp_params(self.services))
with worker as active:
if not active:
return
for job in active.get(active.filenames):
if job.filename not in self.recent:
logger.info("Found a file to publish",
filename=job.filename)
data = job._asdict()
data["file"] = base64.standard_b64encode(
job.file).decode("ascii")
data["ts"] = job.ts.isoformat()
payload = encrypt(data, self.key_store,
self.key_purpose)
tx_id = str(uuid.uuid4())
msg_id = self.publisher.publish_message(
payload, headers={'tx_id': tx_id})
if msg_id is None:
logger.warning("Failed to publish file",
filename=job.filename)
else:
self.recent[job.filename] = msg_id
logger.info("Published file",
filename=job.filename,
tx_id=tx_id)
logger.info(
"Finished publishing files, checking if any files need to be deleted"
)
for filename, msg_id in self.recent.items():
logger.info("Recently published file found",
filename=filename,
msg_id=msg_id)
# The file might not be in confirmed_deliveries as the publisher waits for the delivery
# confirmation adding it to the list
if msg_id in self.publisher.confirmed_deliveries:
logger.info(
"Deleting file as it has its delivery confirmed",
filename=filename,
msg_id=msg_id)
file_deleted = active.delete(filename)
if file_deleted:
del self.recent[filename]
logger.info("Succssfully deleted file",
filename=filename,
msg_id=msg_id)
else:
logger.info(
"Not deleting file as it hasn't had its delivery confirmed",
filename=filename,
msg_id=msg_id)
finally:
self.transfer = False
logger.info("Finished looking for files.")
def generate_token(self, payload):
return encrypt(payload, self._key_store, KEY_PURPOSE_AUTHENTICATION)
async def get_token(self, case_json):
eq_payload = await EqPayloadConstructor(case_json, self.request.app,
self.iac).build()
return encrypt(eq_payload,
key_store=self.request.app['key_store'],
key_purpose="authentication")
class View:
valid_display_regions = r'{display_region:\ben|cy|ni\b}'
valid_ew_display_regions = r'{display_region:\ben|cy\b}'
valid_user_journeys = r'{user_journey:\bstart|request\b}'
valid_sub_user_journeys = \
r'{sub_user_journey:\blink-address|change-address|access-code|paper-questionnaire|continuation-questionnaire\b}'
page_title_error_prefix_en = 'Error: '
page_title_error_prefix_cy = 'Gwall: '
@staticmethod
def get_now_utc():
return datetime.utcnow()
@staticmethod
def single_client_ip(request):
if request['client_ip']:
client_ip = request['client_ip']
single_ip_validation_pattern = re.compile(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$')
if client_ip.count(',') > 1:
single_ip_value = client_ip.split(', ', -1)[-3]
if single_ip_validation_pattern.fullmatch(single_ip_value):
single_ip = single_ip_value
else:
logger.warn('clientIP failed validation. Provided IP - ' + client_ip,
client_id=request['client_id'],
trace=request['trace'])
single_ip = ''
else:
logger.warn('clientIP failed validation. Provided IP - ' + client_ip,
client_id=request['client_id'],
trace=request['trace'])
single_ip = ''
elif request.headers.get('Origin', None) and 'localhost' in request.headers.get('Origin', None):
single_ip = '127.0.0.1'
else:
single_ip = ''
return single_ip
@staticmethod
def log_entry(request, endpoint):
method = request.method
logger.info(f"received {method} on endpoint '{endpoint}'",
client_ip=request['client_ip'],
client_id=request['client_id'],
trace=request['trace'],
method=request.method,
path=request.path)
@staticmethod
def gen_page_url(request):
full_url = str(request.rel_url)
if full_url[:3] == '/en' or full_url[:3] == '/cy' or full_url[:3] == '/ni':
generic_url = full_url[3:]
else:
generic_url = full_url
return generic_url
@staticmethod
def get_call_centre_number(display_region):
if display_region == 'ni':
call_centre_number = '0800 328 2021'
elif display_region == 'cy':
call_centre_number = '0800 169 2021'
else:
call_centre_number = '0800 141 2021'
return call_centre_number
@staticmethod
def check_if_after_census_day():
wall_clock = utc.localize(View.get_now_utc()).astimezone(uk_zone)
now_date = wall_clock.date()
if now_date > census_day:
after_census_day = True
else:
after_census_day = False
return after_census_day
@staticmethod
def get_campaign_site_link(request, display_region, requested_link):
base_en = request.app['DOMAIN_URL_PROTOCOL'] + request.app['DOMAIN_URL_EN']
base_cy = request.app['DOMAIN_URL_PROTOCOL'] + request.app['DOMAIN_URL_CY']
base_ni = request.app['DOMAIN_URL_PROTOCOL'] + request.app['DOMAIN_URL_EN'] + '/ni'
link = '/'
if requested_link == 'census-home':
if display_region == 'ni':
link = base_ni
elif display_region == 'cy':
link = base_cy
else:
link = base_en
elif requested_link == 'contact-us':
if display_region == 'ni':
link = base_ni + '/contact-us/'
elif display_region == 'cy':
link = base_cy + '/cysylltu-a-ni/'
else:
link = base_en + '/contact-us/'
elif requested_link == 'privacy':
if display_region == 'ni':
link = base_ni + '/privacy-and-data-protection/'
elif display_region == 'cy':
link = base_cy + '/preifatrwydd-a-diogelu-data/'
else:
link = base_en + '/privacy-and-data-protection/'
return link
@staticmethod
async def _make_request(request,
method,
url,
auth=None,
headers=None,
request_json=None,
return_json=False):
"""
:param request: The AIOHTTP user request, used for logging and app access
:param method: The HTTP verb
:param url: The target URL
:param auth: Authorization
:param headers: Any needed headers as a python dictionary
:param request_json: JSON payload to pass as request data
:param return_json: If True, the response JSON will be returned
"""
retry_request = RetryRequest(request, method, url, auth, headers, request_json, return_json)
return await retry_request.make_request()
@staticmethod
def validate_case(case_json):
if not case_json.get('active', False):
raise InactiveCaseError(case_json.get('caseType'))
if not case_json.get('caseStatus', None) == 'OK':
raise InvalidEqPayLoad('CaseStatus is not OK')
@staticmethod
async def call_questionnaire(request, case, attributes, app, adlocation):
eq_payload = await EqPayloadConstructor(case, attributes, app,
adlocation).build()
token = encrypt(eq_payload,
key_store=app['key_store'],
key_purpose='authentication')
try:
await RHService.post_surveylaunched(request, case, adlocation)
except ClientResponseError as ex:
if ex.status == 429:
raise TooManyRequestsEQLaunch()
else:
raise ex
logger.info('redirecting to eq',
client_ip=request['client_ip'], client_id=request['client_id'], trace=request['trace'])
eq_url = app['EQ_URL']
raise HTTPFound(f'{eq_url}/session?token={token}')
def flush_cases(case_id):
# Get iac for case
iac_return = requests.get(case_url + case_id + "/iac",
auth=config["CASE_AUTH"])
iac_return.raise_for_status()
iac_return = iac_return.json()
# Get case details
case_return = requests.get(case_url + case_id, auth=config["CASE_AUTH"])
case_return.raise_for_status()
case = case_return.json()
# Collection instrument details
ci_return = requests.get(ci_url + case["collectionInstrumentId"],
auth=config["COLLECTION_INSTRUMENT_AUTH"])
ci_return.raise_for_status()
ci = ci_return.json()
# Get collection exercise info
collex_return = requests.get(collex_url +
case["caseGroup"]["collectionExerciseId"],
auth=config["COLLECTION_EXERCISE_AUTH"])
collex_return.raise_for_status()
collex = collex_return.json()
# Get sample details
sample_return = requests.get(sample_url + case["sampleUnitId"] +
"/attributes",
auth=config["SAMPLE_AUTH"])
sample_return.raise_for_status()
sample = sample_return.json()
sample_attributes = sample["attributes"]
# For each IAC that is returned for a case
for iac in iac_return:
# Put together the payload
flush_payload = {
"jti": str(uuid4()), # required by eQ for creating a new claim
"tx_id": str(uuid4(
)), # not required by eQ (will generate if does not exist)
"user_id": case["sampleUnitId"], # required by eQ
"iat": int(time.time()),
"exp": int(time.time() +
(5 * 60)), # required by eQ for creating a new claim
"eq_id": ci["classifiers"]
["eq_id"], # required but currently only one social survey ('lms')
"period_id": collex["exerciseRef"], # required by eQ
"form_type": ci["classifiers"]
["form_type"], # required by eQ ('2' for lms_2 schema)
"collection_exercise_sid": collex["id"], # required by eQ
"ru_ref": case["caseGroup"]["sampleUnitRef"], # required by eQ
"case_id":
case["id"], # not required by eQ but useful for downstream
"case_ref":
case["caseRef"], # not required by eQ but useful for downstream
"account_service_url":
f'{config["ACCOUNT_SERVICE_URL"]}{config["URL_PATH_PREFIX"]}',
# required for save/continue
"country_code": sample_attributes["COUNTRY"],
"language_code": "en", # currently only 'en' or 'cy'
"response_id": build_response_id(case["id"], collex["id"],
iac["iac"]),
"roles": "flusher"
}
# Get encryption key stuff
config["key_store"] = jwt.key_store(config["JSON_SECRET_KEYS"])
# Encrypt payload into token
token = encrypt(flush_payload,
key_store=config['key_store'],
key_purpose="authentication")
# Call flusher
flush_url = eq_url + "/flush?token=" + token
flush_response = requests.post(flush_url)
print(
f'Response from flushing {case_id}: {flush_response.status_code}')
