def handle_post(self) -> None: session_data: SessionData = self._session_store.session_data # type: ignore session_data.feedback_count += 1 feedback_metadata = FeedbackMetadata( session_data.case_id, session_data.tx_id) # type: ignore # pylint: disable=no-member # wtforms Form parents are not discoverable in the 2.3.3 implementation feedback_message = FeedbackPayload( metadata=self._questionnaire_store.metadata, response_metadata=self._questionnaire_store.response_metadata, schema=self._schema, case_id=session_data.case_id, submission_language_code=session_data.language_code, feedback_count=session_data.feedback_count, feedback_text=self.form.data.get("feedback-text"), feedback_type=self.form.data.get("feedback-type"), ) message = feedback_message() metadata = feedback_metadata() message.update(metadata) encrypted_message = encrypt( message, current_app.eq["key_store"], KEY_PURPOSE_SUBMISSION # type: ignore ) if not current_app.eq["feedback_submitter"].upload( # type: ignore metadata, encrypted_message): raise FeedbackUploadFailed() self._session_store.save()
def submit(): if request.method == 'POST': data = request.get_data().decode('UTF8') logger.info("Encrypting data") unencrypted_json = json.loads(data) no_of_submissions = int(unencrypted_json['quantity']) with open("./keys.yml") as file: secrets_from_file = yaml.safe_load(file) key_store = KeyStore(secrets_from_file) tx_id = unencrypted_json['survey']['tx_id'] for _ in range(0, no_of_submissions): # If submitting more than one then randomise the tx_id if tx_id is None: tx_id = str(uuid.uuid4()) unencrypted_json['survey']['tx_id'] = tx_id logger.info("Auto setting tx_id", tx_id=tx_id) payload = encrypt(unencrypted_json['survey'], key_store, 'submission') send_payload(payload, tx_id, 1) # let the loop handle the submission return data else: return render_template('submit.html')
def test_encrypt_transfer_decrypt(self): files = [ f for f in listdir(TEST_FILES_PATH) if isfile(join(TEST_FILES_PATH, f)) ] for file in files: with open(TEST_FILES_PATH + file, "rb") as fb: contents = fb.read() encoded_contents = base64.b64encode(contents) payload = '{"file":"' + encoded_contents.decode() + '"}' payload_as_json = json.loads(payload) jwt = encrypt(payload_as_json, self.ras_key_store, KEY_PURPOSE_CONSUMER) decrypted_payload = decrypt(jwt, self.sdx_key_store, KEY_PURPOSE_CONSUMER) file_contents = decrypted_payload.get("file") decoded_contents = base64.b64decode(file_contents) self.assertEqual(contents, decoded_contents) with open(TEST_FILES_RECOVERED_PATH + file, "wb") as fb: fb.write(decoded_contents) self.assertTrue( filecmp.cmp(TEST_FILES_PATH + file, TEST_FILES_RECOVERED_PATH + file))
def send_feedback(): if 'action[sign_out]' in request.form: return redirect(url_for('session.get_sign_out')) metadata = get_metadata(current_user) if not metadata: raise NoTokenException(401) form = FeedbackForm() if form.validate(): message = convert_feedback( escape(request.form.get('message')), escape(request.form.get('name')), escape(request.form.get('email')), request.referrer or '', metadata, g.schema.json['survey_id'], ) encrypted_message = encrypt(message, current_app.eq['key_store'], key_purpose=KEY_PURPOSE_SUBMISSION) sent = current_app.eq['submitter'].send_message(encrypted_message, current_app.config['EQ_RABBITMQ_QUEUE_NAME'], metadata['tx_id']) if not sent: raise SubmissionFailedException() if request.form.get('redirect', 'true') == 'true': return redirect(url_for('feedback.thank_you')) return '', 200
def _submit_data(user): answer_store = get_answer_store(user) if answer_store.answers: metadata = get_metadata(user) collection_metadata = get_collection_metadata(user) schema = load_schema_from_metadata(metadata) completed_blocks = get_completed_blocks(user) routing_path = PathFinder(schema, answer_store, metadata, completed_blocks).get_full_routing_path() message = convert_answers(metadata, collection_metadata, schema, answer_store, routing_path, flushed=True) encrypted_message = encrypt(message, current_app.eq['key_store'], KEY_PURPOSE_SUBMISSION) sent = current_app.eq['submitter'].send_message( encrypted_message, current_app.config['EQ_RABBITMQ_QUEUE_NAME'], metadata['tx_id']) if not sent: raise SubmissionFailedException() get_questionnaire_store(user.user_id, user.user_ik).delete() return True return False
def submit_answers(schema, questionnaire_store, full_routing_path): answer_store = questionnaire_store.answer_store list_store = questionnaire_store.list_store metadata = questionnaire_store.metadata message = json.dumps( convert_answers(schema, questionnaire_store, full_routing_path), for_json=True ) encrypted_message = encrypt( message, current_app.eq["key_store"], KEY_PURPOSE_SUBMISSION ) sent = current_app.eq["submitter"].send_message( encrypted_message, questionnaire_id=metadata.get("questionnaire_id"), case_id=metadata.get("case_id"), tx_id=metadata.get("tx_id"), ) if not sent: raise SubmissionFailedException() submitted_time = datetime.utcnow() _store_submitted_time_in_session(submitted_time) if is_view_submitted_response_enabled(schema.json): _store_viewable_submission( answer_store.serialise(), list_store.serialise(), metadata, submitted_time ) get_questionnaire_store(current_user.user_id, current_user.user_ik).delete() return redirect(url_for("post_submission.get_thank_you"))
def submit(): if request.method == 'POST': data = request.get_data().decode('UTF8') logger.debug(" [x] Encrypting data") unencrypted_json = json.loads(data) no_of_submissions = int(unencrypted_json['quantity']) with open("./keys.yml") as file: secrets_from_file = yaml.safe_load(file) key_store = KeyStore(secrets_from_file) tx_id = unencrypted_json['survey']['tx_id'] for _ in range(0, no_of_submissions): # If submitting more than one then randomise the tx_id if tx_id is None: tx_id = str(uuid.uuid4()) unencrypted_json['survey']['tx_id'] = tx_id logger.info("Auto setting tx_id", tx_id=tx_id) payload = encrypt(unencrypted_json['survey'], key_store, 'submission') send_payload(payload, tx_id, 1) # let the loop handle the submission return data else: return render_template('submit.html')
def test_on_message_fails_with_missing_file_contents(self): payload = '{"filename":"test", "case_id": "601c4ee4-83ed-11e7-bb31-be2e44b06b34", "survey_id": "221"}' payload_as_json = json.loads(payload) encrypted_jwt = encrypt(payload_as_json, self.ras_key_store, KEY_PURPOSE_CONSUMER) with self.assertRaises(QuarantinableError): self.consumer.process(encrypted_jwt, uuid.uuid4())
def encrypter(data, secrets_file='app/tests/encrypter_secrets.json', key_purpose='ci_uploads'): with open(secrets_file) as fp: secrets = json.loads(fp.read()) secret_store = SecretStore(secrets) print(secret_store.__dict__) return encrypt(data, secret_store=secret_store, key_purpose=key_purpose)
def encrypt(self, payload): """ Encrypts the payload using the keystore values :param payload: the value to encrypt :return: string of encrypted data """ encrypted_data = encrypt(payload, key_store=self.key_store, key_purpose=KEY_PURPOSE) return encrypted_data
def get_seft(): messages = [] if request.method == 'POST': seft_file = request.files['file'] if seft_file.filename == '': messages.append('No selected file') logger.info("No selected file") _, file_extension = os.path.splitext(seft_file.filename) if file_extension not in ['.xls', '.xlsx']: messages.append('Incorrect file extension') logger.info("Incorrect file extension") if messages: return render_template('SEFT.html', messages=messages) tx_id = str(uuid.uuid4()) case_id = str(uuid.uuid4()) ru_ref = "12345678901A" # The submission will be put into a folder of the same name in the FTP server survey_ref = "survey_ref" _, file_extension = os.path.splitext(seft_file.filename) file_as_string = convert_file_object_to_string_base64( seft_file.stream.read()) time_date_stamp = time.strftime("%Y%m%d%H%M%S") file_name = "{ru_ref}_{exercise_ref}_" \ "{survey_ref}_{time_date_stamp}{file_format}".format(ru_ref=ru_ref, exercise_ref="exercise_ref", survey_ref=survey_ref, time_date_stamp=time_date_stamp, file_format=file_extension) logger.info("Generated filename for file going to FTP", file_name=file_name, case_id=case_id, tx_id=tx_id) message_json = { 'filename': file_name, 'file': file_as_string, 'case_id': case_id, 'survey_id': survey_ref } with open("./seft-keys.yml") as file: secrets_from_file = yaml.safe_load(file) key_store = KeyStore(secrets_from_file) payload = encrypt(message_json, key_store, 'inbound') send_payload(payload, tx_id) messages.append('File queued successfully') return render_template('SEFT.html', messages=messages)
def generate_token(json_secret_keys, payload): """ Generates the token by encrypting the payload with sdc-cryptography. """ print("Generating token") keys = json.loads(json_secret_keys) validate_required_keys(keys, KEY_PURPOSE) key_store = KeyStore(keys) encrypted_data = encrypt(payload, key_store=key_store, key_purpose=KEY_PURPOSE) return encrypted_data
def encrypt(self, payload, service): """ Encrypts the payload using the keystore values :param payload: the value to encrypt :param service: The target service, needed to know which key in the keystore to use. :return: string of encrypted data """ logger.info("About to encrypt payload", encryption_for_service=service) encrypted_data = encrypt(payload, key_store=self.key_store, key_purpose=KEY_PURPOSE, encryption_for_service=service) return encrypted_data
def test_on_message_fails_with_missing_survey_id(self): with open(join(TEST_FILES_PATH, "test1.xls"), "rb") as fb: contents = fb.read() encoded_contents = base64.b64encode(contents) payload = '{"filename":"test1.xls", "file":"' + encoded_contents.decode() + \ '", "case_id": "601c4ee4-83ed-11e7-bb31-be2e44b06b34"}' payload_as_json = json.loads(payload) encrypted_jwt = encrypt(payload_as_json, self.ras_key_store, KEY_PURPOSE_CONSUMER) with self.assertRaises(QuarantinableError): self.consumer.process(encrypted_jwt, uuid.uuid4())
def _submit_data(user): answer_store = get_answer_store(user) if answer_store: questionnaire_store = get_questionnaire_store(user.user_id, user.user_ik) answer_store = questionnaire_store.answer_store metadata = questionnaire_store.metadata response_metadata = questionnaire_store.response_metadata progress_store = questionnaire_store.progress_store list_store = questionnaire_store.list_store submitted_at = datetime.now(timezone.utc) schema = load_schema_from_metadata(metadata) router = Router( schema, answer_store, list_store, progress_store, metadata, response_metadata, ) full_routing_path = router.full_routing_path() message = json_dumps( convert_answers( schema, questionnaire_store, full_routing_path, submitted_at, flushed=True, )) encrypted_message = encrypt(message, current_app.eq["key_store"], KEY_PURPOSE_SUBMISSION) sent = current_app.eq["submitter"].send_message( encrypted_message, tx_id=metadata.get("tx_id"), case_id=metadata["case_id"], ) if not sent: raise SubmissionFailedException() get_questionnaire_store(user.user_id, user.user_ik).delete() logger.info("successfully flushed answers") return True logger.info("no answers found to flush") return False
def test_end_to_end(self): ''' End to end test - spins up a consumer and FTP server. Encrypts a message including a encoded spread sheet and takes the decrypted and reassemble file are the same files. This test requires a rabbit mq server to be running locally with the default settings It also requires a valid OPSWAT API NOTE this test should only be run manually due to it duration ''' consumer_thread = ConsumerThread(self.sdx_keys) consumer_thread.start() ftp_thread = FTPThread() ftp_thread.start() files = [ f for f in listdir(TEST_FILES_PATH) if isfile(join(TEST_FILES_PATH, f)) and ( f.endswith(".xls") or f.endswith(".xlsx")) ] for file in files: with open(join(TEST_FILES_PATH, file), "rb") as fb: contents = fb.read() encoded_contents = base64.b64encode(contents) payload = '{"filename":"' + file + '", "file":"' + encoded_contents.decode() + \ '", "case_id": "601c4ee4-83ed-11e7-bb31-be2e44b06b34","survey_id": "221"}' payload_as_json = json.loads(payload) jwt = encrypt(payload_as_json, self.ras_key_store, KEY_PURPOSE_CONSUMER) with open("./encrypted_files/" + file, "w") as encrypted_file: encrypted_file.write(jwt) queue_publisher = QueuePublisher(settings.RABBIT_URLS, settings.RABBIT_QUEUE) headers = {'tx_id': str(uuid.uuid4())} queue_publisher.publish_message(jwt, headers=headers) # wait 30 seconds for opswat to process time.sleep(30) time.sleep(1) consumer_thread.stop() ftp_thread.stop() for file in files: if 'infected' not in file: self.assertTrue( filecmp.cmp(join(TEST_FILES_PATH, file), join(EndToEndTest.TARGET_PATH, file)))
def get_seft(): messages = [] if request.method == 'POST': seft_file = request.files['file'] if seft_file.filename == '': messages.append('No selected file') logger.info("No selected file") _, file_extension = os.path.splitext(seft_file.filename) if file_extension not in ['.xls', '.xlsx']: messages.append('Incorrect file extension') logger.info("Incorrect file extension") if messages: return render_template('SEFT.html', messages=messages) tx_id = str(uuid.uuid4()) case_id = str(uuid.uuid4()) ru_ref = "12345678901A" # The submission will be put into a folder of the same name in the FTP server survey_ref = "survey_ref" _, file_extension = os.path.splitext(seft_file.filename) file_as_string = convert_file_object_to_string_base64(seft_file.stream.read()) time_date_stamp = time.strftime("%Y%m%d%H%M%S") file_name = "{ru_ref}_{exercise_ref}_" \ "{survey_ref}_{time_date_stamp}{file_format}".format(ru_ref=ru_ref, exercise_ref="exercise_ref", survey_ref=survey_ref, time_date_stamp=time_date_stamp, file_format=file_extension) logger.info("Generated filename for file going to FTP", file_name=file_name, case_id=case_id, tx_id=tx_id) message_json = { 'filename': file_name, 'file': file_as_string, 'case_id': case_id, 'survey_id': survey_ref } with open("./seft-keys.yml") as file: secrets_from_file = yaml.safe_load(file) key_store = KeyStore(secrets_from_file) payload = encrypt(message_json, key_store, 'inbound') send_payload(payload, tx_id) messages.append('File queued successfully') return render_template('SEFT.html', messages=messages)
def test_decrypt_exception(self): with open(join(TEST_FILES_PATH, "test1.xls"), "rb") as fb: contents = fb.read() encoded_contents = base64.b64encode(contents) payload = '{"filename":"test1", "file":"' + encoded_contents.decode() + \ '", "case_id": "601c4ee4-83ed-11e7-bb31-be2e44b06b34", "survey_id": "221"}' payload_as_json = json.loads(payload) encrypted_jwt = encrypt(payload_as_json, self.ras_key_store, KEY_PURPOSE_CONSUMER) with unittest.mock.patch('app.main.decrypt', side_effect=Exception): with self.assertRaises(QuarantinableError): self.consumer.process(encrypted_jwt, uuid.uuid4())
def test_file_sent_to_av_and_ftp(self, mock_deliver_binary, mock_send_for_av_scan): """Validate the message gets sent to the av and ftp""" tx_id = uuid.uuid4() with open(join(TEST_FILES_PATH, "test1.xls"), "rb") as fb: contents = fb.read() encoded_contents = base64.b64encode(contents) payload = '{"filename":"test1.xls", "file":"' + encoded_contents.decode() + \ '", "case_id": "601c4ee4-83ed-11e7-bb31-be2e44b06b34", "survey_id": "SomeSurveyId"}' payload_as_json = json.loads(payload) encrypted_jwt = encrypt(payload_as_json, self.ras_key_store, KEY_PURPOSE_CONSUMER) self.consumer.process(encrypted_jwt, tx_id) self.assertTrue(mock_deliver_binary.called) self.assertTrue(mock_send_for_av_scan.called)
def test_send_ftp_IO_error(self, mock_send_for_av_scan): with open(join(TEST_FILES_PATH, "test1.xls"), "rb") as fb: contents = fb.read() encoded_contents = base64.b64encode(contents) payload = '{"filename":"test1", "file":"' + encoded_contents.decode() + \ '", "case_id": "601c4ee4-83ed-11e7-bb31-be2e44b06b34","survey_id": "221"}' payload_as_json = json.loads(payload) encrypted_jwt = encrypt(payload_as_json, self.ras_key_store, KEY_PURPOSE_CONSUMER) with unittest.mock.patch.object(SDXFTP, 'deliver_binary') as mock_method: mock_method.side_effect = IOError with self.assertRaises(RetryableError): self.consumer.process(encrypted_jwt, uuid.uuid4()) self.assertTrue(mock_send_for_av_scan.called)
def submit_answers(routing_path, eq_id, form_type, schema): metadata = get_metadata(current_user) collection_metadata = get_collection_metadata(current_user) answer_store = get_answer_store(current_user) message = json.dumps( convert_answers( metadata, collection_metadata, schema, answer_store, routing_path, )) encrypted_message = encrypt(message, current_app.eq['key_store'], KEY_PURPOSE_SUBMISSION) sent = current_app.eq['submitter'].send_message( encrypted_message, current_app.config['EQ_RABBITMQ_QUEUE_NAME'], metadata['tx_id'], ) if current_app.config['EQ_PUBSUB_ENABLED']: current_app.eq['pubsub_submitter'].send_message( encrypted_message, current_app.config['EQ_PUBSUB_TOPIC_ID'], metadata['tx_id'], ) if not sent: raise SubmissionFailedException() submitted_time = datetime.utcnow() _store_submitted_time_in_session(submitted_time) if is_view_submitted_response_enabled(schema.json): _store_viewable_submission(list(answer_store), metadata, submitted_time) get_questionnaire_store(current_user.user_id, current_user.user_ik).delete() return redirect( url_for('post_submission.get_thank_you', eq_id=eq_id, form_type=form_type))
def test_valid_message_writes_to_log_after_ftp(self, mock_deliver_binary, mock_send_for_av_scan): """Validate that the log entry is written after a successful ftp write""" with open(join(TEST_FILES_PATH, "test1.xls"), "rb") as fb: contents = fb.read() encoded_contents = base64.b64encode(contents) payload = '{"filename":"test1.xls", "file":"' + encoded_contents.decode() + \ '", "case_id": "601c4ee4-83ed-11e7-bb31-be2e44b06b34", "survey_id": "221"}' with self.assertLogs(level="DEBUG") as cm: payload_as_json = json.loads(payload) encrypted_jwt = encrypt(payload_as_json, self.ras_key_store, KEY_PURPOSE_CONSUMER) self.consumer.process(encrypted_jwt, uuid.uuid4()) self.assertTrue( ConsumerTests._contains_statement_in_log_file( "Delivered to FTP server", cm.output)) self.assertTrue(mock_deliver_binary.called) self.assertTrue(mock_send_for_av_scan.called)
def submit_questionnaire(self): payload = self.get_payload() message = json_dumps(payload) encrypted_message = encrypt(message, current_app.eq["key_store"], KEY_PURPOSE_SUBMISSION) submitted = current_app.eq["submitter"].send_message( encrypted_message, case_id=self._metadata["case_id"], tx_id=self._metadata.get("tx_id"), ) if not submitted: raise SubmissionFailedException() cookie_session["submitted"] = True self._store_submitted_time_and_display_address_in_session() self._questionnaire_store.delete()
def test_valid_message_ftp_path_includes_survey_id(self, mock_deliver_binary, mock_send_for_av_scan): """Validates that the correct path and filename are used to deliver the ftp i.e that the survey_id is part of the path ..note:: Pycharm will pass this test even if the url is manually changed to the wrong string. It appears to be a bug in pycharm with multiple patches. The test fails under the same circumstances when run via make . """ self.consumer._ftp.deliver_binary = mock_deliver_binary with open(join(TEST_FILES_PATH, "test1.xls"), "rb") as fb: contents = fb.read() encoded_contents = base64.b64encode(contents) payload = '{"filename":"test1.xls", "file":"' + encoded_contents.decode() + \ '", "case_id": "601c4ee4-83ed-11e7-bb31-be2e44b06b34", "survey_id": "SomeSurveyId"}' payload_as_json = json.loads(payload) encrypted_jwt = encrypt(payload_as_json, self.ras_key_store, KEY_PURPOSE_CONSUMER) self.consumer.process(encrypted_jwt, uuid.uuid4()) mock_deliver_binary.assert_called_with("./SomeSurveyId", 'test1.xls', unittest.mock.ANY) self.assertTrue(mock_send_for_av_scan.called)
def test_fully_encrypted(self): key_store = KeyStore({ "keys": { SR_USER_AUTHENTICATION_PUBLIC_KEY_KID: { "purpose": KEY_PURPOSE_AUTHENTICATION, "type": "public", "value": TEST_DO_NOT_USE_SR_PUBLIC_KEY, }, EQ_USER_AUTHENTICATION_RRM_PRIVATE_KEY_KID: { "purpose": KEY_PURPOSE_AUTHENTICATION, "type": "private", "value": TEST_DO_NOT_USE_UPSTREAM_PRIVATE_KEY, }, } }) payload = self.create_payload() encrypted_token = encrypt(payload, key_store, KEY_PURPOSE_AUTHENTICATION) response = self.client.get("/session?token=" + encrypted_token) self.assertEqual(302, response.status_code)
def test_fully_encrypted(self): key_store = KeyStore({ 'keys': { SR_USER_AUTHENTICATION_PUBLIC_KEY_KID: { 'purpose': KEY_PURPOSE_AUTHENTICATION, 'type': 'public', 'value': TEST_DO_NOT_USE_SR_PUBLIC_KEY }, EQ_USER_AUTHENTICATION_RRM_PRIVATE_KEY_KID: { 'purpose': KEY_PURPOSE_AUTHENTICATION, 'type': 'private', 'value': TEST_DO_NOT_USE_UPSTREAM_PRIVATE_KEY }, } }) payload = self.create_payload() encrypted_token = encrypt(payload, key_store, KEY_PURPOSE_AUTHENTICATION) response = self.client.get('/session?token=' + encrypted_token) self.assertEqual(302, response.status_code)
def transfer_files(self): if not self.publisher.publishing: logger.warning("Publisher is not ready.") return if self.transfer: logger.warning("Cancelling overlapped task.") return else: self.transfer = True try: logger.info("Looking for files...") worker = FTPWorker(**self.ftp_params(self.services)) with worker as active: if not active: return for job in active.get(active.filenames): if job.filename not in self.recent: logger.info("Found a file to publish", filename=job.filename) data = job._asdict() data["file"] = base64.standard_b64encode( job.file).decode("ascii") data["ts"] = job.ts.isoformat() payload = encrypt(data, self.key_store, self.key_purpose) tx_id = str(uuid.uuid4()) msg_id = self.publisher.publish_message( payload, headers={'tx_id': tx_id}) if msg_id is None: logger.warning("Failed to publish file", filename=job.filename) else: self.recent[job.filename] = msg_id logger.info("Published file", filename=job.filename, tx_id=tx_id) logger.info( "Finished publishing files, checking if any files need to be deleted" ) for filename, msg_id in self.recent.items(): logger.info("Recently published file found", filename=filename, msg_id=msg_id) # The file might not be in confirmed_deliveries as the publisher waits for the delivery # confirmation adding it to the list if msg_id in self.publisher.confirmed_deliveries: logger.info( "Deleting file as it has its delivery confirmed", filename=filename, msg_id=msg_id) file_deleted = active.delete(filename) if file_deleted: del self.recent[filename] logger.info("Succssfully deleted file", filename=filename, msg_id=msg_id) else: logger.info( "Not deleting file as it hasn't had its delivery confirmed", filename=filename, msg_id=msg_id) finally: self.transfer = False logger.info("Finished looking for files.")
def generate_token(self, payload): return encrypt(payload, self._key_store, KEY_PURPOSE_AUTHENTICATION)
async def get_token(self, case_json): eq_payload = await EqPayloadConstructor(case_json, self.request.app, self.iac).build() return encrypt(eq_payload, key_store=self.request.app['key_store'], key_purpose="authentication")
class View: valid_display_regions = r'{display_region:\ben|cy|ni\b}' valid_ew_display_regions = r'{display_region:\ben|cy\b}' valid_user_journeys = r'{user_journey:\bstart|request\b}' valid_sub_user_journeys = \ r'{sub_user_journey:\blink-address|change-address|access-code|paper-questionnaire|continuation-questionnaire\b}' page_title_error_prefix_en = 'Error: ' page_title_error_prefix_cy = 'Gwall: ' @staticmethod def get_now_utc(): return datetime.utcnow() @staticmethod def single_client_ip(request): if request['client_ip']: client_ip = request['client_ip'] single_ip_validation_pattern = re.compile(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') if client_ip.count(',') > 1: single_ip_value = client_ip.split(', ', -1)[-3] if single_ip_validation_pattern.fullmatch(single_ip_value): single_ip = single_ip_value else: logger.warn('clientIP failed validation. Provided IP - ' + client_ip, client_id=request['client_id'], trace=request['trace']) single_ip = '' else: logger.warn('clientIP failed validation. Provided IP - ' + client_ip, client_id=request['client_id'], trace=request['trace']) single_ip = '' elif request.headers.get('Origin', None) and 'localhost' in request.headers.get('Origin', None): single_ip = '127.0.0.1' else: single_ip = '' return single_ip @staticmethod def log_entry(request, endpoint): method = request.method logger.info(f"received {method} on endpoint '{endpoint}'", client_ip=request['client_ip'], client_id=request['client_id'], trace=request['trace'], method=request.method, path=request.path) @staticmethod def gen_page_url(request): full_url = str(request.rel_url) if full_url[:3] == '/en' or full_url[:3] == '/cy' or full_url[:3] == '/ni': generic_url = full_url[3:] else: generic_url = full_url return generic_url @staticmethod def get_call_centre_number(display_region): if display_region == 'ni': call_centre_number = '0800 328 2021' elif display_region == 'cy': call_centre_number = '0800 169 2021' else: call_centre_number = '0800 141 2021' return call_centre_number @staticmethod def check_if_after_census_day(): wall_clock = utc.localize(View.get_now_utc()).astimezone(uk_zone) now_date = wall_clock.date() if now_date > census_day: after_census_day = True else: after_census_day = False return after_census_day @staticmethod def get_campaign_site_link(request, display_region, requested_link): base_en = request.app['DOMAIN_URL_PROTOCOL'] + request.app['DOMAIN_URL_EN'] base_cy = request.app['DOMAIN_URL_PROTOCOL'] + request.app['DOMAIN_URL_CY'] base_ni = request.app['DOMAIN_URL_PROTOCOL'] + request.app['DOMAIN_URL_EN'] + '/ni' link = '/' if requested_link == 'census-home': if display_region == 'ni': link = base_ni elif display_region == 'cy': link = base_cy else: link = base_en elif requested_link == 'contact-us': if display_region == 'ni': link = base_ni + '/contact-us/' elif display_region == 'cy': link = base_cy + '/cysylltu-a-ni/' else: link = base_en + '/contact-us/' elif requested_link == 'privacy': if display_region == 'ni': link = base_ni + '/privacy-and-data-protection/' elif display_region == 'cy': link = base_cy + '/preifatrwydd-a-diogelu-data/' else: link = base_en + '/privacy-and-data-protection/' return link @staticmethod async def _make_request(request, method, url, auth=None, headers=None, request_json=None, return_json=False): """ :param request: The AIOHTTP user request, used for logging and app access :param method: The HTTP verb :param url: The target URL :param auth: Authorization :param headers: Any needed headers as a python dictionary :param request_json: JSON payload to pass as request data :param return_json: If True, the response JSON will be returned """ retry_request = RetryRequest(request, method, url, auth, headers, request_json, return_json) return await retry_request.make_request() @staticmethod def validate_case(case_json): if not case_json.get('active', False): raise InactiveCaseError(case_json.get('caseType')) if not case_json.get('caseStatus', None) == 'OK': raise InvalidEqPayLoad('CaseStatus is not OK') @staticmethod async def call_questionnaire(request, case, attributes, app, adlocation): eq_payload = await EqPayloadConstructor(case, attributes, app, adlocation).build() token = encrypt(eq_payload, key_store=app['key_store'], key_purpose='authentication') try: await RHService.post_surveylaunched(request, case, adlocation) except ClientResponseError as ex: if ex.status == 429: raise TooManyRequestsEQLaunch() else: raise ex logger.info('redirecting to eq', client_ip=request['client_ip'], client_id=request['client_id'], trace=request['trace']) eq_url = app['EQ_URL'] raise HTTPFound(f'{eq_url}/session?token={token}')
def flush_cases(case_id): # Get iac for case iac_return = requests.get(case_url + case_id + "/iac", auth=config["CASE_AUTH"]) iac_return.raise_for_status() iac_return = iac_return.json() # Get case details case_return = requests.get(case_url + case_id, auth=config["CASE_AUTH"]) case_return.raise_for_status() case = case_return.json() # Collection instrument details ci_return = requests.get(ci_url + case["collectionInstrumentId"], auth=config["COLLECTION_INSTRUMENT_AUTH"]) ci_return.raise_for_status() ci = ci_return.json() # Get collection exercise info collex_return = requests.get(collex_url + case["caseGroup"]["collectionExerciseId"], auth=config["COLLECTION_EXERCISE_AUTH"]) collex_return.raise_for_status() collex = collex_return.json() # Get sample details sample_return = requests.get(sample_url + case["sampleUnitId"] + "/attributes", auth=config["SAMPLE_AUTH"]) sample_return.raise_for_status() sample = sample_return.json() sample_attributes = sample["attributes"] # For each IAC that is returned for a case for iac in iac_return: # Put together the payload flush_payload = { "jti": str(uuid4()), # required by eQ for creating a new claim "tx_id": str(uuid4( )), # not required by eQ (will generate if does not exist) "user_id": case["sampleUnitId"], # required by eQ "iat": int(time.time()), "exp": int(time.time() + (5 * 60)), # required by eQ for creating a new claim "eq_id": ci["classifiers"] ["eq_id"], # required but currently only one social survey ('lms') "period_id": collex["exerciseRef"], # required by eQ "form_type": ci["classifiers"] ["form_type"], # required by eQ ('2' for lms_2 schema) "collection_exercise_sid": collex["id"], # required by eQ "ru_ref": case["caseGroup"]["sampleUnitRef"], # required by eQ "case_id": case["id"], # not required by eQ but useful for downstream "case_ref": case["caseRef"], # not required by eQ but useful for downstream "account_service_url": f'{config["ACCOUNT_SERVICE_URL"]}{config["URL_PATH_PREFIX"]}', # required for save/continue "country_code": sample_attributes["COUNTRY"], "language_code": "en", # currently only 'en' or 'cy' "response_id": build_response_id(case["id"], collex["id"], iac["iac"]), "roles": "flusher" } # Get encryption key stuff config["key_store"] = jwt.key_store(config["JSON_SECRET_KEYS"]) # Encrypt payload into token token = encrypt(flush_payload, key_store=config['key_store'], key_purpose="authentication") # Call flusher flush_url = eq_url + "/flush?token=" + token flush_response = requests.post(flush_url) print( f'Response from flushing {case_id}: {flush_response.status_code}')