def setUp(self):
with open("./sdx_test_keys/keys.yml") as file:
self.sdx_keys = yaml.safe_load(file)
with open("./ras_test_keys/keys.yml") as file:
self.ras_keys = yaml.safe_load(file)
self.ras_key_store = KeyStore(self.ras_keys)
self.sdx_key_store = KeyStore(self.sdx_keys)
def decrypt_and_write():
with open(settings.SDX_SEFT_CONSUMER_KEYS_FILE) as file:
keys = yaml.safe_load(file)
key_store = KeyStore(keys)
connection = BlockingConnection(URLParameters(settings.RABBIT_URL))
channel = connection.channel()
method, properties, body = channel.basic_get(
settings.RABBIT_QUARANTINE_QUEUE)
if method:
logger.info("Recovered quarantine message",
body=body,
headers=properties.headers)
try:
decrypted_message = decrypt(body.decode("utf-8"), key_store,
KEY_PURPOSE_CONSUMER)
payload = SeftConsumer.extract_file(decrypted_message,
properties.headers['tx_id'])
with open('/tmp/{}'.format(payload.file_name),
'wb') as recovered_file:
recovered_file.write(payload.decoded_contents)
channel.basic_ack(method.delivery_tag)
logger.info("Message ACK")
except (InvalidTokenException, ValueError):
logger.exception("Bad decrypt")
channel.basic_nack(method.delivery_tag)
logger.info("Nacking message")
except Exception:
logger.exception("Failed to process")
channel.basic_nack(method.delivery_tag)
logger.info("Nacking message")
else:
logger.info('No message found on quarantine queue')
def key_store(keys: str) -> KeyStore:
secrets = json.loads(keys)
logger.info("Validating key file")
validate_required_keys(secrets, "authentication")
return KeyStore(secrets)
def submit():
if request.method == 'POST':
data = request.get_data().decode('UTF8')
logger.info("Encrypting data")
unencrypted_json = json.loads(data)
no_of_submissions = int(unencrypted_json['quantity'])
with open("./keys.yml") as file:
secrets_from_file = yaml.safe_load(file)
key_store = KeyStore(secrets_from_file)
tx_id = unencrypted_json['survey']['tx_id']
for _ in range(0, no_of_submissions):
# If submitting more than one then randomise the tx_id
if tx_id is None:
tx_id = str(uuid.uuid4())
unencrypted_json['survey']['tx_id'] = tx_id
logger.info("Auto setting tx_id", tx_id=tx_id)
payload = encrypt(unencrypted_json['survey'], key_store,
'submission')
send_payload(payload, tx_id,
1) # let the loop handle the submission
return data
else:
return render_template('submit.html')
def setUp(self):
# creates a test client
self.app = app.test_client()
# propagate the exceptions to the test client
self.app.testing = True
with open(settings.SDX_KEYS_FILE) as file:
secrets_from_file = yaml.safe_load(file)
secret_store = KeyStore(secrets_from_file)
jwt_key = secret_store.get_key_for_purpose_and_type(
KEY_PURPOSE_SUBMISSION, "private")
jwe_key = secret_store.get_key_for_purpose_and_type(
KEY_PURPOSE_SUBMISSION, "public")
self.encrypter = Encrypter(jwt_key.kid, jwe_key.kid)
def test_incomplete_key():
"""Tests that an exception is thrown a malformed key is created with the keystore"""
with pytest.raises(CryptoError):
KeyStore({
"keys": {
"e19091072f920cbf3ca9f436ceba309e7d814a62": {'purpose': KEY_PURPOSE_AUTHENTICATION,
'value': TEST_DO_NOT_USE_SR_PRIVATE_PEM},
}
})
def load_secrets(key_purpose_submission, expected_secrets=[]):
"""Load secrets from a local yaml file."""
logger.info("Loading keys from environment")
json_string = os.getenv('CI_SECRETS')
secrets = json.loads(json_string)
logger.info("Loaded keys from environment")
secrets = KeyStore(secrets)
return secrets
def get_seft():
messages = []
if request.method == 'POST':
seft_file = request.files['file']
if seft_file.filename == '':
messages.append('No selected file')
logger.info("No selected file")
_, file_extension = os.path.splitext(seft_file.filename)
if file_extension not in ['.xls', '.xlsx']:
messages.append('Incorrect file extension')
logger.info("Incorrect file extension")
if messages:
return render_template('SEFT.html', messages=messages)
tx_id = str(uuid.uuid4())
case_id = str(uuid.uuid4())
ru_ref = "12345678901A"
# The submission will be put into a folder of the same name in the FTP server
survey_ref = "survey_ref"
_, file_extension = os.path.splitext(seft_file.filename)
file_as_string = convert_file_object_to_string_base64(
seft_file.stream.read())
time_date_stamp = time.strftime("%Y%m%d%H%M%S")
file_name = "{ru_ref}_{exercise_ref}_" \
"{survey_ref}_{time_date_stamp}{file_format}".format(ru_ref=ru_ref,
exercise_ref="exercise_ref",
survey_ref=survey_ref,
time_date_stamp=time_date_stamp,
file_format=file_extension)
logger.info("Generated filename for file going to FTP",
file_name=file_name,
case_id=case_id,
tx_id=tx_id)
message_json = {
'filename': file_name,
'file': file_as_string,
'case_id': case_id,
'survey_id': survey_ref
}
with open("./seft-keys.yml") as file:
secrets_from_file = yaml.safe_load(file)
key_store = KeyStore(secrets_from_file)
payload = encrypt(message_json, key_store, 'inbound')
send_payload(payload, tx_id)
messages.append('File queued successfully')
return render_template('SEFT.html', messages=messages)
def create_app():
app = Flask(__name__)
app.config.from_object(settings)
app.sdx = {}
with open(app.config['SDX_KEYS_FILE']) as file:
keys = yaml.safe_load(file)
validate_required_keys(keys, KEY_PURPOSE_SUBMISSION)
app.sdx['key_store'] = KeyStore(keys)
return app
def generate_token(json_secret_keys, payload):
"""
Generates the token by encrypting the payload with sdc-cryptography.
"""
print("Generating token")
keys = json.loads(json_secret_keys)
validate_required_keys(keys, KEY_PURPOSE)
key_store = KeyStore(keys)
encrypted_data = encrypt(payload,
key_store=key_store,
key_purpose=KEY_PURPOSE)
return encrypted_data
def setUp(self):
with open("./sdx_test_keys/keys.yml") as file:
self.sdx_keys = yaml.safe_load(file)
with open("./ras_test_keys/keys.yml") as file:
self.ras_keys = yaml.safe_load(file)
self.ras_key_store = KeyStore(self.ras_keys)
dir_path = "./ftp/221"
file_list = os.listdir(dir_path)
for file_name in file_list:
if not file_name == ".placeholder":
os.remove(EndToEndTest.TARGET_PATH + file_name)
def __init__(self, args, services):
self.args = args
self.services = services
self.publisher = DurableTopicPublisher(**self.amqp_params(services))
self.executor = ThreadPoolExecutor(max_workers=4)
self.rabbit_check = None
self.ftp_check = None
self.transfer = False
self.key_purpose = 'outbound'
keys_file_location = os.getenv('SDX_SEFT_CONSUMER_KEYS_FILE',
'./jwt-test-keys/keys.yml')
with open(keys_file_location) as file:
self.secrets_from_file = yaml.safe_load(file)
validate_required_keys(self.secrets_from_file, self.key_purpose)
self.key_store = KeyStore(self.secrets_from_file)
def __init__(self, keys):
self.bound_logger = logger
self.key_store = KeyStore(keys)
self._ftp = SDXFTP(logger, settings.FTP_HOST, settings.FTP_USER,
settings.FTP_PASS, settings.FTP_PORT)
self.publisher = QueuePublisher(urls=settings.RABBIT_URLS,
queue=settings.RABBIT_QUARANTINE_QUEUE)
self.consumer = MessageConsumer(durable_queue=True,
exchange=settings.RABBIT_EXCHANGE,
exchange_type="topic",
rabbit_queue=settings.RABBIT_QUEUE,
rabbit_urls=settings.RABBIT_URLS,
quarantine_publisher=self.publisher,
process=self.process)
self.session = requests.Session()
retries = Retry(total=SERVICE_REQUEST_TOTAL_RETRIES,
backoff_factor=SERVICE_REQUEST_BACKOFF_FACTOR)
self.session.mount('http://', HTTPAdapter(max_retries=retries))
self.session.mount('https://', HTTPAdapter(max_retries=retries))
class TestDecrypter:
key_store = KeyStore({
"keys": {
"e19091072f920cbf3ca9f436ceba309e7d814a62": {
'purpose': KEY_PURPOSE_AUTHENTICATION,
'type': 'private',
'value': TEST_DO_NOT_USE_SR_PRIVATE_PEM
},
"EQ_USER_AUTHENTICATION_SR_PRIVATE_KEY": {
'purpose': KEY_PURPOSE_AUTHENTICATION,
'type': 'private',
'value': TEST_DO_NOT_USE_SR_PRIVATE_PEM
},
"EDCRRM": {
'purpose': KEY_PURPOSE_AUTHENTICATION,
'type': 'public',
'value': TEST_DO_NOT_USE_PUBLIC_KEY
},
"709eb42cfee5570058ce0711f730bfbb7d4c8ade": {
'purpose': KEY_PURPOSE_AUTHENTICATION,
'type': 'public',
'value': TEST_DO_NOT_USE_UPSTREAM_PUBLIC_PEM
},
}
})
def test_decrypt(self):
json = decrypt(VALID_JWE, self.key_store, KEY_PURPOSE_AUTHENTICATION)
assert json == {
'user': '******',
'iat': 1498137519.135479,
'exp': 1.0000000000014982e+21
}
def test_decrypt_too_few_tokens_in_jwe(self):
"""Tests an InvalidTokenException when the token isn't comprised of 5 parts, seperated by several '.' characters"""
with pytest.raises(InvalidTokenException):
decrypt(TOO_FEW_TOKENS_JWE, self.key_store,
KEY_PURPOSE_AUTHENTICATION)
def test_fully_encrypted(self):
key_store = KeyStore({
'keys': {
SR_USER_AUTHENTICATION_PUBLIC_KEY_KID: {
'purpose': KEY_PURPOSE_AUTHENTICATION,
'type': 'public',
'value': TEST_DO_NOT_USE_SR_PUBLIC_KEY
},
EQ_USER_AUTHENTICATION_RRM_PRIVATE_KEY_KID: {
'purpose': KEY_PURPOSE_AUTHENTICATION,
'type': 'private',
'value': TEST_DO_NOT_USE_UPSTREAM_PRIVATE_KEY
},
}
})
payload = self.create_payload()
encrypted_token = encrypt(payload, key_store,
KEY_PURPOSE_AUTHENTICATION)
response = self.client.get('/session?token=' + encrypted_token)
self.assertEqual(302, response.status_code)
def test_fully_encrypted(self):
key_store = KeyStore({
"keys": {
SR_USER_AUTHENTICATION_PUBLIC_KEY_KID: {
"purpose": KEY_PURPOSE_AUTHENTICATION,
"type": "public",
"value": TEST_DO_NOT_USE_SR_PUBLIC_KEY,
},
EQ_USER_AUTHENTICATION_RRM_PRIVATE_KEY_KID: {
"purpose": KEY_PURPOSE_AUTHENTICATION,
"type": "private",
"value": TEST_DO_NOT_USE_UPSTREAM_PRIVATE_KEY,
},
}
})
payload = self.create_payload()
encrypted_token = encrypt(payload, key_store,
KEY_PURPOSE_AUTHENTICATION)
response = self.client.get("/session?token=" + encrypted_token)
self.assertEqual(302, response.status_code)
def create_app( # noqa: C901 pylint: disable=too-complex, too-many-statements
setting_overrides=None,
):
application = Flask(__name__, template_folder="../templates")
application.config.from_object(settings)
if setting_overrides:
application.config.update(setting_overrides)
application.eq = {}
with open(application.config["EQ_SECRETS_FILE"]) as secrets_file:
secrets = yaml.safe_load(secrets_file)
conditional_required_secrets = []
if application.config["ADDRESS_LOOKUP_API_AUTH_ENABLED"]:
conditional_required_secrets.append("ADDRESS_LOOKUP_API_AUTH_TOKEN_SECRET")
validate_required_secrets(secrets, conditional_required_secrets)
application.eq["secret_store"] = SecretStore(secrets)
with open(application.config["EQ_KEYS_FILE"]) as keys_file:
keys = yaml.safe_load(keys_file)
validate_required_keys(keys, KEY_PURPOSE_SUBMISSION)
application.eq["key_store"] = KeyStore(keys)
if application.config["EQ_APPLICATION_VERSION"]:
logger.info(
"starting eq survey runner",
version=application.config["EQ_APPLICATION_VERSION"],
)
# IMPORTANT: This must be initialised *before* any other Flask plugins that add
# before_request hooks. Otherwise any logging by the plugin in their before
# request will use the logger context of the previous request.
@application.before_request
def before_request(): # pylint: disable=unused-variable
request_id = str(uuid4())
logger.new(request_id=request_id)
span, trace = get_span_and_trace(flask_request.headers)
if span and trace:
logger.bind(span=span, trace=trace)
logger.info(
"request",
method=flask_request.method,
url_path=flask_request.full_path,
session_cookie_present="session" in flask_request.cookies,
csrf_token_present="csrf_token" in cookie_session,
user_agent=flask_request.user_agent.string,
)
setup_storage(application)
setup_submitter(application)
setup_feedback(application)
setup_publisher(application)
setup_task_client(application)
application.eq["id_generator"] = UserIDGenerator(
application.config["EQ_SERVER_SIDE_STORAGE_USER_ID_ITERATIONS"],
application.eq["secret_store"].get_secret_by_name(
"EQ_SERVER_SIDE_STORAGE_USER_ID_SALT"
),
application.eq["secret_store"].get_secret_by_name(
"EQ_SERVER_SIDE_STORAGE_USER_IK_SALT"
),
)
cache_questionnaire_schemas()
setup_secure_cookies(application)
setup_secure_headers(application)
setup_babel(application)
application.wsgi_app = AWSReverseProxied(application.wsgi_app)
application.url_map.strict_slashes = False
add_blueprints(application)
login_manager.init_app(application)
add_safe_health_check(application)
setup_compression(application)
setup_jinja_env(application)
@application.after_request
def apply_caching(response): # pylint: disable=unused-variable
if "text/html" in response.content_type:
for k, v in CACHE_HEADERS.items():
response.headers[k] = v
else:
response.headers["Cache-Control"] = "max-age=2628000, public"
return response
@application.after_request
def response_minify(response): # pylint: disable=unused-variable
"""
minify html response to decrease site traffic
"""
if (
application.config["EQ_ENABLE_HTML_MINIFY"]
and response.content_type == "text/html; charset=utf-8"
):
response.set_data(
minify(
response.get_data(as_text=True),
remove_comments=True,
remove_empty_space=True,
remove_optional_attribute_quotes=False,
)
)
return response
return response
@application.after_request
def after_request(response): # pylint: disable=unused-variable
# We're using the stringified version of the Flask session to get a rough
# length for the cookie. The real length won't be known yet as Flask
# serializes and adds the cookie header after this method is called.
logger.info(
"response",
status_code=response.status_code,
session_modified=cookie_session.modified,
)
return response
return application
def _set_up_app(self):
self._ddb = mock_dynamodb2()
self._ddb.start()
from application import configure_logging
configure_logging()
setting_overrides = {
'SQLALCHEMY_DATABASE_URI': 'sqlite://',
'EQ_DYNAMODB_ENDPOINT': None
}
self._application = create_app(setting_overrides)
self._key_store = KeyStore({
'keys': {
EQ_USER_AUTHENTICATION_RRM_PRIVATE_KEY_KID: {
'purpose':
KEY_PURPOSE_AUTHENTICATION,
'type':
'private',
'value':
get_file_contents(
'third-party/sdc-rrm-authentication-signing-private-v1.pem'
)
},
SR_USER_AUTHENTICATION_PUBLIC_KEY_KID: {
'purpose':
KEY_PURPOSE_AUTHENTICATION,
'type':
'public',
'value':
get_file_contents(
'third-party/sdc-sr-authentication-encryption-public-v1.pem'
)
},
EQ_SUBMISSION_SDX_PRIVATE_KEY: {
'purpose':
KEY_PURPOSE_SUBMISSION,
'type':
'private',
'value':
get_file_contents(
'third-party/sdc-sdx-submission-encryption-private-v1.pem'
)
},
EQ_SUBMISSION_SR_PRIVATE_SIGNING_KEY: {
'purpose':
KEY_PURPOSE_SUBMISSION,
'type':
'public',
'value':
get_file_contents(
'sdc-sr-submission-signing-private-v1.pem')
},
}
})
self.token_generator = TokenGenerator(
self._key_store, EQ_USER_AUTHENTICATION_RRM_PRIVATE_KEY_KID,
SR_USER_AUTHENTICATION_PUBLIC_KEY_KID)
self._client = self._application.test_client()
with self._application.app_context():
setup_tables()
if trim:
data = data.rstrip('\r\n')
return data
_key_store = KeyStore({
'keys': {
EQ_USER_AUTHENTICATION_RRM_PRIVATE_KEY_KID: {
'purpose':
KEY_PURPOSE_AUTHENTICATION,
'type':
'private',
'value':
get_file_contents(
'sdc-user-authentication-signing-rrm-private-key.pem'),
},
SR_USER_AUTHENTICATION_PUBLIC_KEY_KID: {
'purpose':
KEY_PURPOSE_AUTHENTICATION,
'type':
'public',
'value':
get_file_contents(
'sdc-user-authentication-encryption-sr-public-key.pem'),
},
}
})
def _get_payload_with_params(schema_name, survey_url=None, **extra_payload):
payload_vars = PAYLOAD.copy()
payload_vars['tx_id'] = str(uuid4())
class TestJWEHelper:
CHECK_CLAIMS = {
"exp": None,
"iat": None,
}
key_store = KeyStore({
"keys": {
"e19091072f920cbf3ca9f436ceba309e7d814a62": {
'purpose': KEY_PURPOSE_AUTHENTICATION,
'type': 'private',
'value': TEST_DO_NOT_USE_SR_PRIVATE_PEM
},
"EQ_USER_AUTHENTICATION_SR_PRIVATE_KEY": {
'purpose': KEY_PURPOSE_AUTHENTICATION,
'type': 'private',
'value': TEST_DO_NOT_USE_SR_PRIVATE_PEM
},
"EDCRRM": {
'purpose': KEY_PURPOSE_AUTHENTICATION,
'type': 'public',
'value': TEST_DO_NOT_USE_PUBLIC_KEY
},
"709eb42cfee5570058ce0711f730bfbb7d4c8ade": {
'purpose': KEY_PURPOSE_AUTHENTICATION,
'type': 'public',
'value': TEST_DO_NOT_USE_UPSTREAM_PUBLIC_PEM
},
}
})
kid = "e19091072f920cbf3ca9f436ceba309e7d814a62"
encoder_args = (TEST_DO_NOT_USE_UPSTREAM_PRIVATE_KEY,
TEST_DO_NOT_USE_SR_PUBLIC_KEY)
def test_decrypt_jwe_valid(self):
token = JWEHelper.decrypt(VALID_JWE, self.key_store,
KEY_PURPOSE_AUTHENTICATION)
assert VALID_SIGNED_JWT == token
def test_decrypt_with_valid_key(self):
result = JWEHelper.decrypt_with_key(
VALID_JWE,
self.key_store.get_private_key_by_kid(KEY_PURPOSE_AUTHENTICATION,
self.kid).as_jwk())
assert result == "eyJraWQiOiI3MDllYjQyY2ZlZTU1NzAwNThjZTA3MTFmNzMwYmZiYjdkNGM4YWRlIiwiYWxnIjoiUlMyNTYiLCJ0eXAiOiJqd3QifQ.eyJ1c2VyIjoi" \
"amltbXkiLCJpYXQiOjE0OTgxMzc1MTkuMTM1NDc5LCJleHAiOjEuMDAwMDAwMDAwMDAxNDk4MmUrMjF9.tXGcIZfbTIgxrd7ILj_XqcoiRLtmgjnJ0W" \
"ORPBJ4M9Kd3zKTBkoIM6pN5XWdqsfvdby53mxQzi3_DZS4Ab4XvF29Wce49GVv7k69ZZJ-5g2NX9iJy4_Be8uTZNKSwMpfrnkRrsbaWAGrXe9NKC3WC" \
"_Iq4UuE3KM7ltvOae4be-2863DP7_QEUtaAtXSwUkjPcgkvMPns-SurtFNXgFFVToNnwIuJ9UWsY8JlX1UB56wfqu68hbl88lenIf9Ym0r5hq0DlOZY" \
"NtjVizVDFciRx_52d4oeKMSzwJ1jB5aZ7YKRNHTo38Kltb5FkHRcIkV1Ae68-5dZeE9Yu_JHPMi_hw"
@staticmethod
def test_decrypt_with_key_with_invalid_key():
with pytest.raises(InvalidTokenException):
JWEHelper.decrypt_with_key(VALID_JWE, "not_a_jwk")
def test_decrypt_jwe_does_not_contain_four_instances_of_full_stop(self):
jwe = VALID_JWE.replace('.', '', 1)
self.assert_in_decrypt_exception(jwe, "InvalidJWEData")
def test_missing_algorithm(self):
jwe_protected_header = bytes(
'{"enc":"A256GCM","kid":"' + self.kid + '"}', 'utf-8')
encoder = Encoder(*self.encoder_args)
jwe = encoder.encrypt_token(VALID_SIGNED_JWT.encode(),
self.kid,
jwe_protected_header=jwe_protected_header)
self.assert_in_decrypt_exception(jwe.decode(), "Algorithm not allowed")
def test_invalid_algorithm(self):
jwe_protected_header = bytes(
'{"alg":"PBES2_HS256_A128KW","enc":"A256GCM","kid":"' + self.kid +
'"}', 'utf-8')
encoder = Encoder(*self.encoder_args)
jwe = encoder.encrypt_token(VALID_SIGNED_JWT.encode(),
self.kid,
jwe_protected_header=jwe_protected_header)
self.assert_in_decrypt_exception(jwe.decode(), "Algorithm not allowed")
def test_enc_missing(self):
jwe_protected_header = bytes(
'{"alg":"PBES2_HS256_A128KW","kid":"' + self.kid + '"}', 'utf-8')
encoder = Encoder(*self.encoder_args)
jwe = encoder.encrypt_token(VALID_SIGNED_JWT.encode(),
self.kid,
jwe_protected_header=jwe_protected_header)
self.assert_in_decrypt_exception(jwe.decode(), "Algorithm not allowed")
def test_missing_kid(self):
jwe_protected_header = bytes('{"alg":"RSA-OAEP","enc":"A256GCM"}',
'utf-8')
encoder = Encoder(*self.encoder_args)
jwe = encoder.encrypt_token(VALID_SIGNED_JWT.encode(),
self.kid,
jwe_protected_header=jwe_protected_header)
self.assert_in_decrypt_exception(jwe.decode(), "Missing kid")
def test_invalid_enc(self):
jwe_protected_header = bytes(
'{"alg":"PBES2_HS256_A128KW","enc":"A128GCM","kid":"' + self.kid +
'"}', 'utf-8')
encoder = Encoder(*self.encoder_args)
jwe = encoder.encrypt_token(VALID_SIGNED_JWT.encode(),
self.kid,
jwe_protected_header=jwe_protected_header)
self.assert_in_decrypt_exception(jwe.decode(), "Algorithm not allowed")
def test_jwe_header_contains_kid(self):
jwe_protected_header = bytes('{"alg":"RSA-OAEP","enc":"A256GCM"}',
'utf-8')
encoder = Encoder(*self.encoder_args)
jwe = encoder.encrypt_token(VALID_SIGNED_JWT.encode(),
self.kid,
jwe_protected_header=jwe_protected_header)
self.assert_in_decrypt_exception(jwe.decode(), "Missing kid")
def test_jwe_key_not_2048_bits(self):
cek = os.urandom(32)
encoder = Encoder(*self.encoder_args)
encoder.cek = cek
encrypted_key = encoder._encrypted_key(cek) # pylint: disable=protected-access
encrypted_key = encrypted_key[0:-2]
jwe = encoder.encrypt_token(VALID_SIGNED_JWT.encode(),
self.kid,
encrypted_key=encrypted_key)
self.assert_in_decrypt_exception(jwe.decode(), "ValueError")
def test_cek_not_256_bits(self):
cek = os.urandom(24)
encoder = Encoder(*self.encoder_args)
encoder.cek = cek
jwe = encoder.encrypt_token(VALID_SIGNED_JWT.encode(), self.kid)
self.assert_in_decrypt_exception(
jwe.decode(), "Expected key of length 256, got 192")
def test_authentication_tag_not_128_bits(self):
encoder = Encoder(*self.encoder_args)
jwe = encoder.encrypt_token(VALID_SIGNED_JWT.encode(),
self.kid,
tag=os.urandom(10))
self.assert_in_decrypt_exception(
jwe.decode(), "Authentication tag must be 16 bytes or longer")
def assert_in_decrypt_exception(self, jwe, error):
with pytest.raises(InvalidTokenException) as ite:
JWEHelper.decrypt(jwe, self.key_store, KEY_PURPOSE_AUTHENTICATION)
# Looks weird, but ite.value is an exception object. The error message is contained in the 'value' attribute
# of that object.
if error not in ite.value.value:
raise AssertionError(
'"{}" not found in decrypt exception. Actual exception message [{}]'
.format(error, ite.value.value))
def test_authentication_tag_corrupted(self):
encoder = Encoder(*self.encoder_args)
jwe = encoder.encrypt_token(VALID_SIGNED_JWT.encode(),
self.kid,
tag=b'adssadsadsadsadasdasdasads')
with pytest.raises(InvalidTokenException):
JWEHelper.decrypt(jwe.decode(), self.key_store,
KEY_PURPOSE_AUTHENTICATION)
def test_cipher_text_corrupted(self):
encoder = Encoder(*self.encoder_args)
jwe = encoder.encrypt_token(VALID_SIGNED_JWT.encode(), self.kid)
tokens = jwe.decode().split('.')
jwe_protected_header = tokens[0]
encrypted_key = tokens[1]
encoded_iv = tokens[2]
encoded_cipher_text = tokens[3]
encoded_tag = tokens[4]
corrupted_cipher = encoded_cipher_text[0:-1]
reassembled = jwe_protected_header + "." + encrypted_key + "." + encoded_iv + "." + corrupted_cipher + "." + encoded_tag
with pytest.raises(InvalidTokenException):
JWEHelper.decrypt(reassembled, self.key_store,
KEY_PURPOSE_AUTHENTICATION)
def test_encrypt_with_missing_key_store(self):
with pytest.raises(AttributeError):
JWEHelper.encrypt(VALID_JWE, self.kid)
def test_encrypt_with_bad_payload(self):
with pytest.raises(InvalidTokenException):
JWEHelper.encrypt(None, "709eb42cfee5570058ce0711f730bfbb7d4c8ade",
self.key_store, KEY_PURPOSE_AUTHENTICATION)
def test_encrypt_with_key_with_bad_payload(self):
with pytest.raises(InvalidTokenException):
JWEHelper.encrypt_with_key(
None, self.kid,
self.key_store.get_private_key_by_kid(
KEY_PURPOSE_AUTHENTICATION, self.kid).as_jwk())
class TestTokenHelper:
CHECK_CLAIMS = {
"exp": None,
"iat": None,
}
key_store = KeyStore({
"keys": {
"e19091072f920cbf3ca9f436ceba309e7d814a62": {
'purpose': KEY_PURPOSE_AUTHENTICATION,
'type': 'private',
'value': TEST_DO_NOT_USE_SR_PRIVATE_PEM
},
"EQ_USER_AUTHENTICATION_SR_PRIVATE_KEY": {
'purpose': KEY_PURPOSE_AUTHENTICATION,
'type': 'private',
'value': TEST_DO_NOT_USE_SR_PRIVATE_PEM
},
"EDCRRM": {
'purpose': KEY_PURPOSE_AUTHENTICATION,
'type': 'public',
'value': TEST_DO_NOT_USE_PUBLIC_KEY
},
"709eb42cfee5570058ce0711f730bfbb7d4c8ade": {
'purpose': KEY_PURPOSE_AUTHENTICATION,
'type': 'public',
'value': TEST_DO_NOT_USE_UPSTREAM_PUBLIC_PEM
},
"EQ_USER_AUTHENTICATION_EQ_KEY": {
'purpose': KEY_PURPOSE_AUTHENTICATION,
'type': 'private',
'value': TEST_DO_NOT_USE_EQ_PRIVATE_KEY
},
}
})
key_store_secondary = KeyStore({
"keys": {
"EQ_USER_AUTHENTICATION_EQ_KEY": {
'purpose': KEY_PURPOSE_AUTHENTICATION,
'type': 'public',
'value': TEST_DO_NOT_USE_EQ_PUBLIC_KEY
},
}
})
kid = "e19091072f920cbf3ca9f436ceba309e7d814a62"
encoder_args = (TEST_DO_NOT_USE_UPSTREAM_PRIVATE_KEY,
TEST_DO_NOT_USE_SR_PUBLIC_KEY)
def test_jwt_io(self):
token = JWTHelper.decode(jwtio_signed,
self.key_store,
purpose=KEY_PURPOSE_AUTHENTICATION,
leeway=100,
check_claims=self.CHECK_CLAIMS)
assert token.get("user") == "jimmy"
def test_does_not_contain_two_instances_of_full_stop(self):
jwe = jwtio_signed.replace('.', '', 1)
self.assert_in_decode_signed_jwt_exception(jwe, "Invalid Header")
def test_jwt_contains_empty_header(self):
token_without_header = "e30." + jwtio_payload + "." + jwtio_signature
self.assert_in_decode_signed_jwt_exception(token_without_header,
"Missing kid")
def test_jwt_does_not_contain_header_at_all(self):
token_without_header = "." + jwtio_payload + "." + jwtio_signature
self.assert_in_decode_signed_jwt_exception(token_without_header,
"Missing Headers")
def test_jwt_contains_empty_payload(self):
token_without_payload = jwtio_header + ".e30." + jwtio_signature
self.assert_in_decode_signed_jwt_exception(token_without_payload,
"InvalidSignature")
def test_jwt_does_not_contain_payload(self):
token_without_payload = jwtio_header + ".." + jwtio_signature
self.assert_in_decode_signed_jwt_exception(token_without_payload,
"InvalidSignature")
def test_jwt_does_not_contain_signature(self):
jwt = jwtio_header + "." + jwtio_payload + ".e30"
self.assert_in_decode_signed_jwt_exception(jwt, "InvalidSignature")
def test_jose_header_missing_type(self):
header = base64.urlsafe_b64encode(b'{"alg":"RS256", "kid":"EDCRRM"}')
jwt = header.decode() + "." + jwtio_payload + "." + jwtio_signature
self.assert_in_decode_signed_jwt_exception(jwt, "InvalidSignature")
def test_jose_header_invalid_type(self):
header = base64.urlsafe_b64encode(
b'{"alg":"RS256", "kid":"EDCRRM", "typ":"TEST"}')
jwt = header.decode() + "." + jwtio_payload + "." + jwtio_signature
self.assert_in_decode_signed_jwt_exception(jwt, "InvalidSignature")
def test_jose_header_contains_multiple_type(self):
header = base64.urlsafe_b64encode(
b'{"alg":"RS256", "kid":"EDCRRM","typ":"JWT","typ":"TEST"}')
jwt = header.decode() + "." + jwtio_payload + "." + jwtio_signature
self.assert_in_decode_signed_jwt_exception(jwt, "InvalidSignature")
def test_jose_header_missing_alg(self):
header = base64.urlsafe_b64encode(b'{"kid":"EDCRRM","typ":"JWT"}')
jwt = header.decode() + "." + jwtio_payload + "." + jwtio_signature
self.assert_in_decode_signed_jwt_exception(jwt,
"No \"alg\" in headers")
def test_jose_header_invalid_alg(self):
header = base64.urlsafe_b64encode(
b'{"alg":"invalid","kid":"EDCRRM","typ":"JWT"}')
jwt = header.decode() + "." + jwtio_payload + "." + jwtio_signature
self.assert_in_decode_signed_jwt_exception(jwt,
"Algorithm not allowed")
def test_jose_header_none_alg(self):
header = base64.urlsafe_b64encode(
b'{"alg":"None","kid":"EDCRRM","typ":"JWT"}')
jwt = header.decode() + "." + jwtio_payload + "." + jwtio_signature
self.assert_in_decode_signed_jwt_exception(jwt,
"Algorithm not allowed")
def test_jose_header_contains_multiple_alg(self):
header = base64.urlsafe_b64encode(
b'{"alg":"RS256", "alg":"HS256","kid":"EDCRRM", "typ":"JWT"}')
jwt = header.decode() + "." + jwtio_payload + "." + jwtio_signature
self.assert_in_decode_signed_jwt_exception(jwt,
"Algorithm not allowed")
def test_jose_header_missing_kid(self):
header = base64.urlsafe_b64encode(b'{"alg":"RS256", "typ":"JWT"}')
jwt = header.decode() + "." + jwtio_payload + "." + jwtio_signature
self.assert_in_decode_signed_jwt_exception(jwt, "Missing kid")
def test_jose_header_contains_multiple_kid(self):
header = base64.urlsafe_b64encode(
b'{"alg":"RS256", "kid":"test", "kid":"EDCRRM", "typ":"JWT"}')
jwt = header.decode() + "." + jwtio_payload + "." + jwtio_signature
self.assert_in_decode_signed_jwt_exception(jwt, "InvalidSignature")
def test_jose_header_contains_invalid_kid(self):
header = base64.urlsafe_b64encode(
b'{"alg":"RS256", "kid":"UNKNOWN", "typ":"JWT"}')
jwt = header.decode() + "." + jwtio_payload + "." + jwtio_signature
self.assert_in_decode_signed_jwt_exception(
jwt, "Invalid public Key Identifier")
def test_signature_not_2048_bits(self):
jwt = jwtio_header + "." + jwtio_payload + "." + base64.urlsafe_b64encode(
os.urandom(255)).decode()
self.assert_in_decode_signed_jwt_exception(jwt, "InvalidSignature")
def test_payload_corrupt(self):
jwt = jwtio_header + ".asdasd." + jwtio_signature
self.assert_in_decode_signed_jwt_exception(jwt, "InvalidSignature")
def test_header_corrupt(self):
jwt = "asdsadsa" + "." + jwtio_payload + "." + jwtio_signature
self.assert_in_decode_signed_jwt_exception(jwt, "Invalid Header")
def test_signature_corrupt(self):
jwt = jwtio_header + "." + jwtio_payload + ".asdasddas"
self.assert_in_decode_signed_jwt_exception(jwt,
"Invalid base64 string")
def test_payload_contains_malformed_json(self):
payload = base64.urlsafe_b64encode(
b'{"user":"******"iat": "1454935765","exp": "2075297148"')
jwt = jwtio_header + "." + payload.decode() + "." + jwtio_signature
self.assert_in_decode_signed_jwt_exception(jwt, "InvalidSignature")
def test_payload_contains_corrupted_json(self):
payload = base64.urlsafe_b64encode(
b'{"user":"******","iat": "1454935765","exp": "2075297148"}ABDCE')
jwt = jwtio_header + "." + payload.decode() + "." + jwtio_signature
self.assert_in_decode_signed_jwt_exception(jwt, "InvalidSignature")
def test_payload_does_not_contain_exp(self):
valid_token_no_exp = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IkVEQ1JSTSJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibm" \
"FtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6IjE0NTQ5MzU3NjcifQ.VupTBEOEzeDjxd37PQ34xv" \
"BlLzeGTA0xFdGnLZDcnxAS1AjNcJ66edRmr4tmPIXnD6Mgen3HSB36xuXSnfzPld2msFHUXmB18CoaJQK19BXEY" \
"vosrBPzc1ohSvam_DgXCzdSMAcWSE63e6LTWNCT93-npD3p9tjdY_TWpEOOg14"
self.assert_in_decode_signed_jwt_exception(valid_token_no_exp,
"Claim exp is missing")
def test_payload_does_not_contain_iat(self):
valid_token_no_iat = "eyJhbGciOiJSUzI1NiIsImtpZCI6IjcwOWViNDJjZmVlNTU3MDA1OGNlMDcxMWY3MzBiZmJiN2Q0YzhhZGUiLCJ" \
"0eXAiOiJqd3QifQ.eyJlcV9pZCI6IjEiLCJleHAiOjIwNzcxODg5MDksImZvcm1fdHlwZSI6IjAyMDUiLCJqdGk" \
"iOiIzMmIxNDdjNS04OWEzLTQxMzUtYjgxMy02YzQzNTE1Yzk3MTkifQ.lPTbkzQhrktcRCgn2-ku4eqr5zpgetn" \
"I8JjipBsm3WrxALnnQc4QebtsPIP9vxv9cRLkis6FMZa3Lm6A5fVAHwsCKMOsDjBFf3QXVtLIgRMW-Q8VNowj5F" \
"UW5TAQhRAka-Og9lI3gTpcN-ynhnb0arlGKhbzJU03K0KEBPTT6TDRUeKZAUTAA29qxmPIVbhuQNAjmHX7uSW4z" \
"_OKLi1OdIlFEvC6X5rddkfv2yhGDNpO4ZfUcHvcfCgyg16WQDSBKVLQf2uk8-Ju_zOv4818Obb12N7CJvAb5eys" \
"vnW3MSbAQhvvJJYe8WCN7j1uHZxRpwIPgAGvGiN9Sa1Gq14EWA"
self.assert_in_decode_signed_jwt_exception(valid_token_no_iat,
"Claim iat is missing")
def test_payload_invalid_exp(self):
valid_token_with_invalid_exp = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IkVEQ1JSTSJ9.eyJzdWIiOiIxMjM0NTY3" \
"ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6IjE0NTQ5MzU3NjUiLCJle" \
"HAiOiI_In0.0ApxEXw1rzo21XQo8WgcPvnz0e8QnT0GaoXVbCj-OdJtB7GArPzaiQ1cU53WaJsvGE" \
"zHTczc6Y0xN7WzcTdcXN8Yjenf4VqoiYc6_FXGJ1s9Brd0JOFPyVipTFxPoWvYTWLXE-CAEpXrEb3" \
"0kB3nRjHFV_yVhLiiZUU-gpUHqNQ"
self.assert_in_decode_signed_jwt_exception(
valid_token_with_invalid_exp, "Claim exp is not an integer")
def test_payload_invalid_iat(self):
valid_token_with_invalid_iat = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IkVEQ1JSTSJ9.eyJzdWIiOiIxMjM0NTY3" \
"ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6ImEiLCJleHAiOiIyMDc1M" \
"jk3MTQ4In0.1NIuxcD1FsZlU17NxK4UHdCfzl7qTV03qEaTRcqTC6A1Fs2Alc7mSQgkF_SpUw4Ylt" \
"n-7DhO2InfcwDA0VhxBOHDL6ZzcEvzw-49iD-AaSd4aINIkDK-Iim5uzbKzgQCuZqSXFqxsZlezA4" \
"BtwV7Lv2puqdPrXT8k3SvM2rOwRw"
self.assert_in_decode_signed_jwt_exception(
valid_token_with_invalid_iat, "Claim iat is not an integer")
def test_payload_expired_exp(self):
valid_token_with_exp_in_the_past = "eyJraWQiOiI3MDllYjQyY2ZlZTU1NzAwNThjZTA3MTFmNzMwYmZiYjdkNGM4YWRlIiwidHlwI" \
"joiand0IiwiYWxnIjoiUlMyNTYifQ.eyJpYXQiOjE0OTg2NTQzOTcuMDgxOTE1LCJlcV9pZCI" \
"6IjEiLCJmb3JtX3R5cGUiOiIwMjA1IiwiZXhwIjoxNDk4NjU0Mzk2LjA4MTkxNSwianRpIjoi" \
"NzZlNjllYTAtZWRlYi00NGY5LThkYWEtY2Q1ZDQzNzg5YmM1In0.CKWYyIcDbZaUXvdDno2B3" \
"0w599_VXqicKkVjoeF4kNxc8aUcc_6J-rxTI8OU0OEoy8ywUTMBwYQnCHAuleBUYcmE9oNaHA" \
"HHbfvTRVDpi1rIFc3vnoy37hx7v-iRElNJ_CNrGw5aURZ_eFarH2EiSNf7tdIy8H1xn0GnHMB" \
"3-fmFylj9wvNR4td5MteAAeZlvsRf4uPj2GCm44re-n4iRY9z3ocZcKvUYVIJFOEK3XUerUdy" \
"zZBGqbf-uIPB615nJgZF0PPS6e85VzrmyLD54fqrDrSnklKhu4dfMf_YdbegWvi7lUv7z_QIH" \
"PRlUgxPsWKmV2G1SeVKRqbx1n_raA"
self.assert_in_decode_signed_jwt_exception(
valid_token_with_exp_in_the_past, "Expired at")
def test_payload_exp_less_than_iat(self):
valid_token_with_exp_less_than_iat = "eyJraWQiOiI3MDllYjQyY2ZlZTU1NzAwNThjZTA3MTFmNzMwYmZiYjdkNGM4YWRlIiwiYW" \
"xnIjoiUlMyNTYiLCJ0eXAiOiJqd3QifQ.eyJmb3JtX3R5cGUiOiIwMjA1IiwiaWF0IjoxNDk" \
"4NjU0MjEzLjk5NjQ2MywianRpIjoiNWFkODdjMGQtZjZlOC00MDEyLWEyM2UtMjc4MzY4YjF" \
"kZmFmIiwiZXFfaWQiOiIxIiwiZXhwIjoxNDk4NjUwNjEzLjk5NjQ2M30.kAAO0uZG02sTJpQ" \
"DzUFkIU7UGR9ulJV6idZJsWkJcsIu4G1JHfCoyNCzJr9xT8RRPbUrgkdVkuLD0gzOnD0Ylqj" \
"xKxpoRTVUtD4p2l-5FuXcqIpy6jtQWsx1YGvMfdCRwsvpVVAUiFAhSddC0QRHvqweet7WgMq" \
"SAvNz6zkOTVvW5ChjrK3IaGOAl3T6jWFN1xJCHcdlMef6S8t3ECP5NaP5HRnRxiVmV63x_RR" \
"uSBwLbz_IMHUPPe6JcMRTMnzL8qM2Kwg227mHlmQhn3OMjagzraZZeQ4aedghalYoItZE80d" \
"AcfDWs8DPJPqhJ0JGdA08A7ningHV67LRm6zkYw"
self.assert_in_decode_signed_jwt_exception(
valid_token_with_exp_less_than_iat, "Expired at")
def test_payload_contains_more_than_one_iat(self):
payload = base64.urlsafe_b64encode(b'{"user":"******",'
b'"iat": "1454935765",'
b'"iat": "1454935765",'
b'"exp": "2075297148"}')
jwt = jwtio_header + "." + payload.decode() + "." + jwtio_signature
self.assert_in_decode_signed_jwt_exception(jwt, "InvalidSignature")
def test_payload_contains_more_than_one_exp(self):
payload = base64.urlsafe_b64encode(b'{"user":"******",'
b'"iat": "1454935765",'
b'"exp": "1454935765",'
b'"exp": "2075297148"}')
jwt = jwtio_header + "." + payload.decode() + "." + jwtio_signature
self.assert_in_decode_signed_jwt_exception(jwt, "InvalidSignature")
def assert_in_decode_signed_jwt_exception(self, jwe, error):
with pytest.raises(InvalidTokenException) as ite:
JWTHelper.decode(jwe,
self.key_store,
purpose=KEY_PURPOSE_AUTHENTICATION,
check_claims=self.CHECK_CLAIMS)
# Looks weird, but ite.value is an exception object. The error message is contained in the 'value' attribute
# of that object.
if error not in ite.value.value:
raise AssertionError(
'"{}" not found in decode exception. Actual exception message [{}]'
.format(error, ite.value.value))
def test_encode_with_dict_and_string(self):
claims_as_dict = {
'data': [{
'string': 'something',
'boolean': True,
'number': 10,
'decimal': 10.1,
'null': None
}]
}
claims_as_string = json.dumps(claims_as_dict)
string_token = JWTHelper.encode(claims=claims_as_string,
kid='EQ_USER_AUTHENTICATION_EQ_KEY',
key_store=self.key_store,
purpose=KEY_PURPOSE_AUTHENTICATION)
dict_token = JWTHelper.encode(claims=claims_as_dict,
kid='EQ_USER_AUTHENTICATION_EQ_KEY',
key_store=self.key_store,
purpose=KEY_PURPOSE_AUTHENTICATION)
string_token_decode = JWTHelper.decode(
jwt_token=string_token,
key_store=self.key_store_secondary,
purpose=KEY_PURPOSE_AUTHENTICATION)
dict_token_decode = JWTHelper.decode(
jwt_token=dict_token,
key_store=self.key_store_secondary,
purpose=KEY_PURPOSE_AUTHENTICATION)
assert string_token_decode == dict_token_decode
def create_app(setting_overrides=None): # noqa: C901 pylint: disable=too-complex
application = Flask(__name__,
static_url_path='/s',
static_folder='../static')
application.config.from_object(settings)
application.eq = {}
with open(application.config['EQ_SECRETS_FILE']) as secrets_file:
secrets = yaml.safe_load(secrets_file)
with open(application.config['EQ_KEYS_FILE']) as keys_file:
keys = yaml.safe_load(keys_file)
validate_required_secrets(secrets)
validate_required_keys(keys, KEY_PURPOSE_SUBMISSION)
application.eq['secret_store'] = SecretStore(secrets)
application.eq['key_store'] = KeyStore(keys)
if setting_overrides:
application.config.update(setting_overrides)
if application.config['EQ_APPLICATION_VERSION']:
logger.info('starting eq survey runner',
version=application.config['EQ_APPLICATION_VERSION'])
if application.config['EQ_NEW_RELIC_ENABLED']:
setup_newrelic()
setup_database(application)
setup_dynamodb(application)
if application.config['EQ_RABBITMQ_ENABLED']:
application.eq['submitter'] = RabbitMQSubmitter(
host=application.config['EQ_RABBITMQ_HOST'],
secondary_host=application.config['EQ_RABBITMQ_HOST_SECONDARY'],
port=application.config['EQ_RABBITMQ_PORT'],
username=application.eq['secret_store'].get_secret_by_name(
'EQ_RABBITMQ_USERNAME'),
password=application.eq['secret_store'].get_secret_by_name(
'EQ_RABBITMQ_PASSWORD'),
)
else:
application.eq['submitter'] = LogSubmitter()
application.eq['id_generator'] = UserIDGenerator(
application.config['EQ_SERVER_SIDE_STORAGE_USER_ID_ITERATIONS'],
application.eq['secret_store'].get_secret_by_name(
'EQ_SERVER_SIDE_STORAGE_USER_ID_SALT'),
application.eq['secret_store'].get_secret_by_name(
'EQ_SERVER_SIDE_STORAGE_USER_IK_SALT'),
)
setup_secure_cookies(application)
setup_secure_headers(application)
setup_babel(application)
application.wsgi_app = AWSReverseProxied(application.wsgi_app)
add_blueprints(application)
configure_flask_logging(application)
login_manager.init_app(application)
add_safe_health_check(application)
if application.config['EQ_DEV_MODE']:
start_dev_mode(application)
if application.config['EQ_ENABLE_CACHE']:
cache.init_app(application, config={'CACHE_TYPE': 'simple'})
else:
# no cache and silence warning
cache.init_app(application, config={'CACHE_NO_NULL_WARNING': True})
# Switch off flask default autoescaping as content is html encoded
# during schema/metadata/summary context (and navigition) generation
application.jinja_env.autoescape = False
# Add theme manager
application.config['THEME_PATHS'] = os.path.dirname(
os.path.abspath(__file__))
Themes(application, app_identifier='surveyrunner')
@application.before_request
def before_request(): # pylint: disable=unused-variable
# While True the session lives for permanent_session_lifetime seconds
# Needed to be able to set the client-side cookie expiration
cookie_session.permanent = True
request_id = str(uuid4())
logger.new(request_id=request_id)
@application.after_request
def apply_caching(response): # pylint: disable=unused-variable
for k, v in CACHE_HEADERS.items():
response.headers[k] = v
return response
@application.context_processor
def override_url_for(): # pylint: disable=unused-variable
return dict(url_for=versioned_url_for)
return application
def create_app(setting_overrides=None): # noqa: C901 pylint: disable=too-complex,too-many-statements
application = Flask(__name__, static_url_path='/s', static_folder='../static')
application.config.from_object(settings)
application.eq = {}
with open(application.config['EQ_SECRETS_FILE']) as secrets_file:
secrets = yaml.safe_load(secrets_file)
with open(application.config['EQ_KEYS_FILE']) as keys_file:
keys = yaml.safe_load(keys_file)
validate_required_secrets(secrets)
validate_required_keys(keys, KEY_PURPOSE_SUBMISSION)
application.eq['secret_store'] = SecretStore(secrets)
application.eq['key_store'] = KeyStore(keys)
if setting_overrides:
application.config.update(setting_overrides)
if application.config['EQ_APPLICATION_VERSION']:
logger.info('starting eq survey runner', version=application.config['EQ_APPLICATION_VERSION'])
if application.config['EQ_NEW_RELIC_ENABLED']:
setup_newrelic()
setup_database(application)
setup_dynamodb(application)
setup_s3(application)
setup_bigtable(application)
setup_gcs(application)
setup_redis(application)
setup_gc_datastore(application)
if application.config['EQ_SUBMITTER'] == 'rabbitmq':
application.eq['submitter'] = RabbitMQSubmitter(
host=application.config['EQ_RABBITMQ_HOST'],
secondary_host=application.config['EQ_RABBITMQ_HOST_SECONDARY'],
port=application.config['EQ_RABBITMQ_PORT'],
username=application.eq['secret_store'].get_secret_by_name('EQ_RABBITMQ_USERNAME'),
password=application.eq['secret_store'].get_secret_by_name('EQ_RABBITMQ_PASSWORD'),
)
elif application.config['EQ_SUBMITTER'] == 'pubsub':
application.eq['submitter'] = PubSubSubmitter(
project_id=application.config['EQ_PUBSUB_PROJECT_ID'],
topic=application.config['EQ_PUBSUB_TOPIC'],
)
elif application.config['EQ_SUBMITTER'] == 'gcs':
application.eq['submitter'] = GCSSubmitter(
bucket_name=application.config['EQ_GCS_SUBMISSION_BUCKET_ID'],
)
else:
application.eq['submitter'] = LogSubmitter()
application.eq['id_generator'] = UserIDGenerator(
application.config['EQ_SERVER_SIDE_STORAGE_USER_ID_ITERATIONS'],
application.eq['secret_store'].get_secret_by_name('EQ_SERVER_SIDE_STORAGE_USER_ID_SALT'),
application.eq['secret_store'].get_secret_by_name('EQ_SERVER_SIDE_STORAGE_USER_IK_SALT'),
)
setup_secure_cookies(application)
setup_secure_headers(application)
setup_babel(application)
application.wsgi_app = AWSReverseProxied(application.wsgi_app)
add_blueprints(application)
configure_flask_logging(application)
login_manager.init_app(application)
add_safe_health_check(application)
if application.config['EQ_DEV_MODE']:
start_dev_mode(application)
# Switch off flask default autoescaping as content is html encoded
# during schema/metadata/summary context (and navigition) generation
application.jinja_env.autoescape = False
# Add theme manager
application.config['THEME_PATHS'] = os.path.dirname(os.path.abspath(__file__))
Themes(application, app_identifier='surveyrunner')
# pylint: disable=maybe-no-member
application.jinja_env.globals['theme'] = flask_theme_cache.get_global_theme_template()
@application.before_request
def before_request(): # pylint: disable=unused-variable
request_id = str(uuid4())
logger.new(request_id=request_id)
@application.after_request
def apply_caching(response): # pylint: disable=unused-variable
for k, v in CACHE_HEADERS.items():
response.headers[k] = v
return response
@application.context_processor
def override_url_for(): # pylint: disable=unused-variable
return dict(url_for=versioned_url_for)
return application
def _set_up_app(self):
self._ds = patch("app.setup.datastore.Client", MockDatastore)
self._ds.start()
self._redis = patch("app.setup.redis.Redis", fakeredis.FakeStrictRedis)
self._redis.start()
from application import ( # pylint: disable=import-outside-toplevel
configure_logging, )
configure_logging()
setting_overrides = {
"EQ_ENABLE_HTML_MINIFY": False,
"EQ_SUBMISSION_CONFIRMATION_BACKEND": "log",
}
with patch(
"google.auth._default._get_explicit_environ_credentials",
return_value=(Mock(), "test-project-id"),
):
self._application = create_app(setting_overrides)
self._key_store = KeyStore({
"keys": {
EQ_USER_AUTHENTICATION_RRM_PRIVATE_KEY_KID: {
"purpose":
KEY_PURPOSE_AUTHENTICATION,
"type":
"private",
"value":
get_file_contents(
"sdc-rrm-authentication-signing-private-v1.pem"),
},
SR_USER_AUTHENTICATION_PUBLIC_KEY_KID: {
"purpose":
KEY_PURPOSE_AUTHENTICATION,
"type":
"public",
"value":
get_file_contents(
"sdc-sr-authentication-encryption-public-v1.pem"),
},
EQ_SUBMISSION_SDX_PRIVATE_KEY: {
"purpose":
KEY_PURPOSE_SUBMISSION,
"type":
"private",
"value":
get_file_contents(
"sdc-sdx-submission-encryption-private-v1.pem"),
},
EQ_SUBMISSION_SR_PRIVATE_SIGNING_KEY: {
"purpose":
KEY_PURPOSE_SUBMISSION,
"type":
"public",
"value":
get_file_contents(
"sdc-sr-submission-signing-private-v1.pem"),
},
}
})
self.token_generator = TokenGenerator(
self._key_store,
EQ_USER_AUTHENTICATION_RRM_PRIVATE_KEY_KID,
SR_USER_AUTHENTICATION_PUBLIC_KEY_KID,
)
self._client = self._application.test_client()
self.session = self._client.session_transaction()
def __init__(self, json_secret_keys):
keys = json.loads(json_secret_keys)
validate_required_keys(keys, KEY_PURPOSE)
self.keystore = KeyStore(keys)
class TestKeyStore:
key_store = KeyStore({
"keys": {
"e19091072f920cbf3ca9f436ceba309e7d814a62": {'purpose': KEY_PURPOSE_AUTHENTICATION,
'type': 'private',
'value': TEST_DO_NOT_USE_SR_PRIVATE_PEM},
"EQ_USER_AUTHENTICATION_SR_PRIVATE_KEY": {'purpose': KEY_PURPOSE_AUTHENTICATION,
'type': 'private',
'value': TEST_DO_NOT_USE_SR_PRIVATE_PEM},
"EDCRRM": {'purpose': KEY_PURPOSE_AUTHENTICATION,
'type': 'public',
'value': TEST_DO_NOT_USE_PUBLIC_KEY},
"709eb42cfee5570058ce0711f730bfbb7d4c8ade": {'purpose': KEY_PURPOSE_AUTHENTICATION,
'type': 'public',
'value': TEST_DO_NOT_USE_UPSTREAM_PUBLIC_PEM},
}
})
def test_get_key_by_kid_with_incorrect_type(self):
with pytest.raises(InvalidTokenException):
self.key_store.get_key_by_kid(KEY_PURPOSE_AUTHENTICATION, "e19091072f920cbf3ca9f436ceba309e7d814a62", "public")
def test_get_key_by_kid_with_incorrect_purpose(self):
with pytest.raises(InvalidTokenException):
self.key_store.get_key_by_kid(KEY_PURPOSE_SUBMISSION, "e19091072f920cbf3ca9f436ceba309e7d814a62", "private")
def test_get_key_for_purpose_and_type(self):
"""
Test that we get a key if there is one in the store that matches the criteria
Note that if there are many, you'll get a random one.
"""
result = self.key_store.get_key_for_purpose_and_type(KEY_PURPOSE_AUTHENTICATION, "private")
assert result.kid in ["e19091072f920cbf3ca9f436ceba309e7d814a62", "EQ_USER_AUTHENTICATION_SR_PRIVATE_KEY"]
assert result.purpose == KEY_PURPOSE_AUTHENTICATION
assert result.key_type == "private"
assert result.value == TEST_DO_NOT_USE_SR_PRIVATE_PEM
def test_get_key_for_purpose_and_type_not_found(self):
"""Test that None is returned if no keys in the store have both the specified key_type and purpose"""
result = self.key_store.get_key_for_purpose_and_type(KEY_PURPOSE_SUBMISSION, "private")
assert result is None
@staticmethod
def test_incomplete_key():
"""Tests that an exception is thrown a malformed key is created with the keystore"""
with pytest.raises(CryptoError):
KeyStore({
"keys": {
"e19091072f920cbf3ca9f436ceba309e7d814a62": {'purpose': KEY_PURPOSE_AUTHENTICATION,
'value': TEST_DO_NOT_USE_SR_PRIVATE_PEM},
}
})
@staticmethod
def test_validate_required_keys_missing_public_keys_for_purpose():
"""Tests exeception is raised if there are no public keys with 'authentication' purpose"""
with pytest.raises(CryptoError):
keystore_dict = {
"keys": {
"insert_kid_here": {'purpose': KEY_PURPOSE_AUTHENTICATION,
'type': 'private',
'value': TEST_DO_NOT_USE_SR_PRIVATE_PEM},
}
}
validate_required_keys(keystore_dict, KEY_PURPOSE_AUTHENTICATION)
@staticmethod
def test_validate_required_keys_missing_private_keys_for_purpose():
"""Tests exeception is raised if there are no private keys with 'authentication' purpose"""
with pytest.raises(CryptoError):
keystore_dict = {
"keys": {
"insert_kid_here": {'purpose': KEY_PURPOSE_AUTHENTICATION,
'type': 'public',
'value': TEST_DO_NOT_USE_SR_PRIVATE_PEM},
}
}
validate_required_keys(keystore_dict, KEY_PURPOSE_AUTHENTICATION)