def jwt_login(self, request):
"""
Login using a JWT token, this must be an encrypted JWT.
:param request: The flask request
"""
# clear the session entry in the database
session_manager.clear()
# also clear the secure cookie data
session.clear()
if request.args.get(EQ_URL_QUERY_STRING_JWT_FIELD_NAME) is None:
raise NoTokenException("Please provide a token")
token = self._jwt_decrypt(request)
# once we've decrypted the token correct
# check we have the required user data
self._check_user_data(token)
# get the hashed user id for eq
user_id = UserIDGenerator.generate_id(token)
user_ik = UserIDGenerator.generate_ik(token)
# store the user id in the session
session_manager.store_user_id(user_id)
# store the user ik in the cookie
session_manager.store_user_ik(user_ik)
# store the meta data
metadata = parse_metadata(token)
questionnaire_store = get_questionnaire_store(user_id, user_ik)
questionnaire_store.metadata = metadata
questionnaire_store.save()
logger.info("User authenticated with tx_id=%s", metadata["tx_id"])
def jwt_login(request):
"""
Login using a JWT token, this must be an encrypted JWT.
:param request: The flask request
"""
# clear the session entry in the database
session_storage.clear()
# also clear the secure cookie data
session.clear()
if request.args.get('token') is None:
raise NoTokenException("Please provide a token")
token = _jwt_decrypt(request)
# once we've decrypted the token correct
# check we have the required user data
_check_user_data(token)
# get the hashed user id for eq
user_id = UserIDGenerator.generate_id(token)
user_ik = UserIDGenerator.generate_ik(token)
# store the user id in the session
session_storage.store_user_id(user_id)
# store the user ik in the cookie
session_storage.store_user_ik(user_ik)
# store the meta data
metadata = parse_metadata(token)
logger.bind(tx_id=metadata["tx_id"])
questionnaire_store = get_questionnaire_store(user_id, user_ik)
questionnaire_store.metadata = metadata
questionnaire_store.add_or_update()
logger.info("user authenticated")
def test_generate_id():
id_generator = UserIDGenerator(ITERATIONS, "", "")
user_id_1 = id_generator.generate_id("1234567890123456")
user_id_2 = id_generator.generate_id("1234567890123456")
user_id_3 = id_generator.generate_id("0000000000000000")
assert user_id_1 == user_id_2
assert user_id_1 != user_id_3
def test_generate_ik(self):
id_generator = UserIDGenerator(self._iterations, "", "")
user_ik_1 = id_generator.generate_ik("1234567890123456")
user_ik_2 = id_generator.generate_ik("1234567890123456")
user_ik_3 = id_generator.generate_ik("1111111111111111")
self.assertEqual(user_ik_1, user_ik_2)
self.assertNotEqual(user_ik_1, user_ik_3)
def test_generate_ik():
id_generator = UserIDGenerator(ITERATIONS, "", "")
user_ik_1 = id_generator.generate_ik("1234567890123456")
user_ik_2 = id_generator.generate_ik("1234567890123456")
user_ik_3 = id_generator.generate_ik("1111111111111111")
assert user_ik_1 == user_ik_2
assert user_ik_1 != user_ik_3
def test_generate_ik_throws_invalid_token_exception(self):
with self.assertRaises(InvalidTokenException):
UserIDGenerator.generate_ik(self.create_token('1', '2', None, '4'))
with self.assertRaises(InvalidTokenException):
UserIDGenerator.generate_ik(self.create_token('1', None, '3', '4'))
with self.assertRaises(InvalidTokenException):
UserIDGenerator.generate_ik(self.create_token(None, '2', '3', '4'))
with self.assertRaises(InvalidTokenException):
UserIDGenerator.generate_ik(self.create_token(None, None, None, '4'))
with self.assertRaises(InvalidTokenException):
UserIDGenerator.generate_ik(self.create_token(None, None, None, None))
def test_generate_ik_throws_invalid_token_exception(self):
with self.assertRaises(InvalidTokenException) as ite:
UserIDGenerator.generate_ik(self.create_token('1', '2', None, '4'))
with self.assertRaises(InvalidTokenException) as ite:
UserIDGenerator.generate_ik(self.create_token('1', None, '3', '4'))
with self.assertRaises(InvalidTokenException) as ite:
UserIDGenerator.generate_ik(self.create_token(None, '2', '3', '4'))
with self.assertRaises(InvalidTokenException) as ite:
UserIDGenerator.generate_ik(self.create_token(None, None, None, '4'))
with self.assertRaises(InvalidTokenException) as ite:
UserIDGenerator.generate_ik(self.create_token(None, None, None, None))
def test_different_salt_creates_different_user_ik():
id_generator_1 = UserIDGenerator(ITERATIONS, "", "")
user_ik_1 = id_generator_1.generate_ik("1234567890123456")
id_generator_2 = UserIDGenerator(ITERATIONS, "", "random")
user_ik_2 = id_generator_2.generate_ik("1234567890123456")
assert user_ik_1 != user_ik_2
def test_different_salt_creates_different_user_ik(self):
id_generator_1 = UserIDGenerator(self._iterations, '', '')
user_ik_1 = id_generator_1.generate_ik(self.create_token('1', '2', '3', '4'))
id_generator_2 = UserIDGenerator(self._iterations, '', 'random')
user_ik_2 = id_generator_2.generate_ik(self.create_token('1', '2', '3', '4'))
self.assertNotEqual(user_ik_1, user_ik_2)
def test_different_salt_creates_different_user_ik(self):
id_generator_1 = UserIDGenerator(self._iterations, "", "")
user_ik_1 = id_generator_1.generate_ik("1234567890123456")
id_generator_2 = UserIDGenerator(self._iterations, "", "random")
user_ik_2 = id_generator_2.generate_ik("1234567890123456")
self.assertNotEqual(user_ik_1, user_ik_2)
def store_session(metadata):
"""
Store new session and metadata
:param metadata: metadata parsed from jwt token
"""
# clear the session entry in the database
current_app.eq['session_storage'].delete_session_from_db()
# also clear the secure cookie data
session.clear()
# get the hashed user id for eq
user_id = UserIDGenerator.generate_id(metadata)
user_ik = UserIDGenerator.generate_ik(metadata)
# store the user id in the session
current_app.eq['session_storage'].store_user_id(user_id)
# store the user ik in the cookie
current_app.eq['session_storage'].store_user_ik(user_ik)
questionnaire_store = get_questionnaire_store(user_id, user_ik)
questionnaire_store.metadata = metadata
questionnaire_store.add_or_update()
logger.info("user authenticated")
def test_generate_ik(self):
user_ik_1 = UserIDGenerator.generate_ik(self.create_token('1', '2', '3', '4'))
user_ik_2 = UserIDGenerator.generate_ik(self.create_token('1', '2', '3', '4'))
user_ik_3 = UserIDGenerator.generate_ik(self.create_token('1', '2', '4', '4'))
user_ik_4 = UserIDGenerator.generate_ik(self.create_token('2', '2', '3', '4'))
user_ik_5 = UserIDGenerator.generate_ik(self.create_token('1', '1', '3', '4'))
user_ik_6 = UserIDGenerator.generate_ik(self.create_token('2', '2', '4', '4'))
user_ik_7 = UserIDGenerator.generate_ik(self.create_token('2', '2', '4', '5'))
user_ik_8 = UserIDGenerator.generate_ik(self.create_token('1', '2', '3', '5'))
self.assertEqual(user_ik_1, user_ik_2)
self.assertNotEquals(user_ik_1, user_ik_3)
self.assertNotEquals(user_ik_1, user_ik_3)
self.assertNotEquals(user_ik_1, user_ik_4)
self.assertNotEquals(user_ik_1, user_ik_5)
self.assertNotEquals(user_ik_1, user_ik_6)
self.assertNotEquals(user_ik_1, user_ik_7)
self.assertNotEquals(user_ik_1, user_ik_8)
def test_generate_ik(self):
user_ik_1 = UserIDGenerator.generate_ik(self.create_token('1', '2', '3', '4'))
user_ik_2 = UserIDGenerator.generate_ik(self.create_token('1', '2', '3', '4'))
user_ik_3 = UserIDGenerator.generate_ik(self.create_token('1', '2', '4', '4'))
user_ik_4 = UserIDGenerator.generate_ik(self.create_token('2', '2', '3', '4'))
user_ik_5 = UserIDGenerator.generate_ik(self.create_token('1', '1', '3', '4'))
user_ik_6 = UserIDGenerator.generate_ik(self.create_token('2', '2', '4', '4'))
user_ik_7 = UserIDGenerator.generate_ik(self.create_token('2', '2', '4', '5'))
user_ik_8 = UserIDGenerator.generate_ik(self.create_token('1', '2', '3', '5'))
self.assertEqual(user_ik_1, user_ik_2)
self.assertNotEqual(user_ik_1, user_ik_3)
self.assertNotEqual(user_ik_1, user_ik_3)
self.assertNotEqual(user_ik_1, user_ik_4)
self.assertNotEqual(user_ik_1, user_ik_5)
self.assertNotEqual(user_ik_1, user_ik_6)
self.assertNotEqual(user_ik_1, user_ik_7)
self.assertNotEqual(user_ik_1, user_ik_8)
def test_generate_ik(self):
id_generator = UserIDGenerator(self._iterations, '', '')
user_ik_1 = id_generator.generate_ik(self.create_token('1', '2', '3', '4'))
user_ik_2 = id_generator.generate_ik(self.create_token('1', '2', '3', '4'))
user_ik_3 = id_generator.generate_ik(self.create_token('1', '2', '4', '4'))
user_ik_4 = id_generator.generate_ik(self.create_token('2', '2', '3', '4'))
user_ik_5 = id_generator.generate_ik(self.create_token('1', '1', '3', '4'))
user_ik_6 = id_generator.generate_ik(self.create_token('2', '2', '4', '4'))
user_ik_7 = id_generator.generate_ik(self.create_token('2', '2', '4', '5'))
user_ik_8 = id_generator.generate_ik(self.create_token('1', '2', '3', '5'))
self.assertEqual(user_ik_1, user_ik_2)
self.assertNotEqual(user_ik_1, user_ik_3)
self.assertNotEqual(user_ik_1, user_ik_3)
self.assertNotEqual(user_ik_1, user_ik_4)
self.assertNotEqual(user_ik_1, user_ik_5)
self.assertNotEqual(user_ik_1, user_ik_6)
self.assertNotEqual(user_ik_1, user_ik_7)
self.assertNotEqual(user_ik_1, user_ik_8)
def create_app( # noqa: C901 pylint: disable=too-complex, too-many-statements
setting_overrides=None,
):
application = Flask(__name__, template_folder="../templates")
application.config.from_object(settings)
if setting_overrides:
application.config.update(setting_overrides)
application.eq = {}
with open(application.config["EQ_SECRETS_FILE"]) as secrets_file:
secrets = yaml.safe_load(secrets_file)
conditional_required_secrets = []
if application.config["ADDRESS_LOOKUP_API_AUTH_ENABLED"]:
conditional_required_secrets.append("ADDRESS_LOOKUP_API_AUTH_TOKEN_SECRET")
validate_required_secrets(secrets, conditional_required_secrets)
application.eq["secret_store"] = SecretStore(secrets)
with open(application.config["EQ_KEYS_FILE"]) as keys_file:
keys = yaml.safe_load(keys_file)
validate_required_keys(keys, KEY_PURPOSE_SUBMISSION)
application.eq["key_store"] = KeyStore(keys)
if application.config["EQ_APPLICATION_VERSION"]:
logger.info(
"starting eq survey runner",
version=application.config["EQ_APPLICATION_VERSION"],
)
# IMPORTANT: This must be initialised *before* any other Flask plugins that add
# before_request hooks. Otherwise any logging by the plugin in their before
# request will use the logger context of the previous request.
@application.before_request
def before_request(): # pylint: disable=unused-variable
request_id = str(uuid4())
logger.new(request_id=request_id)
span, trace = get_span_and_trace(flask_request.headers)
if span and trace:
logger.bind(span=span, trace=trace)
logger.info(
"request",
method=flask_request.method,
url_path=flask_request.full_path,
session_cookie_present="session" in flask_request.cookies,
csrf_token_present="csrf_token" in cookie_session,
user_agent=flask_request.user_agent.string,
)
setup_storage(application)
setup_submitter(application)
setup_feedback(application)
setup_publisher(application)
setup_task_client(application)
application.eq["id_generator"] = UserIDGenerator(
application.config["EQ_SERVER_SIDE_STORAGE_USER_ID_ITERATIONS"],
application.eq["secret_store"].get_secret_by_name(
"EQ_SERVER_SIDE_STORAGE_USER_ID_SALT"
),
application.eq["secret_store"].get_secret_by_name(
"EQ_SERVER_SIDE_STORAGE_USER_IK_SALT"
),
)
cache_questionnaire_schemas()
setup_secure_cookies(application)
setup_secure_headers(application)
setup_babel(application)
application.wsgi_app = AWSReverseProxied(application.wsgi_app)
application.url_map.strict_slashes = False
add_blueprints(application)
login_manager.init_app(application)
add_safe_health_check(application)
setup_compression(application)
setup_jinja_env(application)
@application.after_request
def apply_caching(response): # pylint: disable=unused-variable
if "text/html" in response.content_type:
for k, v in CACHE_HEADERS.items():
response.headers[k] = v
else:
response.headers["Cache-Control"] = "max-age=2628000, public"
return response
@application.after_request
def response_minify(response): # pylint: disable=unused-variable
"""
minify html response to decrease site traffic
"""
if (
application.config["EQ_ENABLE_HTML_MINIFY"]
and response.content_type == "text/html; charset=utf-8"
):
response.set_data(
minify(
response.get_data(as_text=True),
remove_comments=True,
remove_empty_space=True,
remove_optional_attribute_quotes=False,
)
)
return response
return response
@application.after_request
def after_request(response): # pylint: disable=unused-variable
# We're using the stringified version of the Flask session to get a rough
# length for the cookie. The real length won't be known yet as Flask
# serializes and adds the cookie header after this method is called.
logger.info(
"response",
status_code=response.status_code,
session_modified=cookie_session.modified,
)
return response
return application
def _get_user(decrypted_token):
user_id = UserIDGenerator.generate_id(decrypted_token)
user_ik = UserIDGenerator.generate_ik(decrypted_token)
return User(user_id, user_ik)
def create_app(setting_overrides=None): # noqa: C901 pylint: disable=too-complex
application = Flask(__name__,
static_url_path='/s',
static_folder='../static')
application.config.from_object(settings)
application.eq = {}
with open(application.config['EQ_SECRETS_FILE']) as secrets_file:
secrets = yaml.safe_load(secrets_file)
with open(application.config['EQ_KEYS_FILE']) as keys_file:
keys = yaml.safe_load(keys_file)
validate_required_secrets(secrets)
validate_required_keys(keys, KEY_PURPOSE_SUBMISSION)
application.eq['secret_store'] = SecretStore(secrets)
application.eq['key_store'] = KeyStore(keys)
if setting_overrides:
application.config.update(setting_overrides)
if application.config['EQ_APPLICATION_VERSION']:
logger.info('starting eq survey runner',
version=application.config['EQ_APPLICATION_VERSION'])
if application.config['EQ_NEW_RELIC_ENABLED']:
setup_newrelic()
setup_database(application)
setup_dynamodb(application)
if application.config['EQ_RABBITMQ_ENABLED']:
application.eq['submitter'] = RabbitMQSubmitter(
host=application.config['EQ_RABBITMQ_HOST'],
secondary_host=application.config['EQ_RABBITMQ_HOST_SECONDARY'],
port=application.config['EQ_RABBITMQ_PORT'],
username=application.eq['secret_store'].get_secret_by_name(
'EQ_RABBITMQ_USERNAME'),
password=application.eq['secret_store'].get_secret_by_name(
'EQ_RABBITMQ_PASSWORD'),
)
else:
application.eq['submitter'] = LogSubmitter()
application.eq['id_generator'] = UserIDGenerator(
application.config['EQ_SERVER_SIDE_STORAGE_USER_ID_ITERATIONS'],
application.eq['secret_store'].get_secret_by_name(
'EQ_SERVER_SIDE_STORAGE_USER_ID_SALT'),
application.eq['secret_store'].get_secret_by_name(
'EQ_SERVER_SIDE_STORAGE_USER_IK_SALT'),
)
setup_secure_cookies(application)
setup_secure_headers(application)
setup_babel(application)
application.wsgi_app = AWSReverseProxied(application.wsgi_app)
add_blueprints(application)
configure_flask_logging(application)
login_manager.init_app(application)
add_safe_health_check(application)
if application.config['EQ_DEV_MODE']:
start_dev_mode(application)
if application.config['EQ_ENABLE_CACHE']:
cache.init_app(application, config={'CACHE_TYPE': 'simple'})
else:
# no cache and silence warning
cache.init_app(application, config={'CACHE_NO_NULL_WARNING': True})
# Switch off flask default autoescaping as content is html encoded
# during schema/metadata/summary context (and navigition) generation
application.jinja_env.autoescape = False
# Add theme manager
application.config['THEME_PATHS'] = os.path.dirname(
os.path.abspath(__file__))
Themes(application, app_identifier='surveyrunner')
@application.before_request
def before_request(): # pylint: disable=unused-variable
# While True the session lives for permanent_session_lifetime seconds
# Needed to be able to set the client-side cookie expiration
cookie_session.permanent = True
request_id = str(uuid4())
logger.new(request_id=request_id)
@application.after_request
def apply_caching(response): # pylint: disable=unused-variable
for k, v in CACHE_HEADERS.items():
response.headers[k] = v
return response
@application.context_processor
def override_url_for(): # pylint: disable=unused-variable
return dict(url_for=versioned_url_for)
return application
def create_app(setting_overrides=None): # noqa: C901 pylint: disable=too-complex,too-many-statements
application = Flask(__name__, static_url_path='/s', static_folder='../static')
application.config.from_object(settings)
application.eq = {}
with open(application.config['EQ_SECRETS_FILE']) as secrets_file:
secrets = yaml.safe_load(secrets_file)
with open(application.config['EQ_KEYS_FILE']) as keys_file:
keys = yaml.safe_load(keys_file)
validate_required_secrets(secrets)
validate_required_keys(keys, KEY_PURPOSE_SUBMISSION)
application.eq['secret_store'] = SecretStore(secrets)
application.eq['key_store'] = KeyStore(keys)
if setting_overrides:
application.config.update(setting_overrides)
if application.config['EQ_APPLICATION_VERSION']:
logger.info('starting eq survey runner', version=application.config['EQ_APPLICATION_VERSION'])
if application.config['EQ_NEW_RELIC_ENABLED']:
setup_newrelic()
setup_database(application)
setup_dynamodb(application)
setup_s3(application)
setup_bigtable(application)
setup_gcs(application)
setup_redis(application)
setup_gc_datastore(application)
if application.config['EQ_SUBMITTER'] == 'rabbitmq':
application.eq['submitter'] = RabbitMQSubmitter(
host=application.config['EQ_RABBITMQ_HOST'],
secondary_host=application.config['EQ_RABBITMQ_HOST_SECONDARY'],
port=application.config['EQ_RABBITMQ_PORT'],
username=application.eq['secret_store'].get_secret_by_name('EQ_RABBITMQ_USERNAME'),
password=application.eq['secret_store'].get_secret_by_name('EQ_RABBITMQ_PASSWORD'),
)
elif application.config['EQ_SUBMITTER'] == 'pubsub':
application.eq['submitter'] = PubSubSubmitter(
project_id=application.config['EQ_PUBSUB_PROJECT_ID'],
topic=application.config['EQ_PUBSUB_TOPIC'],
)
elif application.config['EQ_SUBMITTER'] == 'gcs':
application.eq['submitter'] = GCSSubmitter(
bucket_name=application.config['EQ_GCS_SUBMISSION_BUCKET_ID'],
)
else:
application.eq['submitter'] = LogSubmitter()
application.eq['id_generator'] = UserIDGenerator(
application.config['EQ_SERVER_SIDE_STORAGE_USER_ID_ITERATIONS'],
application.eq['secret_store'].get_secret_by_name('EQ_SERVER_SIDE_STORAGE_USER_ID_SALT'),
application.eq['secret_store'].get_secret_by_name('EQ_SERVER_SIDE_STORAGE_USER_IK_SALT'),
)
setup_secure_cookies(application)
setup_secure_headers(application)
setup_babel(application)
application.wsgi_app = AWSReverseProxied(application.wsgi_app)
add_blueprints(application)
configure_flask_logging(application)
login_manager.init_app(application)
add_safe_health_check(application)
if application.config['EQ_DEV_MODE']:
start_dev_mode(application)
# Switch off flask default autoescaping as content is html encoded
# during schema/metadata/summary context (and navigition) generation
application.jinja_env.autoescape = False
# Add theme manager
application.config['THEME_PATHS'] = os.path.dirname(os.path.abspath(__file__))
Themes(application, app_identifier='surveyrunner')
# pylint: disable=maybe-no-member
application.jinja_env.globals['theme'] = flask_theme_cache.get_global_theme_template()
@application.before_request
def before_request(): # pylint: disable=unused-variable
request_id = str(uuid4())
logger.new(request_id=request_id)
@application.after_request
def apply_caching(response): # pylint: disable=unused-variable
for k, v in CACHE_HEADERS.items():
response.headers[k] = v
return response
@application.context_processor
def override_url_for(): # pylint: disable=unused-variable
return dict(url_for=versioned_url_for)
return application
def test_create_generator_no_user_ik_salt_raises_error(self):
with self.assertRaises(ValueError):
UserIDGenerator(self._iterations, "", None)
def test_generate_id_no_metadata_raises_error(self):
id_generator = UserIDGenerator(self._iterations, '', '')
self.assertRaises(ValueError, id_generator.generate_id, None)
def test_generate_ik_no_token_raises_error(self):
id_generator = UserIDGenerator(self._iterations, '', '')
self.assertRaises(ValueError, id_generator.generate_ik, None)
def test_generate_id_throws_invalid_token_exception(self):
id_generator = UserIDGenerator(self._iterations, '', '')
with self.assertRaises(InvalidTokenException):
id_generator.generate_id(self.create_token('1', '2', None, '4'))
with self.assertRaises(InvalidTokenException):
id_generator.generate_id(self.create_token('1', None, '3', '4'))
with self.assertRaises(InvalidTokenException):
id_generator.generate_id(self.create_token(None, '2', '3', '4'))
with self.assertRaises(InvalidTokenException):
id_generator.generate_id(self.create_token(None, None, None, '4'))
with self.assertRaises(InvalidTokenException):
id_generator.generate_id(self.create_token(None, None, None, None))
def test_create_generator_no_user_ik_salt_raises_error():
with pytest.raises(ValueError):
UserIDGenerator(1000, "", None)
def test_different_salt_creates_different_useriks(self):
user_id_1 = UserIDGenerator.generate_ik(self.create_token('1', '2', '3', '4'))
settings.EQ_SERVER_SIDE_STORAGE_USER_IK_SALT = "random"
user_id_2 = UserIDGenerator.generate_ik(self.create_token('1', '2', '3', '4'))
self.assertNotEqual(user_id_1, user_id_2)
def test_different_salt_creates_different_useriks(self):
user_id_1 = UserIDGenerator.generate_ik(self.create_token('1', '2', '3', '4'))
settings.EQ_SERVER_SIDE_STORAGE_USER_IK_SALT = "random"
user_id_2 = UserIDGenerator.generate_ik(self.create_token('1', '2', '3', '4'))
self.assertNotEqual(user_id_1, user_id_2)