def test_validate_required_secrets_fails_on_missing(self):
secrets = {"secrets": {}}
with self.assertRaises(Exception) as exception:
validate_required_secrets(secrets)
self.assertIn(
"Missing Secret [EQ_SERVER_SIDE_STORAGE_USER_ID_SALT]",
str(exception.exception),
)
def test_validate_required_secrets_fails_on_conditional_secret(self):
secrets = {"secrets": {secret: "abc" for secret in REQUIRED_SECRETS}}
with self.assertRaises(Exception) as exception:
validate_required_secrets(
secrets, additional_required_secrets=["MY_REQUIRED_SECRET"])
self.assertIn(
"Missing Secret [MY_REQUIRED_SECRET]",
str(exception.exception),
)
def create_app( # noqa: C901 pylint: disable=too-complex, too-many-statements
setting_overrides=None,
):
application = Flask(__name__, template_folder="../templates")
application.config.from_object(settings)
if setting_overrides:
application.config.update(setting_overrides)
application.eq = {}
with open(application.config["EQ_SECRETS_FILE"]) as secrets_file:
secrets = yaml.safe_load(secrets_file)
conditional_required_secrets = []
if application.config["ADDRESS_LOOKUP_API_AUTH_ENABLED"]:
conditional_required_secrets.append("ADDRESS_LOOKUP_API_AUTH_TOKEN_SECRET")
validate_required_secrets(secrets, conditional_required_secrets)
application.eq["secret_store"] = SecretStore(secrets)
with open(application.config["EQ_KEYS_FILE"]) as keys_file:
keys = yaml.safe_load(keys_file)
validate_required_keys(keys, KEY_PURPOSE_SUBMISSION)
application.eq["key_store"] = KeyStore(keys)
if application.config["EQ_APPLICATION_VERSION"]:
logger.info(
"starting eq survey runner",
version=application.config["EQ_APPLICATION_VERSION"],
)
# IMPORTANT: This must be initialised *before* any other Flask plugins that add
# before_request hooks. Otherwise any logging by the plugin in their before
# request will use the logger context of the previous request.
@application.before_request
def before_request(): # pylint: disable=unused-variable
request_id = str(uuid4())
logger.new(request_id=request_id)
span, trace = get_span_and_trace(flask_request.headers)
if span and trace:
logger.bind(span=span, trace=trace)
logger.info(
"request",
method=flask_request.method,
url_path=flask_request.full_path,
session_cookie_present="session" in flask_request.cookies,
csrf_token_present="csrf_token" in cookie_session,
user_agent=flask_request.user_agent.string,
)
setup_storage(application)
setup_submitter(application)
setup_feedback(application)
setup_publisher(application)
setup_task_client(application)
application.eq["id_generator"] = UserIDGenerator(
application.config["EQ_SERVER_SIDE_STORAGE_USER_ID_ITERATIONS"],
application.eq["secret_store"].get_secret_by_name(
"EQ_SERVER_SIDE_STORAGE_USER_ID_SALT"
),
application.eq["secret_store"].get_secret_by_name(
"EQ_SERVER_SIDE_STORAGE_USER_IK_SALT"
),
)
cache_questionnaire_schemas()
setup_secure_cookies(application)
setup_secure_headers(application)
setup_babel(application)
application.wsgi_app = AWSReverseProxied(application.wsgi_app)
application.url_map.strict_slashes = False
add_blueprints(application)
login_manager.init_app(application)
add_safe_health_check(application)
setup_compression(application)
setup_jinja_env(application)
@application.after_request
def apply_caching(response): # pylint: disable=unused-variable
if "text/html" in response.content_type:
for k, v in CACHE_HEADERS.items():
response.headers[k] = v
else:
response.headers["Cache-Control"] = "max-age=2628000, public"
return response
@application.after_request
def response_minify(response): # pylint: disable=unused-variable
"""
minify html response to decrease site traffic
"""
if (
application.config["EQ_ENABLE_HTML_MINIFY"]
and response.content_type == "text/html; charset=utf-8"
):
response.set_data(
minify(
response.get_data(as_text=True),
remove_comments=True,
remove_empty_space=True,
remove_optional_attribute_quotes=False,
)
)
return response
return response
@application.after_request
def after_request(response): # pylint: disable=unused-variable
# We're using the stringified version of the Flask session to get a rough
# length for the cookie. The real length won't be known yet as Flask
# serializes and adds the cookie header after this method is called.
logger.info(
"response",
status_code=response.status_code,
session_modified=cookie_session.modified,
)
return response
return application
def create_app(setting_overrides=None): # noqa: C901 pylint: disable=too-complex
application = Flask(__name__,
static_url_path='/s',
static_folder='../static')
application.config.from_object(settings)
application.eq = {}
with open(application.config['EQ_SECRETS_FILE']) as secrets_file:
secrets = yaml.safe_load(secrets_file)
with open(application.config['EQ_KEYS_FILE']) as keys_file:
keys = yaml.safe_load(keys_file)
validate_required_secrets(secrets)
validate_required_keys(keys, KEY_PURPOSE_SUBMISSION)
application.eq['secret_store'] = SecretStore(secrets)
application.eq['key_store'] = KeyStore(keys)
if setting_overrides:
application.config.update(setting_overrides)
if application.config['EQ_APPLICATION_VERSION']:
logger.info('starting eq survey runner',
version=application.config['EQ_APPLICATION_VERSION'])
if application.config['EQ_NEW_RELIC_ENABLED']:
setup_newrelic()
setup_database(application)
setup_dynamodb(application)
if application.config['EQ_RABBITMQ_ENABLED']:
application.eq['submitter'] = RabbitMQSubmitter(
host=application.config['EQ_RABBITMQ_HOST'],
secondary_host=application.config['EQ_RABBITMQ_HOST_SECONDARY'],
port=application.config['EQ_RABBITMQ_PORT'],
username=application.eq['secret_store'].get_secret_by_name(
'EQ_RABBITMQ_USERNAME'),
password=application.eq['secret_store'].get_secret_by_name(
'EQ_RABBITMQ_PASSWORD'),
)
else:
application.eq['submitter'] = LogSubmitter()
application.eq['id_generator'] = UserIDGenerator(
application.config['EQ_SERVER_SIDE_STORAGE_USER_ID_ITERATIONS'],
application.eq['secret_store'].get_secret_by_name(
'EQ_SERVER_SIDE_STORAGE_USER_ID_SALT'),
application.eq['secret_store'].get_secret_by_name(
'EQ_SERVER_SIDE_STORAGE_USER_IK_SALT'),
)
setup_secure_cookies(application)
setup_secure_headers(application)
setup_babel(application)
application.wsgi_app = AWSReverseProxied(application.wsgi_app)
add_blueprints(application)
configure_flask_logging(application)
login_manager.init_app(application)
add_safe_health_check(application)
if application.config['EQ_DEV_MODE']:
start_dev_mode(application)
if application.config['EQ_ENABLE_CACHE']:
cache.init_app(application, config={'CACHE_TYPE': 'simple'})
else:
# no cache and silence warning
cache.init_app(application, config={'CACHE_NO_NULL_WARNING': True})
# Switch off flask default autoescaping as content is html encoded
# during schema/metadata/summary context (and navigition) generation
application.jinja_env.autoescape = False
# Add theme manager
application.config['THEME_PATHS'] = os.path.dirname(
os.path.abspath(__file__))
Themes(application, app_identifier='surveyrunner')
@application.before_request
def before_request(): # pylint: disable=unused-variable
# While True the session lives for permanent_session_lifetime seconds
# Needed to be able to set the client-side cookie expiration
cookie_session.permanent = True
request_id = str(uuid4())
logger.new(request_id=request_id)
@application.after_request
def apply_caching(response): # pylint: disable=unused-variable
for k, v in CACHE_HEADERS.items():
response.headers[k] = v
return response
@application.context_processor
def override_url_for(): # pylint: disable=unused-variable
return dict(url_for=versioned_url_for)
return application
def test_validate_required_secrets_passes(self):
secrets = {"secrets": {secret: "abc" for secret in EXPECTED_SECRETS}}
self.assertIsNone(validate_required_secrets(secrets))
def create_app(setting_overrides=None): # noqa: C901 pylint: disable=too-complex,too-many-statements
application = Flask(__name__, static_url_path='/s', static_folder='../static')
application.config.from_object(settings)
application.eq = {}
with open(application.config['EQ_SECRETS_FILE']) as secrets_file:
secrets = yaml.safe_load(secrets_file)
with open(application.config['EQ_KEYS_FILE']) as keys_file:
keys = yaml.safe_load(keys_file)
validate_required_secrets(secrets)
validate_required_keys(keys, KEY_PURPOSE_SUBMISSION)
application.eq['secret_store'] = SecretStore(secrets)
application.eq['key_store'] = KeyStore(keys)
if setting_overrides:
application.config.update(setting_overrides)
if application.config['EQ_APPLICATION_VERSION']:
logger.info('starting eq survey runner', version=application.config['EQ_APPLICATION_VERSION'])
if application.config['EQ_NEW_RELIC_ENABLED']:
setup_newrelic()
setup_database(application)
setup_dynamodb(application)
setup_s3(application)
setup_bigtable(application)
setup_gcs(application)
setup_redis(application)
setup_gc_datastore(application)
if application.config['EQ_SUBMITTER'] == 'rabbitmq':
application.eq['submitter'] = RabbitMQSubmitter(
host=application.config['EQ_RABBITMQ_HOST'],
secondary_host=application.config['EQ_RABBITMQ_HOST_SECONDARY'],
port=application.config['EQ_RABBITMQ_PORT'],
username=application.eq['secret_store'].get_secret_by_name('EQ_RABBITMQ_USERNAME'),
password=application.eq['secret_store'].get_secret_by_name('EQ_RABBITMQ_PASSWORD'),
)
elif application.config['EQ_SUBMITTER'] == 'pubsub':
application.eq['submitter'] = PubSubSubmitter(
project_id=application.config['EQ_PUBSUB_PROJECT_ID'],
topic=application.config['EQ_PUBSUB_TOPIC'],
)
elif application.config['EQ_SUBMITTER'] == 'gcs':
application.eq['submitter'] = GCSSubmitter(
bucket_name=application.config['EQ_GCS_SUBMISSION_BUCKET_ID'],
)
else:
application.eq['submitter'] = LogSubmitter()
application.eq['id_generator'] = UserIDGenerator(
application.config['EQ_SERVER_SIDE_STORAGE_USER_ID_ITERATIONS'],
application.eq['secret_store'].get_secret_by_name('EQ_SERVER_SIDE_STORAGE_USER_ID_SALT'),
application.eq['secret_store'].get_secret_by_name('EQ_SERVER_SIDE_STORAGE_USER_IK_SALT'),
)
setup_secure_cookies(application)
setup_secure_headers(application)
setup_babel(application)
application.wsgi_app = AWSReverseProxied(application.wsgi_app)
add_blueprints(application)
configure_flask_logging(application)
login_manager.init_app(application)
add_safe_health_check(application)
if application.config['EQ_DEV_MODE']:
start_dev_mode(application)
# Switch off flask default autoescaping as content is html encoded
# during schema/metadata/summary context (and navigition) generation
application.jinja_env.autoescape = False
# Add theme manager
application.config['THEME_PATHS'] = os.path.dirname(os.path.abspath(__file__))
Themes(application, app_identifier='surveyrunner')
# pylint: disable=maybe-no-member
application.jinja_env.globals['theme'] = flask_theme_cache.get_global_theme_template()
@application.before_request
def before_request(): # pylint: disable=unused-variable
request_id = str(uuid4())
logger.new(request_id=request_id)
@application.after_request
def apply_caching(response): # pylint: disable=unused-variable
for k, v in CACHE_HEADERS.items():
response.headers[k] = v
return response
@application.context_processor
def override_url_for(): # pylint: disable=unused-variable
return dict(url_for=versioned_url_for)
return application
def test_validate_required_secrets_passes(self):
secrets = {
'secrets': {secret: 'abc' for secret in EXPECTED_SECRETS}
}
self.assertIsNone(validate_required_secrets(secrets))
