def key_store(keys: str) -> KeyStore:
secrets = json.loads(keys)
logger.info("Validating key file")
validate_required_keys(secrets, "authentication")
return KeyStore(secrets)
def test_validate_required_keys_missing_private_keys_for_purpose():
"""Tests exeception is raised if there are no private keys with 'authentication' purpose"""
with pytest.raises(CryptoError):
keystore_dict = {
"keys": {
"insert_kid_here": {'purpose': KEY_PURPOSE_AUTHENTICATION,
'type': 'public',
'value': TEST_DO_NOT_USE_SR_PRIVATE_PEM},
}
}
validate_required_keys(keystore_dict, KEY_PURPOSE_AUTHENTICATION)
def generate_token(json_secret_keys, payload):
"""
Generates the token by encrypting the payload with sdc-cryptography.
"""
print("Generating token")
keys = json.loads(json_secret_keys)
validate_required_keys(keys, KEY_PURPOSE)
key_store = KeyStore(keys)
encrypted_data = encrypt(payload,
key_store=key_store,
key_purpose=KEY_PURPOSE)
return encrypted_data
def create_app():
app = Flask(__name__)
app.config.from_object(settings)
app.sdx = {}
with open(app.config['SDX_KEYS_FILE']) as file:
keys = yaml.safe_load(file)
validate_required_keys(keys, KEY_PURPOSE_SUBMISSION)
app.sdx['key_store'] = KeyStore(keys)
return app
def __init__(self, args, services):
self.args = args
self.services = services
self.publisher = DurableTopicPublisher(**self.amqp_params(services))
self.executor = ThreadPoolExecutor(max_workers=4)
self.rabbit_check = None
self.ftp_check = None
self.transfer = False
self.key_purpose = 'outbound'
keys_file_location = os.getenv('SDX_SEFT_CONSUMER_KEYS_FILE',
'./jwt-test-keys/keys.yml')
with open(keys_file_location) as file:
self.secrets_from_file = yaml.safe_load(file)
validate_required_keys(self.secrets_from_file, self.key_purpose)
self.key_store = KeyStore(self.secrets_from_file)
def main():
logger.debug("Starting SEFT consumer service")
app = make_app()
server = tornado.httpserver.HTTPServer(app)
server.bind(int(os.getenv("SDX_SEFT_CONSUMER_SERVICE_PORT", '8080')))
server.start(0)
with open(settings.SDX_SEFT_CONSUMER_KEYS_FILE) as file:
keys = yaml.safe_load(file)
try:
# Create the scheduled health task
task = GetHealth()
sched = tornado.ioloop.PeriodicCallback(
task.determine_health,
HEALTHCHECK_DELAY_MILLISECONDS,
)
sched.start()
logger.info("Scheduled healthcheck started.")
# Get initial health
loop = tornado.ioloop.IOLoop.current()
loop.call_later(HEALTHCHECK_DELAY_MILLISECONDS, task.determine_health)
validate_required_keys(keys, KEY_PURPOSE_CONSUMER)
seft_consumer = SeftConsumer(keys)
seft_consumer.run()
except CryptoError as e:
logger.critical("Unable to find valid keys", error=str(e))
except KeyboardInterrupt:
logger.debug("SEFT consumer service stopping")
seft_consumer.stop()
logger.debug("SEFT consumer service stopped")
def create_app( # noqa: C901 pylint: disable=too-complex, too-many-statements
setting_overrides=None,
):
application = Flask(__name__, template_folder="../templates")
application.config.from_object(settings)
if setting_overrides:
application.config.update(setting_overrides)
application.eq = {}
with open(application.config["EQ_SECRETS_FILE"]) as secrets_file:
secrets = yaml.safe_load(secrets_file)
conditional_required_secrets = []
if application.config["ADDRESS_LOOKUP_API_AUTH_ENABLED"]:
conditional_required_secrets.append("ADDRESS_LOOKUP_API_AUTH_TOKEN_SECRET")
validate_required_secrets(secrets, conditional_required_secrets)
application.eq["secret_store"] = SecretStore(secrets)
with open(application.config["EQ_KEYS_FILE"]) as keys_file:
keys = yaml.safe_load(keys_file)
validate_required_keys(keys, KEY_PURPOSE_SUBMISSION)
application.eq["key_store"] = KeyStore(keys)
if application.config["EQ_APPLICATION_VERSION"]:
logger.info(
"starting eq survey runner",
version=application.config["EQ_APPLICATION_VERSION"],
)
# IMPORTANT: This must be initialised *before* any other Flask plugins that add
# before_request hooks. Otherwise any logging by the plugin in their before
# request will use the logger context of the previous request.
@application.before_request
def before_request(): # pylint: disable=unused-variable
request_id = str(uuid4())
logger.new(request_id=request_id)
span, trace = get_span_and_trace(flask_request.headers)
if span and trace:
logger.bind(span=span, trace=trace)
logger.info(
"request",
method=flask_request.method,
url_path=flask_request.full_path,
session_cookie_present="session" in flask_request.cookies,
csrf_token_present="csrf_token" in cookie_session,
user_agent=flask_request.user_agent.string,
)
setup_storage(application)
setup_submitter(application)
setup_feedback(application)
setup_publisher(application)
setup_task_client(application)
application.eq["id_generator"] = UserIDGenerator(
application.config["EQ_SERVER_SIDE_STORAGE_USER_ID_ITERATIONS"],
application.eq["secret_store"].get_secret_by_name(
"EQ_SERVER_SIDE_STORAGE_USER_ID_SALT"
),
application.eq["secret_store"].get_secret_by_name(
"EQ_SERVER_SIDE_STORAGE_USER_IK_SALT"
),
)
cache_questionnaire_schemas()
setup_secure_cookies(application)
setup_secure_headers(application)
setup_babel(application)
application.wsgi_app = AWSReverseProxied(application.wsgi_app)
application.url_map.strict_slashes = False
add_blueprints(application)
login_manager.init_app(application)
add_safe_health_check(application)
setup_compression(application)
setup_jinja_env(application)
@application.after_request
def apply_caching(response): # pylint: disable=unused-variable
if "text/html" in response.content_type:
for k, v in CACHE_HEADERS.items():
response.headers[k] = v
else:
response.headers["Cache-Control"] = "max-age=2628000, public"
return response
@application.after_request
def response_minify(response): # pylint: disable=unused-variable
"""
minify html response to decrease site traffic
"""
if (
application.config["EQ_ENABLE_HTML_MINIFY"]
and response.content_type == "text/html; charset=utf-8"
):
response.set_data(
minify(
response.get_data(as_text=True),
remove_comments=True,
remove_empty_space=True,
remove_optional_attribute_quotes=False,
)
)
return response
return response
@application.after_request
def after_request(response): # pylint: disable=unused-variable
# We're using the stringified version of the Flask session to get a rough
# length for the cookie. The real length won't be known yet as Flask
# serializes and adds the cookie header after this method is called.
logger.info(
"response",
status_code=response.status_code,
session_modified=cookie_session.modified,
)
return response
return application
def create_app(setting_overrides=None): # noqa: C901 pylint: disable=too-complex
application = Flask(__name__,
static_url_path='/s',
static_folder='../static')
application.config.from_object(settings)
application.eq = {}
with open(application.config['EQ_SECRETS_FILE']) as secrets_file:
secrets = yaml.safe_load(secrets_file)
with open(application.config['EQ_KEYS_FILE']) as keys_file:
keys = yaml.safe_load(keys_file)
validate_required_secrets(secrets)
validate_required_keys(keys, KEY_PURPOSE_SUBMISSION)
application.eq['secret_store'] = SecretStore(secrets)
application.eq['key_store'] = KeyStore(keys)
if setting_overrides:
application.config.update(setting_overrides)
if application.config['EQ_APPLICATION_VERSION']:
logger.info('starting eq survey runner',
version=application.config['EQ_APPLICATION_VERSION'])
if application.config['EQ_NEW_RELIC_ENABLED']:
setup_newrelic()
setup_database(application)
setup_dynamodb(application)
if application.config['EQ_RABBITMQ_ENABLED']:
application.eq['submitter'] = RabbitMQSubmitter(
host=application.config['EQ_RABBITMQ_HOST'],
secondary_host=application.config['EQ_RABBITMQ_HOST_SECONDARY'],
port=application.config['EQ_RABBITMQ_PORT'],
username=application.eq['secret_store'].get_secret_by_name(
'EQ_RABBITMQ_USERNAME'),
password=application.eq['secret_store'].get_secret_by_name(
'EQ_RABBITMQ_PASSWORD'),
)
else:
application.eq['submitter'] = LogSubmitter()
application.eq['id_generator'] = UserIDGenerator(
application.config['EQ_SERVER_SIDE_STORAGE_USER_ID_ITERATIONS'],
application.eq['secret_store'].get_secret_by_name(
'EQ_SERVER_SIDE_STORAGE_USER_ID_SALT'),
application.eq['secret_store'].get_secret_by_name(
'EQ_SERVER_SIDE_STORAGE_USER_IK_SALT'),
)
setup_secure_cookies(application)
setup_secure_headers(application)
setup_babel(application)
application.wsgi_app = AWSReverseProxied(application.wsgi_app)
add_blueprints(application)
configure_flask_logging(application)
login_manager.init_app(application)
add_safe_health_check(application)
if application.config['EQ_DEV_MODE']:
start_dev_mode(application)
if application.config['EQ_ENABLE_CACHE']:
cache.init_app(application, config={'CACHE_TYPE': 'simple'})
else:
# no cache and silence warning
cache.init_app(application, config={'CACHE_NO_NULL_WARNING': True})
# Switch off flask default autoescaping as content is html encoded
# during schema/metadata/summary context (and navigition) generation
application.jinja_env.autoescape = False
# Add theme manager
application.config['THEME_PATHS'] = os.path.dirname(
os.path.abspath(__file__))
Themes(application, app_identifier='surveyrunner')
@application.before_request
def before_request(): # pylint: disable=unused-variable
# While True the session lives for permanent_session_lifetime seconds
# Needed to be able to set the client-side cookie expiration
cookie_session.permanent = True
request_id = str(uuid4())
logger.new(request_id=request_id)
@application.after_request
def apply_caching(response): # pylint: disable=unused-variable
for k, v in CACHE_HEADERS.items():
response.headers[k] = v
return response
@application.context_processor
def override_url_for(): # pylint: disable=unused-variable
return dict(url_for=versioned_url_for)
return application
def __init__(self, json_secret_keys):
keys = json.loads(json_secret_keys)
validate_required_keys(keys, KEY_PURPOSE)
self.keystore = KeyStore(keys)
def create_app(setting_overrides=None): # noqa: C901 pylint: disable=too-complex,too-many-statements
application = Flask(__name__, static_url_path='/s', static_folder='../static')
application.config.from_object(settings)
application.eq = {}
with open(application.config['EQ_SECRETS_FILE']) as secrets_file:
secrets = yaml.safe_load(secrets_file)
with open(application.config['EQ_KEYS_FILE']) as keys_file:
keys = yaml.safe_load(keys_file)
validate_required_secrets(secrets)
validate_required_keys(keys, KEY_PURPOSE_SUBMISSION)
application.eq['secret_store'] = SecretStore(secrets)
application.eq['key_store'] = KeyStore(keys)
if setting_overrides:
application.config.update(setting_overrides)
if application.config['EQ_APPLICATION_VERSION']:
logger.info('starting eq survey runner', version=application.config['EQ_APPLICATION_VERSION'])
if application.config['EQ_NEW_RELIC_ENABLED']:
setup_newrelic()
setup_database(application)
setup_dynamodb(application)
setup_s3(application)
setup_bigtable(application)
setup_gcs(application)
setup_redis(application)
setup_gc_datastore(application)
if application.config['EQ_SUBMITTER'] == 'rabbitmq':
application.eq['submitter'] = RabbitMQSubmitter(
host=application.config['EQ_RABBITMQ_HOST'],
secondary_host=application.config['EQ_RABBITMQ_HOST_SECONDARY'],
port=application.config['EQ_RABBITMQ_PORT'],
username=application.eq['secret_store'].get_secret_by_name('EQ_RABBITMQ_USERNAME'),
password=application.eq['secret_store'].get_secret_by_name('EQ_RABBITMQ_PASSWORD'),
)
elif application.config['EQ_SUBMITTER'] == 'pubsub':
application.eq['submitter'] = PubSubSubmitter(
project_id=application.config['EQ_PUBSUB_PROJECT_ID'],
topic=application.config['EQ_PUBSUB_TOPIC'],
)
elif application.config['EQ_SUBMITTER'] == 'gcs':
application.eq['submitter'] = GCSSubmitter(
bucket_name=application.config['EQ_GCS_SUBMISSION_BUCKET_ID'],
)
else:
application.eq['submitter'] = LogSubmitter()
application.eq['id_generator'] = UserIDGenerator(
application.config['EQ_SERVER_SIDE_STORAGE_USER_ID_ITERATIONS'],
application.eq['secret_store'].get_secret_by_name('EQ_SERVER_SIDE_STORAGE_USER_ID_SALT'),
application.eq['secret_store'].get_secret_by_name('EQ_SERVER_SIDE_STORAGE_USER_IK_SALT'),
)
setup_secure_cookies(application)
setup_secure_headers(application)
setup_babel(application)
application.wsgi_app = AWSReverseProxied(application.wsgi_app)
add_blueprints(application)
configure_flask_logging(application)
login_manager.init_app(application)
add_safe_health_check(application)
if application.config['EQ_DEV_MODE']:
start_dev_mode(application)
# Switch off flask default autoescaping as content is html encoded
# during schema/metadata/summary context (and navigition) generation
application.jinja_env.autoescape = False
# Add theme manager
application.config['THEME_PATHS'] = os.path.dirname(os.path.abspath(__file__))
Themes(application, app_identifier='surveyrunner')
# pylint: disable=maybe-no-member
application.jinja_env.globals['theme'] = flask_theme_cache.get_global_theme_template()
@application.before_request
def before_request(): # pylint: disable=unused-variable
request_id = str(uuid4())
logger.new(request_id=request_id)
@application.after_request
def apply_caching(response): # pylint: disable=unused-variable
for k, v in CACHE_HEADERS.items():
response.headers[k] = v
return response
@application.context_processor
def override_url_for(): # pylint: disable=unused-variable
return dict(url_for=versioned_url_for)
return application
