def test_dictionary_lookup_standardiser_appends_columns_to_data():
standardiser = EthnicityDictionaryLookup(
"tests/test_data/test_dictionary_lookup/test_lookup.csv")
# given data
data_set = EthnicityDataset(
data=[["Ethnicity", "Ethnicity type"], ["a", "any ethnicity type"]])
# when we add_columns
standardiser.process_data_set(data_set)
# then 4 columns are appended to the data
assert 6 == data_set.get_data()[0].__len__()
def test_dictionary_lookup_standardiser_can_handle_empty_rows():
standardiser = EthnicityDictionaryLookup(
"tests/test_data/test_dictionary_lookup/test_lookup.csv")
# given a dataset with a blank row
data = [["Ethnicity", "Ethnicity type"], [" a", "xxx"], []]
data_set = EthnicityDataset(data=data)
# when we add_columns
try:
standardiser.process_data_set(data_set)
except IndexError:
assert False
def test_dictionary_lookup_standardiser_without_default_values_appends_blanks_when_not_found(
):
standardiser = EthnicityDictionaryLookup(
"tests/test_data/test_dictionary_lookup/test_lookup.csv")
# given a dataset with a strange value
data = [["Ethnicity", "Ethnicity type"], ["strange", "missing"]]
data_set = EthnicityDataset(data=data)
# when we add_columns
standardiser.process_data_set(data_set)
# then 4 blank values are appended for the four columns
assert data_set.get_data()[1] == ["strange", "missing", "", "", "", ""]
def test_dictionary_lookup_standardiser_appends_columns_using_defaults_for_unknown_ethnicity_type(
):
standardiser = EthnicityDictionaryLookup(
"tests/test_data/test_dictionary_lookup/test_lookup.csv")
# given data from an ethnicity type not in the lookup
data = [["Ethnicity", "Ethnicity type"], [" a", "xxx"], ["b ", "xxx"]]
data_set = EthnicityDataset(data=data)
# when we add_columns
standardiser.process_data_set(data_set)
# then values are added
assert data_set.get_data()[0][2] == "Label"
assert data_set.get_data()[1][2] == "A"
assert data_set.get_data()[2][2] == "B"
def test_dictionary_lookup_standardiser_appends_columns_trimming_white_space_for_lookup(
):
standardiser = EthnicityDictionaryLookup(
"tests/test_data/test_dictionary_lookup/test_lookup.csv")
# given data where one has forward white space and the other has trailing
data = [["Ethnicity", "Ethnicity type"], [" a", "phonetic"],
["b ", "phonetic"]]
data_set = EthnicityDataset(data=data)
# when we add_columns
standardiser.process_data_set(data_set)
# then values are added
assert data_set.get_data()[0][2] == "Label"
assert data_set.get_data()[1][2] == "alpha"
assert data_set.get_data()[2][2] == "bravo"
def test_dictionary_lookup_standardiser_appends_columns_using_case_insensitive_lookup(
):
standardiser = EthnicityDictionaryLookup(
"tests/test_data/test_dictionary_lookup/test_lookup.csv")
# given data where one is capitalised
data = [["Ethnicity", "Ethnicity type"], ["A", "phonetic"],
["b", "phonetic"]]
data_set = EthnicityDataset(data=data)
# when we add_columns
standardiser.process_data_set(data_set)
# then values are added
assert data_set.get_data()[0][2] == "Label"
assert data_set.get_data()[1][2] == "alpha"
assert data_set.get_data()[2][2] == "bravo"
def test_dictionary_lookup_standardiser_appends_columns_using_specific_ethnicity_type_in_lookup(
):
standardiser = EthnicityDictionaryLookup(
"tests/test_data/test_dictionary_lookup/test_lookup.csv")
# given data from an ethnicity type in the lookup
data = [["Ethnicity", "Ethnicity type"], ["a", "phonetic"],
["b", "phonetic"]]
data_set = EthnicityDataset(data=data)
# when we add_columns
standardiser.process_data_set(data_set)
# then added values come from entries in the lookup with ethnicity_type = ''
assert data_set.get_data()[0][2] == "Label"
assert data_set.get_data()[1][2] == "alpha"
assert data_set.get_data()[2][2] == "bravo"
def test_dictionary_lookup_standardiser_with_wildcard_values_inserts_custom_defaults_when_not_found(
):
default_values = ["*", "two", "Unknown - *", "four"]
standardiser = EthnicityDictionaryLookup(
"tests/test_data/test_dictionary_lookup/test_lookup.csv",
default_values=default_values)
# given a dataset with a strange value
data = [["Ethnicity", "Ethnicity type"], ["strange", "missing"]]
data_set = EthnicityDataset(data=data)
# when we add_columns
standardiser.process_data_set(data_set)
# then the default values are appended with * substituted with the ethnicity value
assert data_set.get_data()[1] == [
"strange", "missing", "strange", "two", "Unknown - strange", "four"
]
def create_app(config_object):
from application.static_site import static_site_blueprint
from application.cms import cms_blueprint
from application.admin import admin_blueprint
from application.register import register_blueprint
from application.auth import auth_blueprint
from application.dashboard import dashboard_blueprint
from application.review import review_blueprint
from application.redirects import redirects_blueprint
if isinstance(config_object, str):
from application.config import DevConfig, Config, TestConfig
if config_object.lower().startswith("production"):
config_object = Config
elif config_object.lower().startswith("dev"):
config_object = DevConfig
elif config_object.lower().startswith("test"):
config_object = TestConfig
else:
raise ValueError(
f"Invalid config name passed into create_app: {config_object}")
app = Flask(__name__)
app.config.from_object(config_object)
app.file_service = FileService()
app.file_service.init_app(app)
page_service.init_app(app)
upload_service.init_app(app)
scanner_service.init_app(app)
dimension_service.init_app(app)
trello_service.init_app(app)
trello_service.set_credentials(config_object.TRELLO_API_KEY,
config_object.TRELLO_API_TOKEN)
db.init_app(app)
app.url_map.strict_slashes = False
app.dictionary_lookup = EthnicityDictionaryLookup(
lookup_file=config_object.DICTIONARY_LOOKUP_FILE,
default_values=config_object.DICTIONARY_LOOKUP_DEFAULTS)
app.classification_finder = ethnicity_classification_finder_from_file(
config_object.ETHNICITY_CLASSIFICATION_FINDER_LOOKUP,
config_object.ETHNICITY_CLASSIFICATION_FINDER_CLASSIFICATIONS,
)
# Note not using Flask-Security role model
user_datastore = SQLAlchemyUserDatastore(db, User, None)
Security(app, user_datastore)
if os.environ.get("SENTRY_DSN") is not None:
sentry = Sentry(app, dsn=os.environ["SENTRY_DSN"])
app.register_blueprint(cms_blueprint)
app.register_blueprint(static_site_blueprint)
app.register_blueprint(admin_blueprint)
app.register_blueprint(register_blueprint)
app.register_blueprint(auth_blueprint)
app.register_blueprint(dashboard_blueprint)
app.register_blueprint(review_blueprint)
app.register_blueprint(redirects_blueprint)
# To stop url clash between this and the measure page url (which is made of four variables.
# See: https://stackoverflow.com/questions/17135006/url-routing-conflicts-for-static-files-in-flask-dev-server
@app.route("/static/<path:subdir1>/<subdir2>/<file_name>")
def static_subdir(subdir1, subdir2, file_name):
file_path = "%s/%s/%s" % (subdir1, subdir2, file_name)
return send_from_directory("static", file_path)
register_errorhandlers(app)
app.after_request(harden_app)
# Render jinja templates with less whitespace; applies to both CMS and static build
app.jinja_env.trim_blocks = True
app.jinja_env.lstrip_blocks = True
app.jinja_env.add_extension(jinja_do)
app.add_template_filter(format_page_guid)
app.add_template_filter(format_approve_button)
app.add_template_filter(format_date_time)
app.add_template_filter(render_markdown)
app.add_template_filter(filesize)
app.add_template_filter(format_friendly_date)
app.add_template_filter(format_friendly_short_date)
app.add_template_filter(format_friendly_short_date_with_year)
app.add_template_filter(format_versions)
app.add_template_filter(format_status)
app.add_template_filter(value_filter)
app.add_template_filter(flatten)
app.add_template_filter(flatten_chart)
app.add_template_filter(version_filter)
app.add_template_filter(strip_trailing_slash)
app.add_template_filter(join_enum_display_names)
app.add_template_filter(slugify_value)
app.add_template_filter(first_bullet)
app.add_template_filter(index_of_last_initial_zero)
app.add_template_filter(yesno)
# There is a CSS caching problem in chrome
app.config["SEND_FILE_MAX_AGE_DEFAULT"] = 10
setup_app_logging(app, config_object)
if os.environ.get("SQREEN_TOKEN") is not None:
setup_sqreen_audit(app)
from werkzeug.contrib.fixers import ProxyFix
app.wsgi_app = ProxyFix(app.wsgi_app)
from flask_sslify import SSLify
SSLify(app)
mail.init_app(app)
@app.context_processor
def inject_globals():
from application.auth.models import (
COPY_MEASURE,
CREATE_MEASURE,
CREATE_VERSION,
DELETE_MEASURE,
MANAGE_SYSTEM,
MANAGE_USERS,
ORDER_MEASURES,
PUBLISH,
READ,
UPDATE_MEASURE,
VIEW_DASHBOARDS,
)
return dict(
COPY_MEASURE=COPY_MEASURE,
CREATE_MEASURE=CREATE_MEASURE,
CREATE_VERSION=CREATE_VERSION,
DELETE_MEASURE=DELETE_MEASURE,
MANAGE_SYSTEM=MANAGE_SYSTEM,
MANAGE_USERS=MANAGE_USERS,
ORDER_MEASURES=ORDER_MEASURES,
PUBLISH=PUBLISH,
READ=READ,
UPDATE_MEASURE=UPDATE_MEASURE,
VIEW_DASHBOARDS=VIEW_DASHBOARDS,
get_content_security_policy=get_content_security_policy,
)
return app