class TestServerConfig(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
self.server_config = self.config.server_config
self.config.complete_config()
message_list = []
def noop(message):
message_list.append(message)
messagefn = noop
self.context = dict(messagefn=messagefn, messages=message_list)
def get_config(self, **kwargs):
file_name = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}",
config_file_name=self.config_file_name,
**kwargs)
config = AppConfig()
config.update_from_config_file(file_name)
return config
def test_init_raises_error_if_default_config_is_invalid(self):
invalid_config = self.get_config(port="not_valid")
with self.assertRaises(ConfigurationError):
invalid_config.complete_config()
@patch(
"server.common.config.server_config.BaseConfig.validate_correct_type_of_configuration_attribute"
)
def test_complete_config_checks_all_attr(self, mock_check_attrs):
mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute(
)
self.server_config.complete_config(self.context)
self.assertEqual(mock_check_attrs.call_count, 40)
def test_handle_app__throws_error_if_port_doesnt_exist(self):
config = self.get_config(port=99999999)
with self.assertRaises(ConfigurationError):
config.server_config.handle_app(self.context)
@patch("server.common.config.server_config.discover_s3_region_name")
def test_handle_data_locator_works_for_default_types(
self, mock_discover_region_name):
mock_discover_region_name.return_value = None
# Default config
self.assertEqual(
self.config.server_config.data_locator__s3__region_name, None)
# hard coded
config = self.get_config()
self.assertEqual(config.server_config.data_locator__s3__region_name,
"us-east-1")
# incorrectly formatted
dataroot = {
"d1": {
"base_url": "set1",
"dataroot": "/path/to/set1_datasets/"
},
"d2": {
"base_url": "set2/subdir",
"dataroot": "s3://shouldnt/work"
},
}
file_name = self.custom_app_config(
dataroot=dataroot,
config_file_name=self.config_file_name,
data_locater_region_name="true")
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_data_locator()
@patch("server.common.config.server_config.discover_s3_region_name")
def test_handle_data_locator_can_read_from_dataroot(
self, mock_discover_region_name):
mock_discover_region_name.return_value = "us-west-2"
dataroot = {
"d1": {
"base_url": "set1",
"dataroot": "/path/to/set1_datasets/"
},
"d2": {
"base_url": "set2/subdir",
"dataroot": "s3://hosted-cellxgene-dev"
},
}
file_name = self.custom_app_config(
dataroot=dataroot,
config_file_name=self.config_file_name,
data_locater_region_name="true")
config = AppConfig()
config.update_from_config_file(file_name)
config.server_config.handle_data_locator()
self.assertEqual(config.server_config.data_locator__s3__region_name,
"us-west-2")
mock_discover_region_name.assert_called_once_with(
"s3://hosted-cellxgene-dev")
def test_handle_app___can_use_envar_port(self):
config = self.get_config(port=24)
self.assertEqual(config.server_config.app__port, 24)
# Note if the port is set in the config file it will NOT be overwritten by a different envvar
os.environ["CXG_SERVER_PORT"] = "4008"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.server_config.handle_app(self.context)
self.assertEqual(self.config.server_config.app__port, 4008)
del os.environ["CXG_SERVER_PORT"]
def test_handle_app__can_get_secret_key_from_envvar_or_config_file_with_envvar_given_preference(
self):
config = self.get_config(flask_secret_key="KEY_FROM_FILE")
self.assertEqual(config.server_config.app__flask_secret_key,
"KEY_FROM_FILE")
os.environ["CXG_SECRET_KEY"] = "KEY_FROM_ENV"
config.external_config.handle_environment(self.context)
self.assertEqual(config.server_config.app__flask_secret_key,
"KEY_FROM_ENV")
def test_handle_app__sets_web_base_url(self):
config = self.get_config(web_base_url="anything.com")
self.assertEqual(config.server_config.app__web_base_url,
"anything.com")
def test_handle_auth__gets_client_secret_from_envvars_or_config_with_envvars_given_preference(
self):
config = self.get_config(client_secret="KEY_FROM_FILE")
config.server_config.handle_authentication()
self.assertEqual(
config.server_config.authentication__params_oauth__client_secret,
"KEY_FROM_FILE")
os.environ["CXG_OAUTH_CLIENT_SECRET"] = "KEY_FROM_ENV"
config.external_config.handle_environment(self.context)
self.assertEqual(
config.server_config.authentication__params_oauth__client_secret,
"KEY_FROM_ENV")
def test_handle_data_source__errors_when_passed_zero_or_two_dataroots(
self):
file_name = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}",
config_file_name="two_data_roots.yml",
dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
)
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_data_source()
file_name = self.custom_app_config(config_file_name="zero_roots.yml")
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_data_source()
def test_get_api_base_url_works(self):
# test the api_base_url feature, and that it can contain a path
config = AppConfig()
backend_port = find_available_port("localhost", 10000)
config.update_server_config(
app__flask_secret_key="secret",
app__api_base_url=
f"http://*****:*****@patch("server.common.config.server_config.diffexp_tiledb.set_config")
def test_handle_diffexp(self, mock_tiledb_config):
custom_config_file = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}",
cpu_multiplier=3,
diffexp_max_workers=1,
target_workunit=4,
config_file_name=self.config_file_name,
)
config = AppConfig()
config.update_from_config_file(custom_config_file)
config.server_config.handle_diffexp()
# called with the min of diffexp_max_workers and cpus*cpu_multiplier
mock_tiledb_config.assert_called_once_with(1, 4)
@patch("server.data_cxg.cxg_adaptor.CxgAdaptor.set_tiledb_context")
def test_handle_adaptor(self, mock_tiledb_context):
custom_config = self.custom_app_config(dataroot=f"{FIXTURES_ROOT}",
cxg_tile_cache_size=10,
cxg_num_reader_threads=2)
config = AppConfig()
config.update_from_config_file(custom_config)
config.server_config.handle_adaptor()
mock_tiledb_context.assert_called_once_with({
"sm.tile_cache_size":
10,
"sm.num_reader_threads":
2,
"vfs.s3.region":
"us-east-1"
})
def test_multi_dataset(self):
config = AppConfig()
# test for illegal url_dataroots
for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
config.update_server_config(
app__flask_secret_key="secret",
multi_dataset__dataroot={
"tag": {
"base_url": illegal,
"dataroot": f"{PROJECT_ROOT}/example-dataset"
}
},
)
with self.assertRaises(ConfigurationError):
config.complete_config()
# test for legal url_dataroots
for legal in ("d", "this.is-okay_", "a/b"):
config.update_server_config(
app__flask_secret_key="secret",
multi_dataset__dataroot={
"tag": {
"base_url": legal,
"dataroot": f"{PROJECT_ROOT}/example-dataset"
}
},
)
config.complete_config()
# test that multi dataroots work end to end
config.update_server_config(
app__flask_secret_key="secret",
multi_dataset__dataroot=dict(
s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset",
base_url="set1/1/2"),
s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
),
)
# Change this default to test if the dataroot overrides below work.
config.update_default_dataset_config(
app__about_legal_tos="tos_default.html")
# specialize the configs for set1
config.add_dataroot_config("s1",
user_annotations__enable=False,
diffexp__enable=True,
app__about_legal_tos="tos_set1.html")
# specialize the configs for set2
config.add_dataroot_config("s2",
user_annotations__enable=True,
diffexp__enable=False,
app__about_legal_tos="tos_set2.html")
# no specializations for set3 (they get the default dataset config)
config.complete_config()
with test_server(app_config=config) as server:
session = requests.Session()
response = session.get(
f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is False
assert data_config["config"]["parameters"][
"disable-diffexp"] is False
assert data_config["config"]["parameters"][
"about_legal_tos"] == "tos_set1.html"
response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"][
"about_legal_tos"] == "tos_set2.html"
response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"][
"disable-diffexp"] is False
assert data_config["config"]["parameters"][
"about_legal_tos"] == "tos_default.html"
response = session.get(f"{server}/health")
assert response.json()["status"] == "pass"
def test_simple_update_single_config_from_path_and_value(self):
"""Update a simple config parameter"""
config = AppConfig()
config.server_config.multi_dataset__dataroot = dict(
s1=dict(dataroot="my_dataroot_s1", base_url="my_baseurl_s1"),
s2=dict(dataroot="my_dataroot_s2", base_url="my_baseurl_s2"),
)
config.add_dataroot_config("s1")
config.add_dataroot_config("s2")
# test simple value in server
config.update_single_config_from_path_and_value(
["server", "app", "flask_secret_key"], "mysecret")
self.assertEqual(config.server_config.app__flask_secret_key,
"mysecret")
# test simple value in default dataset
config.update_single_config_from_path_and_value(
["dataset", "user_annotations", "hosted_tiledb_array", "db_uri"],
"mydburi",
)
self.assertEqual(
config.default_dataset_config.
user_annotations__hosted_tiledb_array__db_uri, "mydburi")
self.assertEqual(
config.dataroot_config["s1"].
user_annotations__hosted_tiledb_array__db_uri, "mydburi")
self.assertEqual(
config.dataroot_config["s2"].
user_annotations__hosted_tiledb_array__db_uri, "mydburi")
# test simple value in specific dataset
config.update_single_config_from_path_and_value([
"per_dataset_config", "s1", "user_annotations",
"hosted_tiledb_array", "db_uri"
], "s1dburi")
self.assertEqual(
config.default_dataset_config.
user_annotations__hosted_tiledb_array__db_uri, "mydburi")
self.assertEqual(
config.dataroot_config["s1"].
user_annotations__hosted_tiledb_array__db_uri, "s1dburi")
self.assertEqual(
config.dataroot_config["s2"].
user_annotations__hosted_tiledb_array__db_uri, "mydburi")
# error checking
bad_paths = [
(
["dataset", "does", "not", "exist"],
"unknown config parameter at path: '['dataset', 'does', 'not', 'exist']'",
),
(["does", "not", "exist"],
"path must start with 'server', 'dataset', or 'per_dataset_config'"
),
([],
"path must start with 'server', 'dataset', or 'per_dataset_config'"
),
(["per_dataset_config"],
"missing dataroot when using per_dataset_config: got '['per_dataset_config']'"
),
(
["per_dataset_config", "unknown"],
"unknown dataroot when using per_dataset_config: got '['per_dataset_config', 'unknown']',"
" dataroots specified in config are ['s1', 's2']",
),
([1, 2, 3], "path must be a list of strings, got '[1, 2, 3]'"),
("string", "path must be a list of strings, got 'string'"),
]
for bad_path, error_message in bad_paths:
with self.assertRaises(ConfigurationError) as config_error:
config.update_single_config_from_path_and_value(
bad_path, "value")
self.assertEqual(config_error.exception.message, error_message)
def test_auth_test(self):
app_config = AppConfig()
app_config.update_server_config(app__flask_secret_key="secret")
app_config.update_server_config(authentication__type="test")
app_config.update_server_config(
authentication__insecure_test_environment=True)
app_config.update_server_config(multi_dataset__dataroot=dict(
a1=dict(dataroot=self.dataset_dataroot, base_url="auth"),
a2=dict(dataroot=self.dataset_dataroot, base_url="no-auth"),
))
# specialize the configs
app_config.add_dataroot_config("a1",
app__authentication_enable=True,
user_annotations__enable=True)
app_config.add_dataroot_config("a2",
app__authentication_enable=False,
user_annotations__enable=False)
app_config.complete_config()
with test_server(app_config=app_config) as server:
session = requests.Session()
# auth datasets
config = session.get(
f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(
f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(
config["config"]["authentication"]["requires_client_login"])
self.assertTrue(config["config"]["parameters"]["annotations"])
login_uri = config["config"]["authentication"]["login"]
logout_uri = config["config"]["authentication"]["logout"]
self.assertEqual(login_uri, "/login?dataset=auth/pbmc3k.cxg")
self.assertEqual(logout_uri, "/logout?dataset=auth/pbmc3k.cxg")
r = session.get(f"{server}/{login_uri}")
# check that the login redirect worked
self.assertEqual(r.history[0].status_code, 302)
self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg")
config = session.get(
f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(
f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["username"], "test_account")
self.assertEqual(userinfo["userinfo"]["picture"], None)
self.assertTrue(config["config"]["parameters"]["annotations"])
r = session.get(f"{server}/{logout_uri}")
# check that the logout redirect worked
self.assertEqual(r.history[0].status_code, 302)
self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg")
config = session.get(
f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(
f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(config["config"]["parameters"]["annotations"])
# no-auth datasets
config = session.get(
f"{server}/no-auth/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(
f"{server}/no-auth/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertIsNone(userinfo)
self.assertFalse(config["config"]["parameters"]["annotations"])
# login with a picture
session.get(f"{server}/{login_uri}&picture=myimage.png")
userinfo = session.get(
f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["picture"], "myimage.png")
