def test_configfile_with_specialization(self):
# test that per_dataset_config config load the default config, then the specialized config
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
with open(configfile, "w") as fconfig:
config = """
server:
single_dataset:
datapath: fake_datapath
dataset:
user_annotations:
enable: false
type: local_file_csv
local_file_csv:
file: fake_file
directory: fake_dir
"""
fconfig.write(config)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
test_config = app_config.dataset_config
# test config from default
self.assertEqual(test_config.user_annotations__type, "local_file_csv")
self.assertEqual(test_config.user_annotations__local_file_csv__file, "fake_file")
def test_handle_data_locator_works_for_default_types(
self, mock_discover_region_name):
mock_discover_region_name.return_value = None
# Default config
self.assertEqual(
self.config.server_config.data_locator__s3__region_name, None)
# hard coded
config = self.get_config()
self.assertEqual(config.server_config.data_locator__s3__region_name,
"us-east-1")
# incorrectly formatted
dataroot = {
"d1": {
"base_url": "set1",
"dataroot": "/path/to/set1_datasets/"
},
"d2": {
"base_url": "set2/subdir",
"dataroot": "s3://shouldnt/work"
},
}
file_name = self.custom_app_config(
dataroot=dataroot,
config_file_name=self.config_file_name,
data_locater_region_name="true")
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_data_locator()
def test_configfile_no_dataset_section(self):
# test a config file without a dataset section
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
with open(configfile, "w") as fconfig:
config = """
server:
app:
flask_secret_key: secret
multi_dataset:
dataroot: test_dataroot
"""
fconfig.write(config)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
server_changes = app_config.server_config.changes_from_default()
dataset_changes = app_config.default_dataset_config.changes_from_default(
)
self.assertEqual(
server_changes,
[("app__flask_secret_key", "secret", None),
("multi_dataset__dataroot", "test_dataroot", None)],
)
self.assertEqual(dataset_changes, [])
def get_config(self, **kwargs):
file_name = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
)
config = AppConfig()
config.update_from_config_file(file_name)
return config
def get_config(self, **kwargs):
file_name = self.custom_app_config(
dataset_datapath=f"{H5AD_FIXTURE}",
config_file_name=self.config_file_name,
**kwargs)
config = AppConfig()
config.update_from_config_file(file_name)
return config
def test_get_dataset_config_returns_dataset_config_for_single_datasets(
self):
datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad"
file_name = self.custom_app_config(
dataset_datapath=datapath, config_file_name=self.config_file_name)
config = AppConfig()
config.update_from_config_file(file_name)
self.assertEqual(config.get_dataset_config(), config.dataset_config)
def test_handle_adaptor(self, mock_tiledb_context):
custom_config = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}", cxg_tile_cache_size=10, cxg_num_reader_threads=2
)
config = AppConfig()
config.update_from_config_file(custom_config)
config.server_config.handle_adaptor()
mock_tiledb_context.assert_called_once_with(
{"sm.tile_cache_size": 10, "sm.num_reader_threads": 2, "vfs.s3.region": "us-east-1"}
)
def test_handle_app___can_use_envar_port(self):
config = self.get_config(port=24)
self.assertEqual(config.server_config.app__port, 24)
# Note if the port is set in the config file it will NOT be overwritten by a different envvar
os.environ["CXG_SERVER_PORT"] = "4008"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.server_config.handle_app(self.context)
self.assertEqual(self.config.server_config.app__port, 4008)
del os.environ["CXG_SERVER_PORT"]
def test_handle_embeddings__checks_data_file_types(self):
file_name = self.custom_app_config(
embedding_names=["name1", "name2"],
enable_reembedding="true",
dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
anndata_backed="true",
config_file_name=self.config_file_name,
)
config = AppConfig()
config.update_from_config_file(file_name)
config.server_config.complete_config(self.context)
with self.assertRaises(ConfigurationError):
config.default_dataset_config.handle_embeddings()
def test_handle_diffexp(self, mock_tiledb_config):
custom_config_file = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}",
cpu_multiplier=3,
diffexp_max_workers=1,
target_workunit=4,
config_file_name=self.config_file_name,
)
config = AppConfig()
config.update_from_config_file(custom_config_file)
config.server_config.handle_diffexp()
# called with the min of diffexp_max_workers and cpus*cpu_multiplier
mock_tiledb_config.assert_called_once_with(1, 4)
def setUp(self):
self.data_file = DataLocator(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
config = AppConfig()
config.update_server_config(single_dataset__datapath=self.data_file.path)
config.update_server_config(app__flask_secret_key="secret")
config.complete_config()
self.data = AnndataAdaptor(self.data_file, config)
def data_with_tmp_tiledb_annotations(ext: MatrixDataType):
tmp_dir = tempfile.mkdtemp()
fname = {
MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
MatrixDataType.CXG: "test/fixtures/pbmc3k.cxg",
}[ext]
data_locator = DataLocator(fname)
config = AppConfig()
config.update_server_config(
app__flask_secret_key="secret",
multi_dataset__dataroot=data_locator.path,
authentication__type="test",
authentication__insecure_test_environment=True,
)
config.update_default_dataset_config(
embeddings__names=["umap"],
presentation__max_categories=100,
diffexp__lfc_cutoff=0.01,
user_annotations__type="hosted_tiledb_array",
user_annotations__hosted_tiledb_array__db_uri=
"postgresql://*****:*****@localhost:5432",
user_annotations__hosted_tiledb_array__hosted_file_directory=tmp_dir,
)
config.complete_config()
data = MatrixDataLoader(data_locator.abspath()).open(config)
annotations = AnnotationsHostedTileDB(
tmp_dir,
DbUtils("postgresql://*****:*****@localhost:5432"),
)
return data, tmp_dir, annotations
def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
tmp_dir = tempfile.mkdtemp()
annotations_file = path.join(tmp_dir, "test_annotations.csv")
if annotations_fixture:
shutil.copyfile(
f"{PROJECT_ROOT}/server/test/fixtures/pbmc3k-annotations.csv",
annotations_file)
fname = {
MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
MatrixDataType.CXG: "test/fixtures/pbmc3k.cxg",
}[ext]
data_locator = DataLocator(fname)
config = AppConfig()
config.update_server_config(
app__flask_secret_key="secret",
single_dataset__obs_names=None,
single_dataset__var_names=None,
single_dataset__datapath=data_locator.path,
)
config.update_default_dataset_config(
embeddings__names=["umap"],
presentation__max_categories=100,
diffexp__lfc_cutoff=0.01,
)
config.complete_config()
data = MatrixDataLoader(data_locator.abspath()).open(config)
annotations = AnnotationsLocalFile(None, annotations_file)
return data, tmp_dir, annotations
def test_handle_data_locator_can_read_from_dataroot(self, mock_discover_region_name):
mock_discover_region_name.return_value = "us-west-2"
dataroot = {
"d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"},
"d2": {"base_url": "set2/subdir", "dataroot": "s3://hosted-cellxgene-dev"},
}
file_name = self.custom_app_config(
dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true"
)
config = AppConfig()
config.update_from_config_file(file_name)
config.server_config.handle_data_locator()
self.assertEqual(config.server_config.data_locator__s3__region_name, "us-west-2")
mock_discover_region_name.assert_called_once_with("s3://hosted-cellxgene-dev")
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
self.dataset_config = self.config.dataset_config
self.config.complete_config()
message_list = []
def noop(message):
message_list.append(message)
messagefn = noop
self.context = dict(messagefn=messagefn, messages=message_list)
def test_auth_test_single(self):
app_config = AppConfig()
app_config.update_server_config(app__flask_secret_key="secret")
app_config.update_server_config(
authentication__type="test",
single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg")
app_config.update_server_config(
authentication__insecure_test_environment=True)
app_config.complete_config()
with test_server(app_config=app_config) as server:
session = requests.Session()
config = session.get(f"{server}/api/v0.2/config").json()
userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(
config["config"]["authentication"]["requires_client_login"])
self.assertTrue(config["config"]["parameters"]["annotations"])
login_uri = config["config"]["authentication"]["login"]
logout_uri = config["config"]["authentication"]["logout"]
self.assertEqual(login_uri, "/login")
self.assertEqual(logout_uri, "/logout")
response = session.get(f"{server}/{login_uri}")
# check that the login redirect worked
self.assertEqual(response.history[0].status_code, 302)
self.assertEqual(response.url, f"{server}/")
config = session.get(f"{server}/api/v0.2/config").json()
userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["username"], "test_account")
self.assertTrue(config["config"]["parameters"]["annotations"])
response = session.get(f"{server}/{logout_uri}")
# check that the logout redirect worked
self.assertEqual(response.history[0].status_code, 302)
self.assertEqual(response.url, f"{server}/")
config = session.get(f"{server}/api/v0.2/config").json()
userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(config["config"]["parameters"]["annotations"])
def get_basic_config(self):
config = AppConfig()
config.update_server_config(
single_dataset__obs_names=None, single_dataset__var_names=None,
)
config.update_server_config(app__flask_secret_key="secret")
config.update_default_dataset_config(
embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01,
)
return config
def main():
parser = argparse.ArgumentParser(
"A script to check hosted configuration files")
parser.add_argument("config_file", help="the configuration file")
parser.add_argument(
"-s",
"--show",
default=False,
action="store_true",
help=
"print the configuration. NOTE: this may print secret values to stdout",
)
args = parser.parse_args()
app_config = AppConfig()
try:
app_config.update_from_config_file(args.config_file)
app_config.complete_config()
except Exception as e:
print(f"Error: {str(e)}")
print("FAIL:", args.config_file)
sys.exit(1)
if args.show:
yaml_config = app_config.config_to_dict()
yaml.dump(yaml_config, sys.stdout)
print("PASS:", args.config_file)
sys.exit(0)
def test_mapping_creation_returns_map_of_server_and_dataset_config(self):
config = AppConfig()
mapping = config.default_dataset_config.create_mapping(
config.default_config)
self.assertIsNotNone(mapping["server__app__verbose"])
self.assertIsNotNone(mapping["dataset__presentation__max_categories"])
self.assertIsNotNone(
mapping["dataset__user_annotations__ontology__obo_location"])
self.assertIsNotNone(
mapping["server__multi_dataset__allowed_matrix_types"])
def test_auth_none(self):
app_config = AppConfig()
app_config.update_server_config(app__flask_secret_key="secret")
app_config.update_server_config(
authentication__type=None,
multi_dataset__dataroot=self.dataset_dataroot)
app_config.update_default_dataset_config(
user_annotations__enable=False)
app_config.complete_config()
with test_server(app_config=app_config) as server:
session = requests.Session()
config = session.get(
f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(
f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertNotIn("authentication", config["config"])
self.assertIsNone(userinfo)
def test_init_datatset_config_sets_vars_from_default_config(self):
config = AppConfig()
self.assertEqual(
config.default_dataset_config.presentation__max_categories, 1000)
self.assertEqual(config.default_dataset_config.user_annotations__type,
"local_file_csv")
self.assertEqual(config.default_dataset_config.diffexp__lfc_cutoff,
0.01)
self.assertIsNone(config.default_dataset_config.
user_annotations__ontology__obo_location)
def test_configfile_with_specialization(self):
# test that per_dataset_config config load the default config, then the specialized config
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
with open(configfile, "w") as fconfig:
config = """
server:
multi_dataset:
dataroot:
test:
base_url: test
dataroot: fake_dataroot
dataset:
user_annotations:
enable: false
type: hosted_tiledb_array
hosted_tiledb_array:
db_uri: fake_db_uri
hosted_file_directory: fake_dir
per_dataset_config:
test:
user_annotations:
enable: true
"""
fconfig.write(config)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
test_config = app_config.dataroot_config["test"]
# test config from default
self.assertEqual(test_config.user_annotations__type,
"hosted_tiledb_array")
self.assertEqual(
test_config.user_annotations__hosted_tiledb_array__db_uri,
"fake_db_uri")
# test config from specialization
self.assertTrue(test_config.user_annotations__enable)
def test_auth_session(self):
app_config = AppConfig()
app_config.update_server_config(app__flask_secret_key="secret")
app_config.update_server_config(
authentication__type="session",
multi_dataset__dataroot=self.dataset_dataroot)
app_config.update_default_dataset_config(user_annotations__enable=True)
app_config.complete_config()
with test_server(app_config=app_config) as server:
session = requests.Session()
config = session.get(
f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(
f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertFalse(
config["config"]["authentication"]["requires_client_login"])
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["username"], "anonymous")
def test_handle_data_locator_works_for_default_types(
self, mock_discover_region_name):
mock_discover_region_name.return_value = None
# Default config
self.assertEqual(
self.config.server_config.data_locator__s3__region_name, None)
# hard coded
config = self.get_config()
self.assertEqual(config.server_config.data_locator__s3__region_name,
"us-east-1")
# incorrectly formatted
datapath = "s3://shouldnt/work"
file_name = self.custom_app_config(
dataset_datapath=datapath,
config_file_name=self.config_file_name,
data_locater_region_name="true")
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_data_locator()
def test_configfile_no_server_section(self):
# test a config file without a dataset section
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
with open(configfile, "w") as fconfig:
config = """
dataset:
user_annotations:
enable: false
"""
fconfig.write(config)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
server_changes = app_config.server_config.changes_from_default()
dataset_changes = app_config.dataset_config.changes_from_default()
self.assertEqual(server_changes, [])
self.assertEqual(dataset_changes,
[("user_annotations__enable", False, True)])
def test_get_api_base_url_works(self):
# test the api_base_url feature, and that it can contain a path
config = AppConfig()
backend_port = find_available_port("localhost", 10000)
config.update_server_config(
app__flask_secret_key="secret",
app__api_base_url=
f"http://localhost:{backend_port}/additional/path",
multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset",
)
config.complete_config()
with test_server(["-p", str(backend_port)],
app_config=config) as server:
session = requests.Session()
self.assertEqual(server, f"http://localhost:{backend_port}")
response = session.get(
f"{server}/additional/path/d/pbmc3k.h5ad/api/v0.2/config")
self.assertEqual(response.status_code, 200)
data_config = response.json()
self.assertEqual(data_config["config"]["displayNames"]["dataset"],
"pbmc3k")
# test the health check at the correct url
response = session.get(f"{server}/additional/path/health")
assert response.json()["status"] == "pass"
def test_config(self):
check_config_script = os.path.join(PROJECT_ROOT, "server", "eb",
"check_config.py")
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
app_config = AppConfig()
app_config.update_server_config(
multi_dataset__dataroot=f"{FIXTURES_ROOT}")
app_config.write_config(configfile)
command = ["python", check_config_script, configfile]
# test failure mode (flask_secret_key not set)
env = os.environ.copy()
env.pop("CXG_SECRET_KEY", None)
with self.assertRaises(
subprocess.CalledProcessError) as exception_context:
subprocess.check_output(command, env=env)
output = str(exception_context.exception.stdout, "utf-8")
self.assertTrue(
output.startswith(
"Error: Invalid type for attribute: app__flask_secret_key, expected type str, got NoneType"
))
self.assertEqual(exception_context.exception.returncode, 1)
# test passing case
env = os.environ.copy()
env["CXG_SECRET_KEY"] = "secret"
output = subprocess.check_output(command, env=env)
output = str(output, "utf-8")
self.assertTrue(output.startswith("PASS"))
def test_handle_data_source__errors_when_passed_zero_or_two_dataroots(self):
file_name = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}",
config_file_name="two_data_roots.yml",
dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
)
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_data_source()
file_name = self.custom_app_config(config_file_name="zero_roots.yml")
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_data_source()
def test_get_default_config_correctly_reads_default_config_file(self):
app_default_config = AppConfig().default_config
expected_config = yaml.load(default_config, Loader=yaml.Loader)
server_config = app_default_config["server"]
dataset_config = app_default_config["dataset"]
expected_server_config = expected_config["server"]
expected_dataset_config = expected_config["dataset"]
self.assertDictEqual(app_default_config, expected_config)
self.assertDictEqual(server_config, expected_server_config)
self.assertDictEqual(dataset_config, expected_dataset_config)
class BaseConfigTest(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
self.server_config = self.config.server_config
self.config.complete_config()
message_list = []
def noop(message):
message_list.append(message)
messagefn = noop
self.context = dict(messagefn=messagefn, messages=message_list)
def get_config(self, **kwargs):
file_name = self.custom_app_config(
dataset_datapath=f"{H5AD_FIXTURE}",
config_file_name=self.config_file_name,
**kwargs)
config = AppConfig()
config.update_from_config_file(file_name)
return config
def test_mapping_creation_returns_map_of_server_and_dataset_config(self):
config = AppConfig()
mapping = config.dataset_config.create_mapping(config.default_config)
self.assertIsNotNone(mapping["server__app__verbose"])
self.assertIsNotNone(mapping["dataset__presentation__max_categories"])
def test_changes_from_default_returns_list_of_nondefault_config_values(
self):
config = self.get_config(verbose="true", lfc_cutoff=0.05)
server_changes = config.server_config.changes_from_default()
dataset_changes = config.dataset_config.changes_from_default()
self.assertEqual(
server_changes,
[
("app__verbose", True, False),
("app__flask_secret_key", "secret", None),
("single_dataset__datapath", H5AD_FIXTURE, None),
("data_locator__s3__region_name", "us-east-1", True),
],
)
self.assertEqual(dataset_changes,
[("diffexp__lfc_cutoff", 0.05, 0.01)])
def test_check_config_throws_error_if_attr_has_not_been_checked(self):
config = self.get_config(verbose="true")
config.complete_config()
config.check_config()
config.update_server_config(app__verbose=False)
with self.assertRaises(ConfigurationError):
config.check_config()
