def setUp(self):
self.data_file = DataLocator(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
config = AppConfig()
config.update_server_config(single_dataset__datapath=self.data_file.path)
config.update_server_config(app__flask_secret_key="secret")
config.complete_config()
self.data = AnndataAdaptor(self.data_file, config)
def main():
parser = argparse.ArgumentParser(
"A script to check hosted configuration files")
parser.add_argument("config_file", help="the configuration file")
parser.add_argument(
"-s",
"--show",
default=False,
action="store_true",
help=
"print the configuration. NOTE: this may print secret values to stdout",
)
args = parser.parse_args()
app_config = AppConfig()
try:
app_config.update_from_config_file(args.config_file)
app_config.complete_config()
except Exception as e:
print(f"Error: {str(e)}")
print("FAIL:", args.config_file)
sys.exit(1)
if args.show:
yaml_config = app_config.config_to_dict()
yaml.dump(yaml_config, sys.stdout)
print("PASS:", args.config_file)
sys.exit(0)
def data_with_tmp_tiledb_annotations(ext: MatrixDataType):
tmp_dir = tempfile.mkdtemp()
fname = {
MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
MatrixDataType.CXG: "test/fixtures/pbmc3k.cxg",
}[ext]
data_locator = DataLocator(fname)
config = AppConfig()
config.update_server_config(
app__flask_secret_key="secret",
multi_dataset__dataroot=data_locator.path,
authentication__type="test",
authentication__insecure_test_environment=True,
)
config.update_default_dataset_config(
embeddings__names=["umap"],
presentation__max_categories=100,
diffexp__lfc_cutoff=0.01,
user_annotations__type="hosted_tiledb_array",
user_annotations__hosted_tiledb_array__db_uri=
"postgresql://*****:*****@localhost:5432",
user_annotations__hosted_tiledb_array__hosted_file_directory=tmp_dir,
)
config.complete_config()
data = MatrixDataLoader(data_locator.abspath()).open(config)
annotations = AnnotationsHostedTileDB(
tmp_dir,
DbUtils("postgresql://*****:*****@localhost:5432"),
)
return data, tmp_dir, annotations
def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
tmp_dir = tempfile.mkdtemp()
annotations_file = path.join(tmp_dir, "test_annotations.csv")
if annotations_fixture:
shutil.copyfile(
f"{PROJECT_ROOT}/server/test/fixtures/pbmc3k-annotations.csv",
annotations_file)
fname = {
MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
MatrixDataType.CXG: "test/fixtures/pbmc3k.cxg",
}[ext]
data_locator = DataLocator(fname)
config = AppConfig()
config.update_server_config(
app__flask_secret_key="secret",
single_dataset__obs_names=None,
single_dataset__var_names=None,
single_dataset__datapath=data_locator.path,
)
config.update_default_dataset_config(
embeddings__names=["umap"],
presentation__max_categories=100,
diffexp__lfc_cutoff=0.01,
)
config.complete_config()
data = MatrixDataLoader(data_locator.abspath()).open(config)
annotations = AnnotationsLocalFile(None, annotations_file)
return data, tmp_dir, annotations
def test_get_api_base_url_works(self):
# test the api_base_url feature, and that it can contain a path
config = AppConfig()
backend_port = find_available_port("localhost", 10000)
config.update_server_config(
app__flask_secret_key="secret",
app__api_base_url=
f"http://localhost:{backend_port}/additional/path",
multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset",
)
config.complete_config()
with test_server(["-p", str(backend_port)],
app_config=config) as server:
session = requests.Session()
self.assertEqual(server, f"http://localhost:{backend_port}")
response = session.get(
f"{server}/additional/path/d/pbmc3k.h5ad/api/v0.2/config")
self.assertEqual(response.status_code, 200)
data_config = response.json()
self.assertEqual(data_config["config"]["displayNames"]["dataset"],
"pbmc3k")
# test the health check at the correct url
response = session.get(f"{server}/additional/path/health")
assert response.json()["status"] == "pass"
class BaseConfigTest(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
self.server_config = self.config.server_config
self.config.complete_config()
message_list = []
def noop(message):
message_list.append(message)
messagefn = noop
self.context = dict(messagefn=messagefn, messages=message_list)
def get_config(self, **kwargs):
file_name = self.custom_app_config(
dataset_datapath=f"{H5AD_FIXTURE}",
config_file_name=self.config_file_name,
**kwargs)
config = AppConfig()
config.update_from_config_file(file_name)
return config
def test_mapping_creation_returns_map_of_server_and_dataset_config(self):
config = AppConfig()
mapping = config.dataset_config.create_mapping(config.default_config)
self.assertIsNotNone(mapping["server__app__verbose"])
self.assertIsNotNone(mapping["dataset__presentation__max_categories"])
def test_changes_from_default_returns_list_of_nondefault_config_values(
self):
config = self.get_config(verbose="true", lfc_cutoff=0.05)
server_changes = config.server_config.changes_from_default()
dataset_changes = config.dataset_config.changes_from_default()
self.assertEqual(
server_changes,
[
("app__verbose", True, False),
("app__flask_secret_key", "secret", None),
("single_dataset__datapath", H5AD_FIXTURE, None),
("data_locator__s3__region_name", "us-east-1", True),
],
)
self.assertEqual(dataset_changes,
[("diffexp__lfc_cutoff", 0.05, 0.01)])
def test_check_config_throws_error_if_attr_has_not_been_checked(self):
config = self.get_config(verbose="true")
config.complete_config()
config.check_config()
config.update_server_config(app__verbose=False)
with self.assertRaises(ConfigurationError):
config.check_config()
def test_aws_secrets_manager(self, mock_get_secret_key):
mock_get_secret_key.return_value = {
"oauth_client_secret": "mock_oauth_secret",
"db_uri": "mock_db_uri",
}
configfile = self.custom_external_config(
aws_secrets_manager_region="us-west-2",
aws_secrets_manager_secrets=[
dict(
name="my_secret",
values=[
dict(key="flask_secret_key",
path=["server", "app", "flask_secret_key"],
required=False),
dict(
key="db_uri",
path=[
"dataset", "user_annotations",
"hosted_tiledb_array", "db_uri"
],
required=True,
),
dict(
key="oauth_client_secret",
path=[
"server", "authentication", "params_oauth",
"client_secret"
],
required=True,
),
],
)
],
config_file_name="secret_external_config.yaml",
)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k.cxg"
app_config.server_config.app__flask_secret_key = "original"
app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k.cxg"
app_config.complete_config()
self.assertEqual(app_config.server_config.app__flask_secret_key,
"original")
self.assertEqual(
app_config.server_config.
authentication__params_oauth__client_secret, "mock_oauth_secret")
self.assertEqual(
app_config.default_dataset_config.
user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")
def test_auth_test_single(self):
app_config = AppConfig()
app_config.update_server_config(app__flask_secret_key="secret")
app_config.update_server_config(
authentication__type="test",
single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg")
app_config.update_server_config(
authentication__insecure_test_environment=True)
app_config.complete_config()
with test_server(app_config=app_config) as server:
session = requests.Session()
config = session.get(f"{server}/api/v0.2/config").json()
userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(
config["config"]["authentication"]["requires_client_login"])
self.assertTrue(config["config"]["parameters"]["annotations"])
login_uri = config["config"]["authentication"]["login"]
logout_uri = config["config"]["authentication"]["logout"]
self.assertEqual(login_uri, "/login")
self.assertEqual(logout_uri, "/logout")
response = session.get(f"{server}/{login_uri}")
# check that the login redirect worked
self.assertEqual(response.history[0].status_code, 302)
self.assertEqual(response.url, f"{server}/")
config = session.get(f"{server}/api/v0.2/config").json()
userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["username"], "test_account")
self.assertTrue(config["config"]["parameters"]["annotations"])
response = session.get(f"{server}/{logout_uri}")
# check that the logout redirect worked
self.assertEqual(response.history[0].status_code, 302)
self.assertEqual(response.url, f"{server}/")
config = session.get(f"{server}/api/v0.2/config").json()
userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(config["config"]["parameters"]["annotations"])
def test_auth_none(self):
app_config = AppConfig()
app_config.update_server_config(app__flask_secret_key="secret")
app_config.update_server_config(
authentication__type=None,
multi_dataset__dataroot=self.dataset_dataroot)
app_config.update_default_dataset_config(
user_annotations__enable=False)
app_config.complete_config()
with test_server(app_config=app_config) as server:
session = requests.Session()
config = session.get(
f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(
f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertNotIn("authentication", config["config"])
self.assertIsNone(userinfo)
def test_auth_session(self):
app_config = AppConfig()
app_config.update_server_config(app__flask_secret_key="secret")
app_config.update_server_config(
authentication__type="session",
multi_dataset__dataroot=self.dataset_dataroot)
app_config.update_default_dataset_config(user_annotations__enable=True)
app_config.complete_config()
with test_server(app_config=app_config) as server:
session = requests.Session()
config = session.get(
f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(
f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertFalse(
config["config"]["authentication"]["requires_client_login"])
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["username"], "anonymous")
def app_config(data_locator,
backed=False,
extra_server_config={},
extra_dataset_config={}):
config = AppConfig()
config.update_server_config(
app__flask_secret_key="secret",
single_dataset__obs_names=None,
single_dataset__var_names=None,
adaptor__anndata_adaptor__backed=backed,
single_dataset__datapath=data_locator,
limits__diffexp_cellcount_max=None,
limits__column_request_max=None,
)
config.update_dataset_config(embeddings__names=["umap", "tsne", "pca"],
presentation__max_categories=100,
diffexp__lfc_cutoff=0.01)
config.update_server_config(**extra_server_config)
config.update_dataset_config(**extra_dataset_config)
config.complete_config()
return config
def test_run(self):
tempdir = tempfile.TemporaryDirectory(dir=f"{PROJECT_ROOT}/server")
tempdirname = tempdir.name
config = AppConfig()
# test that eb works
config.update_server_config(multi_dataset__dataroot=f"{FIXTURES_ROOT}",
app__flask_secret_key="open sesame")
config.complete_config()
config.write_config(f"{tempdirname}/config.yaml")
subprocess.check_call(f"git ls-files . | cpio -pdm {tempdirname}",
cwd=f"{PROJECT_ROOT}/server/eb",
shell=True)
subprocess.check_call(["make", "build"], cwd=tempdirname)
with run_eb_app(tempdirname) as server:
session = requests.Session()
r = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config")
data_config = r.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
def test_environment_variable_errors(self):
# no name
app_config = AppConfig()
app_config.external_config.environment = [
dict(required=True, path=["this", "is", "a", "path"])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message,
"environment: 'name' is missing")
# required has wrong type
app_config = AppConfig()
app_config.external_config.environment = [
dict(name="myenvar",
required="optional",
path=["this", "is", "a", "path"])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message,
"environment: 'required' must be a bool")
# no path
app_config = AppConfig()
app_config.external_config.environment = [
dict(name="myenvar", required=True)
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message,
"environment: 'path' is missing")
# required environment variable is not set
app_config = AppConfig()
app_config.external_config.environment = [
dict(name="THIS_ENV_IS_NOT_SET",
required=True,
path=["this", "is", "a", "path"])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(
config_error.exception.message,
"required environment variable 'THIS_ENV_IS_NOT_SET' not set")
def test_aws_secrets_manager_error(self, mock_get_secret_key):
mock_get_secret_key.return_value = {
"oauth_client_secret": "mock_oauth_secret",
"db_uri": "mock_db_uri",
}
# no region
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = None
app_config.external_config.aws_secrets_manager__secrets = [
dict(name="secret1",
values=[
dict(key="key1",
required=True,
path=["this", "is", "my", "path"])
])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(
config_error.exception.message,
"Invalid type for attribute: aws_secrets_manager__region, expected type str, got NoneType",
)
# missing secret name
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(values=[
dict(key="db_uri",
required=True,
path=["this", "is", "my", "path"])
])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message,
"aws_secrets_manager: 'name' is missing")
# secret name wrong type
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(name=1,
values=[
dict(key="db_uri",
required=True,
path=["this", "is", "my", "path"])
])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message,
"aws_secrets_manager: 'name' must be a string")
# missing values name
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(name="mysecret")
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message,
"aws_secrets_manager: 'values' is missing")
# values wrong type
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(name="mysecret",
values=dict(key="db_uri",
required=True,
path=["this", "is", "my", "path"]))
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message,
"aws_secrets_manager: 'values' must be a list")
# entry missing key
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(name="mysecret",
values=[
dict(required=True, path=["this", "is", "my", "path"])
])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message,
"missing 'key' in secret values: mysecret")
# entry required is wrong type
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(name="mysecret",
values=[
dict(key="db_uri",
required="optional",
path=["this", "is", "my", "path"])
])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(
config_error.exception.message,
"wrong type for 'required' in secret values: mysecret")
# entry missing path
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(name="mysecret", values=[dict(key="db_uri", required=True)])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message,
"missing 'path' in secret values: mysecret")
# secret missing required key
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(
name="mysecret",
values=[
dict(key="KEY_DOES_NOT_EXIST",
required=True,
path=["this", "is", "a", "path"])
],
)
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(
config_error.exception.message,
"required secret 'mysecret:KEY_DOES_NOT_EXIST' not set")
class TestServerConfig(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
self.server_config = self.config.server_config
self.config.complete_config()
message_list = []
def noop(message):
message_list.append(message)
messagefn = noop
self.context = dict(messagefn=messagefn, messages=message_list)
def get_config(self, **kwargs):
file_name = self.custom_app_config(
dataset_datapath=f"{H5AD_FIXTURE}",
config_file_name=self.config_file_name,
**kwargs)
config = AppConfig()
config.update_from_config_file(file_name)
return config
def test_init_raises_error_if_default_config_is_invalid(self):
invalid_config = self.get_config(port="not_valid")
with self.assertRaises(ConfigurationError):
invalid_config.complete_config()
@patch(
"server.common.config.server_config.BaseConfig.validate_correct_type_of_configuration_attribute"
)
def test_complete_config_checks_all_attr(self, mock_check_attrs):
mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute(
)
self.server_config.complete_config(self.context)
self.assertEqual(mock_check_attrs.call_count, 19)
def test_handle_app__throws_error_if_port_doesnt_exist(self):
config = self.get_config(port=99999999)
with self.assertRaises(ConfigurationError):
config.server_config.handle_app(self.context)
@patch("server.common.config.server_config.discover_s3_region_name")
def test_handle_data_locator_works_for_default_types(
self, mock_discover_region_name):
mock_discover_region_name.return_value = None
# Default config
self.assertEqual(
self.config.server_config.data_locator__s3__region_name, None)
# hard coded
config = self.get_config()
self.assertEqual(config.server_config.data_locator__s3__region_name,
"us-east-1")
# incorrectly formatted
datapath = "s3://shouldnt/work"
file_name = self.custom_app_config(
dataset_datapath=datapath,
config_file_name=self.config_file_name,
data_locater_region_name="true")
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_data_locator()
def test_handle_app___can_use_envar_port(self):
config = self.get_config(port=24)
self.assertEqual(config.server_config.app__port, 24)
# Note if the port is set in the config file it will NOT be overwritten by a different envvar
os.environ["CXG_SERVER_PORT"] = "4008"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.server_config.handle_app(self.context)
self.assertEqual(self.config.server_config.app__port, 4008)
del os.environ["CXG_SERVER_PORT"]
def test_handle_app__can_get_secret_key_from_envvar_or_config_file_with_envvar_given_preference(
self):
config = self.get_config(flask_secret_key="KEY_FROM_FILE")
self.assertEqual(config.server_config.app__flask_secret_key,
"KEY_FROM_FILE")
os.environ["CXG_SECRET_KEY"] = "KEY_FROM_ENV"
config.external_config.handle_environment(self.context)
self.assertEqual(config.server_config.app__flask_secret_key,
"KEY_FROM_ENV")
def test_config_for_single_dataset(self):
file_name = self.custom_app_config(
config_file_name="single_dataset.yml",
dataset_datapath=f"{H5AD_FIXTURE}")
config = AppConfig()
config.update_from_config_file(file_name)
config.server_config.handle_single_dataset(self.context)
file_name = self.custom_app_config(
config_file_name="single_dataset_with_about.yml",
about="www.cziscience.com",
dataset_datapath=f"{H5AD_FIXTURE}",
)
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_single_dataset(self.context)
class TestDatasetConfig(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
self.dataset_config = self.config.dataset_config
self.config.complete_config()
message_list = []
def noop(message):
message_list.append(message)
messagefn = noop
self.context = dict(messagefn=messagefn, messages=message_list)
def get_config(self, **kwargs):
file_name = self.custom_app_config(dataset_datapath=H5AD_FIXTURE, **kwargs)
config = AppConfig()
config.update_from_config_file(file_name)
return config
def test_init_datatset_config_sets_vars_from_config(self):
config = AppConfig()
self.assertEqual(config.dataset_config.presentation__max_categories, 1000)
self.assertEqual(config.dataset_config.user_annotations__type, "local_file_csv")
self.assertEqual(config.dataset_config.diffexp__lfc_cutoff, 0.01)
@patch("server.common.config.dataset_config.BaseConfig.validate_correct_type_of_configuration_attribute")
def test_complete_config_checks_all_attr(self, mock_check_attrs):
mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
self.dataset_config.complete_config(self.context)
self.assertIsNotNone(self.config.server_config.data_adaptor)
self.assertEqual(mock_check_attrs.call_count, 16)
def test_app_sets_script_vars(self):
config = self.get_config(scripts=["path/to/script"])
config.dataset_config.handle_app()
self.assertEqual(config.dataset_config.app__scripts, [{"src": "path/to/script"}])
config = self.get_config(scripts=[{"src": "path/to/script", "more": "different/script/path"}])
config.dataset_config.handle_app()
self.assertEqual(
config.dataset_config.app__scripts, [{"src": "path/to/script", "more": "different/script/path"}]
)
config = self.get_config(scripts=["path/to/script", "different/script/path"])
config.dataset_config.handle_app()
# TODO @madison -- is this the desired functionality?
self.assertEqual(
config.dataset_config.app__scripts, [{"src": "path/to/script"}, {"src": "different/script/path"}]
)
config = self.get_config(scripts=[{"more": "different/script/path"}])
with self.assertRaises(ConfigurationError):
config.dataset_config.handle_app()
def test_handle_user_annotations__instantiates_user_annotations_class_correctly(self):
config = self.get_config(
enable_users_annotations="true", annotation_type="local_file_csv"
)
config.server_config.complete_config(self.context)
config.dataset_config.handle_user_annotations(self.context)
self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile)
config = self.get_config(
enable_users_annotations="true", annotation_type="NOT_REAL"
)
config.server_config.complete_config(self.context)
with self.assertRaises(ConfigurationError):
config.dataset_config.handle_user_annotations(self.context)
def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in(self):
config = self.get_config(
enable_users_annotations="true", annotation_type="local_file_csv"
)
config.server_config.complete_config(self.context)
config.dataset_config.handle_local_file_csv_annotations(self.context)
self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile)
cwd = os.getcwd()
self.assertEqual(config.dataset_config.user_annotations._get_output_dir(), cwd)
def test_handle_diffexp__raises_warning_for_large_datasets(self):
config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15)
config.server_config.complete_config(self.context)
config.dataset_config.handle_diffexp(self.context)
self.assertEqual(len(self.context["messages"]), 1)
def test_configfile_with_specialization(self):
# test that per_dataset_config config load the default config, then the specialized config
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
with open(configfile, "w") as fconfig:
config = """
server:
single_dataset:
datapath: fake_datapath
dataset:
user_annotations:
enable: false
type: local_file_csv
local_file_csv:
file: fake_file
directory: fake_dir
"""
fconfig.write(config)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
test_config = app_config.dataset_config
# test config from default
self.assertEqual(test_config.user_annotations__type, "local_file_csv")
self.assertEqual(test_config.user_annotations__local_file_csv__file, "fake_file")
def test_multi_dataset(self):
config = AppConfig()
# test for illegal url_dataroots
for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
config.update_server_config(
app__flask_secret_key="secret",
multi_dataset__dataroot={
"tag": {
"base_url": illegal,
"dataroot": f"{PROJECT_ROOT}/example-dataset"
}
},
)
with self.assertRaises(ConfigurationError):
config.complete_config()
# test for legal url_dataroots
for legal in ("d", "this.is-okay_", "a/b"):
config.update_server_config(
app__flask_secret_key="secret",
multi_dataset__dataroot={
"tag": {
"base_url": legal,
"dataroot": f"{PROJECT_ROOT}/example-dataset"
}
},
)
config.complete_config()
# test that multi dataroots work end to end
config.update_server_config(
app__flask_secret_key="secret",
multi_dataset__dataroot=dict(
s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset",
base_url="set1/1/2"),
s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
),
)
# Change this default to test if the dataroot overrides below work.
config.update_default_dataset_config(
app__about_legal_tos="tos_default.html")
# specialize the configs for set1
config.add_dataroot_config("s1",
user_annotations__enable=False,
diffexp__enable=True,
app__about_legal_tos="tos_set1.html")
# specialize the configs for set2
config.add_dataroot_config("s2",
user_annotations__enable=True,
diffexp__enable=False,
app__about_legal_tos="tos_set2.html")
# no specializations for set3 (they get the default dataset config)
config.complete_config()
with test_server(app_config=config) as server:
session = requests.Session()
response = session.get(
f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is False
assert data_config["config"]["parameters"][
"disable-diffexp"] is False
assert data_config["config"]["parameters"][
"about_legal_tos"] == "tos_set1.html"
response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"][
"about_legal_tos"] == "tos_set2.html"
response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"][
"disable-diffexp"] is False
assert data_config["config"]["parameters"][
"about_legal_tos"] == "tos_default.html"
response = session.get(f"{server}/health")
assert response.json()["status"] == "pass"
class AppConfigTest(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
self.server_config = self.config.server_config
self.config.complete_config()
message_list = []
def noop(message):
message_list.append(message)
messagefn = noop
self.context = dict(messagefn=messagefn, messages=message_list)
def get_config(self, **kwargs):
file_name = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}",
config_file_name=self.config_file_name,
**kwargs)
config = AppConfig()
config.update_from_config_file(file_name)
return config
def test_get_default_config_correctly_reads_default_config_file(self):
app_default_config = AppConfig().default_config
expected_config = yaml.load(default_config, Loader=yaml.Loader)
server_config = app_default_config["server"]
dataset_config = app_default_config["dataset"]
expected_server_config = expected_config["server"]
expected_dataset_config = expected_config["dataset"]
self.assertDictEqual(app_default_config, expected_config)
self.assertDictEqual(server_config, expected_server_config)
self.assertDictEqual(dataset_config, expected_dataset_config)
def test_get_dataset_config_returns_default_dataset_config_for_single_datasets(
self):
datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad"
file_name = self.custom_app_config(
dataset_datapath=datapath, config_file_name=self.config_file_name)
config = AppConfig()
config.update_from_config_file(file_name)
self.assertEqual(config.get_dataset_config(""),
config.default_dataset_config)
def test_update_server_config_updates_server_config_and_config_status(
self):
config = self.get_config()
config.complete_config()
config.check_config()
config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
with self.assertRaises(ConfigurationError):
config.server_config.check_config()
def test_write_config_outputs_yaml_with_all_config_vars(self):
config = self.get_config()
config.write_config(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml")
with open(f"{FIXTURES_ROOT}/tmp_dir/{self.config_file_name}",
"r") as default_config:
default_config_yml = yaml.safe_load(default_config)
with open(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml",
"r") as output_config:
output_config_yml = yaml.safe_load(output_config)
self.maxDiff = None
self.assertEqual(default_config_yml, output_config_yml)
def test_update_app_config(self):
config = AppConfig()
config.update_server_config(app__verbose=True,
multi_dataset__dataroot="datadir")
vars = config.server_config.changes_from_default()
self.assertCountEqual(vars,
[("app__verbose", True, False),
("multi_dataset__dataroot", "datadir", None)])
config = AppConfig()
config.update_default_dataset_config(app__scripts=(),
app__inline_scripts=())
vars = config.server_config.changes_from_default()
self.assertCountEqual(vars, [])
config = AppConfig()
config.update_default_dataset_config(app__scripts=[],
app__inline_scripts=[])
vars = config.default_dataset_config.changes_from_default()
self.assertCountEqual(vars, [])
config = AppConfig()
config.update_default_dataset_config(app__scripts=("a", "b"),
app__inline_scripts=["c", "d"])
vars = config.default_dataset_config.changes_from_default()
self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []),
("app__inline_scripts", ["c", "d"], [])])
def test_configfile_no_dataset_section(self):
# test a config file without a dataset section
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
with open(configfile, "w") as fconfig:
config = """
server:
app:
flask_secret_key: secret
multi_dataset:
dataroot: test_dataroot
"""
fconfig.write(config)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
server_changes = app_config.server_config.changes_from_default()
dataset_changes = app_config.default_dataset_config.changes_from_default(
)
self.assertEqual(
server_changes,
[("app__flask_secret_key", "secret", None),
("multi_dataset__dataroot", "test_dataroot", None)],
)
self.assertEqual(dataset_changes, [])
def test_configfile_no_server_section(self):
# test a config file without a dataset section
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
with open(configfile, "w") as fconfig:
config = """
dataset:
user_annotations:
enable: false
"""
fconfig.write(config)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
server_changes = app_config.server_config.changes_from_default()
dataset_changes = app_config.default_dataset_config.changes_from_default(
)
self.assertEqual(server_changes, [])
self.assertEqual(dataset_changes,
[("user_annotations__enable", False, True)])
def test_simple_update_single_config_from_path_and_value(self):
"""Update a simple config parameter"""
config = AppConfig()
config.server_config.multi_dataset__dataroot = dict(
s1=dict(dataroot="my_dataroot_s1", base_url="my_baseurl_s1"),
s2=dict(dataroot="my_dataroot_s2", base_url="my_baseurl_s2"),
)
config.add_dataroot_config("s1")
config.add_dataroot_config("s2")
# test simple value in server
config.update_single_config_from_path_and_value(
["server", "app", "flask_secret_key"], "mysecret")
self.assertEqual(config.server_config.app__flask_secret_key,
"mysecret")
# test simple value in default dataset
config.update_single_config_from_path_and_value(
["dataset", "user_annotations", "hosted_tiledb_array", "db_uri"],
"mydburi",
)
self.assertEqual(
config.default_dataset_config.
user_annotations__hosted_tiledb_array__db_uri, "mydburi")
self.assertEqual(
config.dataroot_config["s1"].
user_annotations__hosted_tiledb_array__db_uri, "mydburi")
self.assertEqual(
config.dataroot_config["s2"].
user_annotations__hosted_tiledb_array__db_uri, "mydburi")
# test simple value in specific dataset
config.update_single_config_from_path_and_value([
"per_dataset_config", "s1", "user_annotations",
"hosted_tiledb_array", "db_uri"
], "s1dburi")
self.assertEqual(
config.default_dataset_config.
user_annotations__hosted_tiledb_array__db_uri, "mydburi")
self.assertEqual(
config.dataroot_config["s1"].
user_annotations__hosted_tiledb_array__db_uri, "s1dburi")
self.assertEqual(
config.dataroot_config["s2"].
user_annotations__hosted_tiledb_array__db_uri, "mydburi")
# error checking
bad_paths = [
(
["dataset", "does", "not", "exist"],
"unknown config parameter at path: '['dataset', 'does', 'not', 'exist']'",
),
(["does", "not", "exist"],
"path must start with 'server', 'dataset', or 'per_dataset_config'"
),
([],
"path must start with 'server', 'dataset', or 'per_dataset_config'"
),
(["per_dataset_config"],
"missing dataroot when using per_dataset_config: got '['per_dataset_config']'"
),
(
["per_dataset_config", "unknown"],
"unknown dataroot when using per_dataset_config: got '['per_dataset_config', 'unknown']',"
" dataroots specified in config are ['s1', 's2']",
),
([1, 2, 3], "path must be a list of strings, got '[1, 2, 3]'"),
("string", "path must be a list of strings, got 'string'"),
]
for bad_path, error_message in bad_paths:
with self.assertRaises(ConfigurationError) as config_error:
config.update_single_config_from_path_and_value(
bad_path, "value")
self.assertEqual(config_error.exception.message, error_message)
def test_dict_update_single_config_from_path_and_value(self):
"""Update a config parameter that has a value of dict"""
# the path leads to a dict config param, set the config parameter to the new value
config = AppConfig()
config.update_single_config_from_path_and_value(
["server", "authentication", "params_oauth", "cookie"],
dict(key="mykey1", max_age=100))
self.assertEqual(
config.server_config.authentication__params_oauth__cookie,
dict(key="mykey1", max_age=100))
# the path leads to an entry within a dict config param, the value is simple
config = AppConfig()
config.server_config.authentication__params_oauth__cookie = dict(
key="mykey1", max_age=100)
config.update_single_config_from_path_and_value(
["server", "authentication", "params_oauth", "cookie", "httponly"],
True,
)
self.assertEqual(
config.server_config.authentication__params_oauth__cookie,
dict(key="mykey1", max_age=100, httponly=True))
class TestDatasetConfig(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
self.dataset_config = self.config.default_dataset_config
self.config.complete_config()
message_list = []
def noop(message):
message_list.append(message)
messagefn = noop
self.context = dict(messagefn=messagefn, messages=message_list)
def get_config(self, **kwargs):
file_name = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}",
config_file_name=self.config_file_name,
**kwargs)
config = AppConfig()
config.update_from_config_file(file_name)
return config
def test_init_datatset_config_sets_vars_from_default_config(self):
config = AppConfig()
self.assertEqual(
config.default_dataset_config.presentation__max_categories, 1000)
self.assertEqual(config.default_dataset_config.user_annotations__type,
"local_file_csv")
self.assertEqual(config.default_dataset_config.diffexp__lfc_cutoff,
0.01)
self.assertIsNone(config.default_dataset_config.
user_annotations__ontology__obo_location)
@patch(
"server.common.config.dataset_config.BaseConfig.validate_correct_type_of_configuration_attribute"
)
def test_complete_config_checks_all_attr(self, mock_check_attrs):
mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute(
)
self.dataset_config.complete_config(self.context)
self.assertEqual(mock_check_attrs.call_count, 21)
def test_app_sets_script_vars(self):
config = self.get_config(scripts=["path/to/script"])
config.default_dataset_config.handle_app()
self.assertEqual(config.default_dataset_config.app__scripts,
[{
"src": "path/to/script"
}])
config = self.get_config(scripts=[{
"src": "path/to/script",
"more": "different/script/path"
}])
config.default_dataset_config.handle_app()
self.assertEqual(config.default_dataset_config.app__scripts,
[{
"src": "path/to/script",
"more": "different/script/path"
}])
config = self.get_config(
scripts=["path/to/script", "different/script/path"])
config.default_dataset_config.handle_app()
# TODO @madison -- is this the desired functionality?
self.assertEqual(config.default_dataset_config.app__scripts,
[{
"src": "path/to/script"
}, {
"src": "different/script/path"
}])
config = self.get_config(scripts=[{"more": "different/script/path"}])
with self.assertRaises(ConfigurationError):
config.default_dataset_config.handle_app()
def test_handle_user_annotations_ensures_auth_is_enabled_with_valid_auth_type(
self):
config = self.get_config(enable_users_annotations="true",
authentication_enable="false")
config.server_config.complete_config(self.context)
with self.assertRaises(ConfigurationError):
config.default_dataset_config.handle_user_annotations(self.context)
config = self.get_config(enable_users_annotations="true",
authentication_enable="true",
auth_type="pretend")
with self.assertRaises(ConfigurationError):
config.server_config.complete_config(self.context)
def test_handle_user_annotations__adds_warning_message_if_annotation_vars_set_when_annotations_disabled(
self):
config = self.get_config(enable_users_annotations="false",
authentication_enable="false",
db_uri="shouldnt/be/set")
config.default_dataset_config.handle_user_annotations(self.context)
self.assertEqual(
self.context["messages"],
["Warning: db_uri ignored as annotations are disabled."])
@patch("server.common.config.dataset_config.DbUtils")
def test_handle_user_annotations__instantiates_user_annotations_class_correctly(
self, mock_db_utils):
mock_db_utils.return_value = "123"
config = self.get_config(enable_users_annotations="true",
authentication_enable="true",
annotation_type="local_file_csv")
config.server_config.complete_config(self.context)
config.default_dataset_config.handle_user_annotations(self.context)
self.assertIsInstance(config.default_dataset_config.user_annotations,
AnnotationsLocalFile)
config = self.get_config(
enable_users_annotations="true",
authentication_enable="true",
annotation_type="hosted_tiledb_array",
db_uri="gotta/set/this",
hosted_file_directory="and/this",
)
config.server_config.complete_config(self.context)
config.default_dataset_config.handle_user_annotations(self.context)
self.assertIsInstance(config.default_dataset_config.user_annotations,
AnnotationsHostedTileDB)
config = self.get_config(enable_users_annotations="true",
authentication_enable="true",
annotation_type="NOT_REAL")
config.server_config.complete_config(self.context)
with self.assertRaises(ConfigurationError):
config.default_dataset_config.handle_user_annotations(self.context)
def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in(
self):
config = self.get_config(enable_users_annotations="true",
authentication_enable="true",
annotation_type="local_file_csv")
config.server_config.complete_config(self.context)
config.default_dataset_config.handle_local_file_csv_annotations()
self.assertIsInstance(config.default_dataset_config.user_annotations,
AnnotationsLocalFile)
cwd = os.getcwd()
self.assertEqual(
config.default_dataset_config.user_annotations._get_output_dir(),
cwd)
def test_handle_embeddings__checks_data_file_types(self):
file_name = self.custom_app_config(
embedding_names=["name1", "name2"],
enable_reembedding="true",
dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
anndata_backed="true",
config_file_name=self.config_file_name,
)
config = AppConfig()
config.update_from_config_file(file_name)
config.server_config.complete_config(self.context)
with self.assertRaises(ConfigurationError):
config.default_dataset_config.handle_embeddings()
def test_handle_diffexp__raises_warning_for_large_datasets(self):
config = self.get_config(lfc_cutoff=0.02,
enable_difexp="true",
top_n=15)
config.server_config.complete_config(self.context)
config.default_dataset_config.handle_diffexp(self.context)
self.assertEqual(len(self.context["messages"]), 0)
def test_multi_dataset(self):
config = AppConfig()
# test for illegal url_dataroots
for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
config.update_server_config(
app__flask_secret_key="secret",
multi_dataset__dataroot={
"tag": {
"base_url": illegal,
"dataroot": f"{PROJECT_ROOT}/example-dataset"
}
},
)
with self.assertRaises(ConfigurationError):
config.complete_config()
# test for legal url_dataroots
for legal in ("d", "this.is-okay_", "a/b"):
config.update_server_config(
app__flask_secret_key="secret",
multi_dataset__dataroot={
"tag": {
"base_url": legal,
"dataroot": f"{PROJECT_ROOT}/example-dataset"
}
},
)
config.complete_config()
# test that multi dataroots work end to end
config.update_server_config(
app__flask_secret_key="secret",
multi_dataset__dataroot=dict(
s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset",
base_url="set1/1/2"),
s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
),
)
# Change this default to test if the dataroot overrides below work.
config.update_default_dataset_config(
app__about_legal_tos="tos_default.html")
# specialize the configs for set1
config.add_dataroot_config("s1",
user_annotations__enable=False,
diffexp__enable=True,
app__about_legal_tos="tos_set1.html")
# specialize the configs for set2
config.add_dataroot_config("s2",
user_annotations__enable=True,
diffexp__enable=False,
app__about_legal_tos="tos_set2.html")
# no specializations for set3 (they get the default dataset config)
config.complete_config()
with test_server(app_config=config) as server:
session = requests.Session()
response = session.get(
f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is False
assert data_config["config"]["parameters"][
"disable-diffexp"] is False
assert data_config["config"]["parameters"][
"about_legal_tos"] == "tos_set1.html"
response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"][
"about_legal_tos"] == "tos_set2.html"
response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"][
"disable-diffexp"] is False
assert data_config["config"]["parameters"][
"about_legal_tos"] == "tos_default.html"
response = session.get(f"{server}/health")
assert response.json()["status"] == "pass"
def test_configfile_with_specialization(self):
# test that per_dataset_config config load the default config, then the specialized config
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
with open(configfile, "w") as fconfig:
config = """
server:
multi_dataset:
dataroot:
test:
base_url: test
dataroot: fake_dataroot
dataset:
user_annotations:
enable: false
type: hosted_tiledb_array
hosted_tiledb_array:
db_uri: fake_db_uri
hosted_file_directory: fake_dir
per_dataset_config:
test:
user_annotations:
enable: true
"""
fconfig.write(config)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
test_config = app_config.dataroot_config["test"]
# test config from default
self.assertEqual(test_config.user_annotations__type,
"hosted_tiledb_array")
self.assertEqual(
test_config.user_annotations__hosted_tiledb_array__db_uri,
"fake_db_uri")
# test config from specialization
self.assertTrue(test_config.user_annotations__enable)
logging.critical("No config file found")
sys.exit(1)
dataroot = os.getenv("CXG_DATAROOT")
if dataroot:
logging.info("Configuration from CXG_DATAROOT")
app_config.update_server_config(multi_dataset__dataroot=dataroot)
# overwrite configuration for the eb app
app_config.update_default_dataset_config(
embeddings__enable_reembedding=False, )
app_config.update_server_config(
multi_dataset__allowed_matrix_types=["cxg"], )
# complete config
app_config.complete_config(logging.info)
server = WSGIServer(app_config)
debug = False
application = server.app
except Exception:
logging.critical("Caught exception during initialization", exc_info=True)
sys.exit(1)
if app_config.is_multi_dataset():
logging.info(
f"starting server with multi_dataset__dataroot={app_config.server_config.multi_dataset__dataroot}"
)
else:
logging.info(
def launch(
datapath,
verbose,
debug,
open_browser,
port,
host,
embedding,
obs_names,
var_names,
max_category_items,
disable_custom_colors,
diffexp_lfc_cutoff,
title,
scripts,
about,
disable_annotations,
annotations_file,
user_generated_data_dir,
gene_sets_file,
disable_gene_sets_save,
backed,
disable_diffexp,
config_file,
dump_default_config,
x_approximate_distribution,
):
"""Launch the cellxgene data viewer.
This web app lets you explore single-cell expression data.
Data must be in a format that cellxgene expects.
Read the "getting started" guide to learn more:
https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/README.md
Examples:
> cellxgene launch example-dataset/pbmc3k.h5ad --title pbmc3k
> cellxgene launch <your data file> --title <your title>
> cellxgene launch <url>"""
if dump_default_config:
print(default_config)
sys.exit(0)
# Startup message
click.echo("[cellxgene] Starting the CLI...")
# app config
app_config = AppConfig()
server_config = app_config.server_config
try:
if config_file:
app_config.update_from_config_file(config_file)
# Determine which config options were give on the command line.
# Those will override the ones provided in the config file (if provided).
cli_config = AppConfig()
cli_config.update_server_config(
app__verbose=verbose,
app__debug=debug,
app__host=host,
app__port=port,
app__open_browser=open_browser,
single_dataset__datapath=datapath,
single_dataset__title=title,
single_dataset__about=about,
single_dataset__obs_names=obs_names,
single_dataset__var_names=var_names,
adaptor__anndata_adaptor__backed=backed,
)
cli_config.update_dataset_config(
app__scripts=scripts,
user_annotations__enable=not disable_annotations,
user_annotations__local_file_csv__file=annotations_file,
user_annotations__local_file_csv__directory=user_generated_data_dir,
user_annotations__local_file_csv__gene_sets_file=gene_sets_file,
user_annotations__gene_sets__readonly=disable_gene_sets_save,
presentation__max_categories=max_category_items,
presentation__custom_colors=not disable_custom_colors,
embeddings__names=embedding,
diffexp__enable=not disable_diffexp,
diffexp__lfc_cutoff=diffexp_lfc_cutoff,
X_approximate_distribution=x_approximate_distribution,
)
diff = cli_config.server_config.changes_from_default()
changes = {key: val for key, val, _ in diff}
app_config.update_server_config(**changes)
diff = cli_config.dataset_config.changes_from_default()
changes = {key: val for key, val, _ in diff}
app_config.update_dataset_config(**changes)
# process the configuration
# any errors will be thrown as an exception.
# any info messages will be passed to the messagefn function.
def messagefn(message):
click.echo("[cellxgene] " + message)
# Use a default secret if one is not provided
if not server_config.app__flask_secret_key:
app_config.update_server_config(
app__flask_secret_key="SparkleAndShine")
app_config.complete_config(messagefn)
except (ConfigurationError, DatasetAccessError) as e:
raise click.ClickException(e)
handle_scripts(scripts)
# create the server
server = CliLaunchServer(app_config)
if not server_config.app__verbose:
log = logging.getLogger("werkzeug")
log.setLevel(logging.ERROR)
cellxgene_url = f"http://{app_config.server_config.app__host}:{app_config.server_config.app__port}"
if server_config.app__open_browser:
click.echo(
f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now."
)
webbrowser.open(cellxgene_url)
else:
click.echo(
f"[cellxgene] Launching! Please go to {cellxgene_url} in your browser."
)
click.echo("[cellxgene] Type CTRL-C at any time to exit.")
if not server_config.app__verbose:
f = open(os.devnull, "w")
sys.stdout = f
try:
server.app.run(
host=server_config.app__host,
debug=server_config.app__debug,
port=server_config.app__port,
threaded=not server_config.app__debug,
use_debugger=False,
use_reloader=False,
)
except OSError as e:
if e.errno == errno.EADDRINUSE:
raise click.ClickException(
"Port is in use, please specify an open port using the --port flag."
) from e
raise
class TestServerConfig(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
self.server_config = self.config.server_config
self.config.complete_config()
message_list = []
def noop(message):
message_list.append(message)
messagefn = noop
self.context = dict(messagefn=messagefn, messages=message_list)
def get_config(self, **kwargs):
file_name = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}",
config_file_name=self.config_file_name,
**kwargs)
config = AppConfig()
config.update_from_config_file(file_name)
return config
def test_init_raises_error_if_default_config_is_invalid(self):
invalid_config = self.get_config(port="not_valid")
with self.assertRaises(ConfigurationError):
invalid_config.complete_config()
@patch(
"server.common.config.server_config.BaseConfig.validate_correct_type_of_configuration_attribute"
)
def test_complete_config_checks_all_attr(self, mock_check_attrs):
mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute(
)
self.server_config.complete_config(self.context)
self.assertEqual(mock_check_attrs.call_count, 40)
def test_handle_app__throws_error_if_port_doesnt_exist(self):
config = self.get_config(port=99999999)
with self.assertRaises(ConfigurationError):
config.server_config.handle_app(self.context)
@patch("server.common.config.server_config.discover_s3_region_name")
def test_handle_data_locator_works_for_default_types(
self, mock_discover_region_name):
mock_discover_region_name.return_value = None
# Default config
self.assertEqual(
self.config.server_config.data_locator__s3__region_name, None)
# hard coded
config = self.get_config()
self.assertEqual(config.server_config.data_locator__s3__region_name,
"us-east-1")
# incorrectly formatted
dataroot = {
"d1": {
"base_url": "set1",
"dataroot": "/path/to/set1_datasets/"
},
"d2": {
"base_url": "set2/subdir",
"dataroot": "s3://shouldnt/work"
},
}
file_name = self.custom_app_config(
dataroot=dataroot,
config_file_name=self.config_file_name,
data_locater_region_name="true")
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_data_locator()
@patch("server.common.config.server_config.discover_s3_region_name")
def test_handle_data_locator_can_read_from_dataroot(
self, mock_discover_region_name):
mock_discover_region_name.return_value = "us-west-2"
dataroot = {
"d1": {
"base_url": "set1",
"dataroot": "/path/to/set1_datasets/"
},
"d2": {
"base_url": "set2/subdir",
"dataroot": "s3://hosted-cellxgene-dev"
},
}
file_name = self.custom_app_config(
dataroot=dataroot,
config_file_name=self.config_file_name,
data_locater_region_name="true")
config = AppConfig()
config.update_from_config_file(file_name)
config.server_config.handle_data_locator()
self.assertEqual(config.server_config.data_locator__s3__region_name,
"us-west-2")
mock_discover_region_name.assert_called_once_with(
"s3://hosted-cellxgene-dev")
def test_handle_app___can_use_envar_port(self):
config = self.get_config(port=24)
self.assertEqual(config.server_config.app__port, 24)
# Note if the port is set in the config file it will NOT be overwritten by a different envvar
os.environ["CXG_SERVER_PORT"] = "4008"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.server_config.handle_app(self.context)
self.assertEqual(self.config.server_config.app__port, 4008)
del os.environ["CXG_SERVER_PORT"]
def test_handle_app__can_get_secret_key_from_envvar_or_config_file_with_envvar_given_preference(
self):
config = self.get_config(flask_secret_key="KEY_FROM_FILE")
self.assertEqual(config.server_config.app__flask_secret_key,
"KEY_FROM_FILE")
os.environ["CXG_SECRET_KEY"] = "KEY_FROM_ENV"
config.external_config.handle_environment(self.context)
self.assertEqual(config.server_config.app__flask_secret_key,
"KEY_FROM_ENV")
def test_handle_app__sets_web_base_url(self):
config = self.get_config(web_base_url="anything.com")
self.assertEqual(config.server_config.app__web_base_url,
"anything.com")
def test_handle_auth__gets_client_secret_from_envvars_or_config_with_envvars_given_preference(
self):
config = self.get_config(client_secret="KEY_FROM_FILE")
config.server_config.handle_authentication()
self.assertEqual(
config.server_config.authentication__params_oauth__client_secret,
"KEY_FROM_FILE")
os.environ["CXG_OAUTH_CLIENT_SECRET"] = "KEY_FROM_ENV"
config.external_config.handle_environment(self.context)
self.assertEqual(
config.server_config.authentication__params_oauth__client_secret,
"KEY_FROM_ENV")
def test_handle_data_source__errors_when_passed_zero_or_two_dataroots(
self):
file_name = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}",
config_file_name="two_data_roots.yml",
dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
)
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_data_source()
file_name = self.custom_app_config(config_file_name="zero_roots.yml")
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_data_source()
def test_get_api_base_url_works(self):
# test the api_base_url feature, and that it can contain a path
config = AppConfig()
backend_port = find_available_port("localhost", 10000)
config.update_server_config(
app__flask_secret_key="secret",
app__api_base_url=
f"http://*****:*****@patch("server.common.config.server_config.diffexp_tiledb.set_config")
def test_handle_diffexp(self, mock_tiledb_config):
custom_config_file = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}",
cpu_multiplier=3,
diffexp_max_workers=1,
target_workunit=4,
config_file_name=self.config_file_name,
)
config = AppConfig()
config.update_from_config_file(custom_config_file)
config.server_config.handle_diffexp()
# called with the min of diffexp_max_workers and cpus*cpu_multiplier
mock_tiledb_config.assert_called_once_with(1, 4)
@patch("server.data_cxg.cxg_adaptor.CxgAdaptor.set_tiledb_context")
def test_handle_adaptor(self, mock_tiledb_context):
custom_config = self.custom_app_config(dataroot=f"{FIXTURES_ROOT}",
cxg_tile_cache_size=10,
cxg_num_reader_threads=2)
config = AppConfig()
config.update_from_config_file(custom_config)
config.server_config.handle_adaptor()
mock_tiledb_context.assert_called_once_with({
"sm.tile_cache_size":
10,
"sm.num_reader_threads":
2,
"vfs.s3.region":
"us-east-1"
})
class AppConfigTest(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
self.server_config = self.config.server_config
self.config.complete_config()
message_list = []
def noop(message):
message_list.append(message)
messagefn = noop
self.context = dict(messagefn=messagefn, messages=message_list)
def get_config(self, **kwargs):
file_name = self.custom_app_config(
dataset_datapath=H5AD_FIXTURE,
config_file_name=self.config_file_name,
**kwargs)
config = AppConfig()
config.update_from_config_file(file_name)
return config
def test_get_default_config_correctly_reads_default_config_file(self):
app_default_config = AppConfig().default_config
expected_config = yaml.load(default_config, Loader=yaml.Loader)
server_config = app_default_config["server"]
dataset_config = app_default_config["dataset"]
expected_server_config = expected_config["server"]
expected_dataset_config = expected_config["dataset"]
self.assertDictEqual(app_default_config, expected_config)
self.assertDictEqual(server_config, expected_server_config)
self.assertDictEqual(dataset_config, expected_dataset_config)
def test_get_dataset_config_returns_dataset_config_for_single_datasets(
self):
datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad"
file_name = self.custom_app_config(
dataset_datapath=datapath, config_file_name=self.config_file_name)
config = AppConfig()
config.update_from_config_file(file_name)
self.assertEqual(config.get_dataset_config(), config.dataset_config)
def test_update_server_config_updates_server_config_and_config_status(
self):
config = self.get_config()
config.complete_config()
config.check_config()
config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
with self.assertRaises(ConfigurationError):
config.server_config.check_config()
def test_write_config_outputs_yaml_with_all_config_vars(self):
config = self.get_config()
config.write_config(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml")
with open(f"{FIXTURES_ROOT}/tmp_dir/{self.config_file_name}",
"r") as default_config:
default_config_yml = yaml.safe_load(default_config)
with open(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml",
"r") as output_config:
output_config_yml = yaml.safe_load(output_config)
self.maxDiff = None
self.assertEqual(default_config_yml, output_config_yml)
def test_update_app_config(self):
config = AppConfig()
config.update_server_config(app__verbose=True,
single_dataset__datapath="datapath")
vars = config.server_config.changes_from_default()
self.assertCountEqual(vars,
[("app__verbose", True, False),
("single_dataset__datapath", "datapath", None)])
config = AppConfig()
config.update_dataset_config(app__scripts=(), app__inline_scripts=())
vars = config.server_config.changes_from_default()
self.assertCountEqual(vars, [])
config = AppConfig()
config.update_dataset_config(app__scripts=[], app__inline_scripts=[])
vars = config.dataset_config.changes_from_default()
self.assertCountEqual(vars, [])
config = AppConfig()
config.update_dataset_config(app__scripts=("a", "b"),
app__inline_scripts=["c", "d"])
vars = config.dataset_config.changes_from_default()
self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []),
("app__inline_scripts", ["c", "d"], [])])
def test_configfile_no_server_section(self):
# test a config file without a dataset section
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
with open(configfile, "w") as fconfig:
config = """
dataset:
user_annotations:
enable: false
"""
fconfig.write(config)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
server_changes = app_config.server_config.changes_from_default()
dataset_changes = app_config.dataset_config.changes_from_default()
self.assertEqual(server_changes, [])
self.assertEqual(dataset_changes,
[("user_annotations__enable", False, True)])
def test_simple_update_single_config_from_path_and_value(self):
"""Update a simple config parameter"""
config = AppConfig()
config.server_config.single_dataset__datapath = "my/data/path"
# test simple value in server
config.update_single_config_from_path_and_value(
["server", "app", "flask_secret_key"], "mysecret")
self.assertEqual(config.server_config.app__flask_secret_key,
"mysecret")
# test simple value in default dataset
config.update_single_config_from_path_and_value(
["dataset", "user_annotations"],
"dummy_location",
)
# error checking
bad_paths = [
(
["dataset", "does", "not", "exist"],
"unknown config parameter at path: '['dataset', 'does', 'not', 'exist']'",
),
(["does", "not",
"exist"], "path must start with 'server', or 'dataset'"),
([], "path must start with 'server', or 'dataset'"),
([1, 2, 3], "path must be a list of strings, got '[1, 2, 3]'"),
("string", "path must be a list of strings, got 'string'"),
]
for bad_path, error_message in bad_paths:
with self.assertRaises(ConfigurationError) as config_error:
config.update_single_config_from_path_and_value(
bad_path, "value")
self.assertEqual(config_error.exception.message, error_message)
def main():
parser = argparse.ArgumentParser("A command to test diffexp")
parser.add_argument("dataset", help="name of a dataset to load")
parser.add_argument("-na",
"--numA",
type=int,
help="number of rows in group A")
parser.add_argument("-nb",
"--numB",
type=int,
help="number of rows in group B")
parser.add_argument("-va",
"--varA",
help="obs variable:value to use for group A")
parser.add_argument("-vb",
"--varB",
help="obs variable:value to use for group B")
parser.add_argument("-t",
"--trials",
default=1,
type=int,
help="number of trials")
parser.add_argument("-a",
"--alg",
choices=("default", "generic"),
default="default",
help="algorithm to use")
parser.add_argument("-s",
"--show",
default=False,
action="store_true",
help="show the results")
parser.add_argument("-n",
"--new-selection",
default=False,
action="store_true",
help="change the selection between each trial")
parser.add_argument("--seed",
default=1,
type=int,
help="set the random seed")
args = parser.parse_args()
app_config = AppConfig()
app_config.update_server_config(single_dataset__datapath=args.dataset)
app_config.update_server_config(app__verbose=True)
app_config.complete_config()
loader = MatrixDataLoader(args.dataset)
adaptor = loader.open(app_config)
random.seed(args.seed)
np.random.seed(args.seed)
rows = adaptor.get_shape()[0]
if args.numA:
filterA = random.sample(range(rows), args.numA)
elif args.varA:
vname, vval = args.varA.split(":")
filterA = get_filter_from_obs(adaptor, vname, vval)
else:
print("must supply numA or varA")
sys.exit(1)
if args.numB:
filterB = random.sample(range(rows), args.numB)
elif args.varB:
vname, vval = args.varB.split(":")
filterB = get_filter_from_obs(adaptor, vname, vval)
else:
print("must supply numB or varB")
sys.exit(1)
for i in range(args.trials):
if args.new_selection:
if args.numA:
filterA = random.sample(range(rows), args.numA)
if args.numB:
filterB = random.sample(range(rows), args.numB)
maskA = np.zeros(rows, dtype=bool)
maskA[filterA] = True
maskB = np.zeros(rows, dtype=bool)
maskB[filterB] = True
t1 = time.time()
if args.alg == "default":
results = adaptor.compute_diffexp_ttest(maskA, maskB)
elif args.alg == "generic":
results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB)
t2 = time.time()
print("TIME=", t2 - t1)
if args.show:
for res in results:
print(res)
def test_auth_test(self):
app_config = AppConfig()
app_config.update_server_config(app__flask_secret_key="secret")
app_config.update_server_config(authentication__type="test")
app_config.update_server_config(
authentication__insecure_test_environment=True)
app_config.update_server_config(multi_dataset__dataroot=dict(
a1=dict(dataroot=self.dataset_dataroot, base_url="auth"),
a2=dict(dataroot=self.dataset_dataroot, base_url="no-auth"),
))
# specialize the configs
app_config.add_dataroot_config("a1",
app__authentication_enable=True,
user_annotations__enable=True)
app_config.add_dataroot_config("a2",
app__authentication_enable=False,
user_annotations__enable=False)
app_config.complete_config()
with test_server(app_config=app_config) as server:
session = requests.Session()
# auth datasets
config = session.get(
f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(
f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(
config["config"]["authentication"]["requires_client_login"])
self.assertTrue(config["config"]["parameters"]["annotations"])
login_uri = config["config"]["authentication"]["login"]
logout_uri = config["config"]["authentication"]["logout"]
self.assertEqual(login_uri, "/login?dataset=auth/pbmc3k.cxg")
self.assertEqual(logout_uri, "/logout?dataset=auth/pbmc3k.cxg")
r = session.get(f"{server}/{login_uri}")
# check that the login redirect worked
self.assertEqual(r.history[0].status_code, 302)
self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg")
config = session.get(
f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(
f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["username"], "test_account")
self.assertEqual(userinfo["userinfo"]["picture"], None)
self.assertTrue(config["config"]["parameters"]["annotations"])
r = session.get(f"{server}/{logout_uri}")
# check that the logout redirect worked
self.assertEqual(r.history[0].status_code, 302)
self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg")
config = session.get(
f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(
f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(config["config"]["parameters"]["annotations"])
# no-auth datasets
config = session.get(
f"{server}/no-auth/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(
f"{server}/no-auth/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertIsNone(userinfo)
self.assertFalse(config["config"]["parameters"]["annotations"])
# login with a picture
session.get(f"{server}/{login_uri}&picture=myimage.png")
userinfo = session.get(
f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["picture"], "myimage.png")
