def setUp(self): self.data_file = DataLocator(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad") config = AppConfig() config.update_server_config(single_dataset__datapath=self.data_file.path) config.update_server_config(app__flask_secret_key="secret") config.complete_config() self.data = AnndataAdaptor(self.data_file, config)
def main(): parser = argparse.ArgumentParser( "A script to check hosted configuration files") parser.add_argument("config_file", help="the configuration file") parser.add_argument( "-s", "--show", default=False, action="store_true", help= "print the configuration. NOTE: this may print secret values to stdout", ) args = parser.parse_args() app_config = AppConfig() try: app_config.update_from_config_file(args.config_file) app_config.complete_config() except Exception as e: print(f"Error: {str(e)}") print("FAIL:", args.config_file) sys.exit(1) if args.show: yaml_config = app_config.config_to_dict() yaml.dump(yaml_config, sys.stdout) print("PASS:", args.config_file) sys.exit(0)
def data_with_tmp_tiledb_annotations(ext: MatrixDataType): tmp_dir = tempfile.mkdtemp() fname = { MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", MatrixDataType.CXG: "test/fixtures/pbmc3k.cxg", }[ext] data_locator = DataLocator(fname) config = AppConfig() config.update_server_config( app__flask_secret_key="secret", multi_dataset__dataroot=data_locator.path, authentication__type="test", authentication__insecure_test_environment=True, ) config.update_default_dataset_config( embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01, user_annotations__type="hosted_tiledb_array", user_annotations__hosted_tiledb_array__db_uri= "postgresql://*****:*****@localhost:5432", user_annotations__hosted_tiledb_array__hosted_file_directory=tmp_dir, ) config.complete_config() data = MatrixDataLoader(data_locator.abspath()).open(config) annotations = AnnotationsHostedTileDB( tmp_dir, DbUtils("postgresql://*****:*****@localhost:5432"), ) return data, tmp_dir, annotations
def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False): tmp_dir = tempfile.mkdtemp() annotations_file = path.join(tmp_dir, "test_annotations.csv") if annotations_fixture: shutil.copyfile( f"{PROJECT_ROOT}/server/test/fixtures/pbmc3k-annotations.csv", annotations_file) fname = { MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", MatrixDataType.CXG: "test/fixtures/pbmc3k.cxg", }[ext] data_locator = DataLocator(fname) config = AppConfig() config.update_server_config( app__flask_secret_key="secret", single_dataset__obs_names=None, single_dataset__var_names=None, single_dataset__datapath=data_locator.path, ) config.update_default_dataset_config( embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01, ) config.complete_config() data = MatrixDataLoader(data_locator.abspath()).open(config) annotations = AnnotationsLocalFile(None, annotations_file) return data, tmp_dir, annotations
def test_get_api_base_url_works(self): # test the api_base_url feature, and that it can contain a path config = AppConfig() backend_port = find_available_port("localhost", 10000) config.update_server_config( app__flask_secret_key="secret", app__api_base_url= f"http://localhost:{backend_port}/additional/path", multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset", ) config.complete_config() with test_server(["-p", str(backend_port)], app_config=config) as server: session = requests.Session() self.assertEqual(server, f"http://localhost:{backend_port}") response = session.get( f"{server}/additional/path/d/pbmc3k.h5ad/api/v0.2/config") self.assertEqual(response.status_code, 200) data_config = response.json() self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k") # test the health check at the correct url response = session.get(f"{server}/additional/path/health") assert response.json()["status"] == "pass"
class BaseConfigTest(ConfigTests): def setUp(self): self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" self.config = AppConfig() self.config.update_server_config(app__flask_secret_key="secret") self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE) self.server_config = self.config.server_config self.config.complete_config() message_list = [] def noop(message): message_list.append(message) messagefn = noop self.context = dict(messagefn=messagefn, messages=message_list) def get_config(self, **kwargs): file_name = self.custom_app_config( dataset_datapath=f"{H5AD_FIXTURE}", config_file_name=self.config_file_name, **kwargs) config = AppConfig() config.update_from_config_file(file_name) return config def test_mapping_creation_returns_map_of_server_and_dataset_config(self): config = AppConfig() mapping = config.dataset_config.create_mapping(config.default_config) self.assertIsNotNone(mapping["server__app__verbose"]) self.assertIsNotNone(mapping["dataset__presentation__max_categories"]) def test_changes_from_default_returns_list_of_nondefault_config_values( self): config = self.get_config(verbose="true", lfc_cutoff=0.05) server_changes = config.server_config.changes_from_default() dataset_changes = config.dataset_config.changes_from_default() self.assertEqual( server_changes, [ ("app__verbose", True, False), ("app__flask_secret_key", "secret", None), ("single_dataset__datapath", H5AD_FIXTURE, None), ("data_locator__s3__region_name", "us-east-1", True), ], ) self.assertEqual(dataset_changes, [("diffexp__lfc_cutoff", 0.05, 0.01)]) def test_check_config_throws_error_if_attr_has_not_been_checked(self): config = self.get_config(verbose="true") config.complete_config() config.check_config() config.update_server_config(app__verbose=False) with self.assertRaises(ConfigurationError): config.check_config()
def test_aws_secrets_manager(self, mock_get_secret_key): mock_get_secret_key.return_value = { "oauth_client_secret": "mock_oauth_secret", "db_uri": "mock_db_uri", } configfile = self.custom_external_config( aws_secrets_manager_region="us-west-2", aws_secrets_manager_secrets=[ dict( name="my_secret", values=[ dict(key="flask_secret_key", path=["server", "app", "flask_secret_key"], required=False), dict( key="db_uri", path=[ "dataset", "user_annotations", "hosted_tiledb_array", "db_uri" ], required=True, ), dict( key="oauth_client_secret", path=[ "server", "authentication", "params_oauth", "client_secret" ], required=True, ), ], ) ], config_file_name="secret_external_config.yaml", ) app_config = AppConfig() app_config.update_from_config_file(configfile) app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k.cxg" app_config.server_config.app__flask_secret_key = "original" app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k.cxg" app_config.complete_config() self.assertEqual(app_config.server_config.app__flask_secret_key, "original") self.assertEqual( app_config.server_config. authentication__params_oauth__client_secret, "mock_oauth_secret") self.assertEqual( app_config.default_dataset_config. user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")
def test_auth_test_single(self): app_config = AppConfig() app_config.update_server_config(app__flask_secret_key="secret") app_config.update_server_config( authentication__type="test", single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg") app_config.update_server_config( authentication__insecure_test_environment=True) app_config.complete_config() with test_server(app_config=app_config) as server: session = requests.Session() config = session.get(f"{server}/api/v0.2/config").json() userinfo = session.get(f"{server}/api/v0.2/userinfo").json() self.assertFalse(userinfo["userinfo"]["is_authenticated"]) self.assertIsNone(userinfo["userinfo"]["username"]) self.assertTrue( config["config"]["authentication"]["requires_client_login"]) self.assertTrue(config["config"]["parameters"]["annotations"]) login_uri = config["config"]["authentication"]["login"] logout_uri = config["config"]["authentication"]["logout"] self.assertEqual(login_uri, "/login") self.assertEqual(logout_uri, "/logout") response = session.get(f"{server}/{login_uri}") # check that the login redirect worked self.assertEqual(response.history[0].status_code, 302) self.assertEqual(response.url, f"{server}/") config = session.get(f"{server}/api/v0.2/config").json() userinfo = session.get(f"{server}/api/v0.2/userinfo").json() self.assertTrue(userinfo["userinfo"]["is_authenticated"]) self.assertEqual(userinfo["userinfo"]["username"], "test_account") self.assertTrue(config["config"]["parameters"]["annotations"]) response = session.get(f"{server}/{logout_uri}") # check that the logout redirect worked self.assertEqual(response.history[0].status_code, 302) self.assertEqual(response.url, f"{server}/") config = session.get(f"{server}/api/v0.2/config").json() userinfo = session.get(f"{server}/api/v0.2/userinfo").json() self.assertFalse(userinfo["userinfo"]["is_authenticated"]) self.assertIsNone(userinfo["userinfo"]["username"]) self.assertTrue(config["config"]["parameters"]["annotations"])
def test_auth_none(self): app_config = AppConfig() app_config.update_server_config(app__flask_secret_key="secret") app_config.update_server_config( authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot) app_config.update_default_dataset_config( user_annotations__enable=False) app_config.complete_config() with test_server(app_config=app_config) as server: session = requests.Session() config = session.get( f"{server}/d/pbmc3k.cxg/api/v0.2/config").json() userinfo = session.get( f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json() self.assertNotIn("authentication", config["config"]) self.assertIsNone(userinfo)
def test_auth_session(self): app_config = AppConfig() app_config.update_server_config(app__flask_secret_key="secret") app_config.update_server_config( authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot) app_config.update_default_dataset_config(user_annotations__enable=True) app_config.complete_config() with test_server(app_config=app_config) as server: session = requests.Session() config = session.get( f"{server}/d/pbmc3k.cxg/api/v0.2/config").json() userinfo = session.get( f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json() self.assertFalse( config["config"]["authentication"]["requires_client_login"]) self.assertTrue(userinfo["userinfo"]["is_authenticated"]) self.assertEqual(userinfo["userinfo"]["username"], "anonymous")
def app_config(data_locator, backed=False, extra_server_config={}, extra_dataset_config={}): config = AppConfig() config.update_server_config( app__flask_secret_key="secret", single_dataset__obs_names=None, single_dataset__var_names=None, adaptor__anndata_adaptor__backed=backed, single_dataset__datapath=data_locator, limits__diffexp_cellcount_max=None, limits__column_request_max=None, ) config.update_dataset_config(embeddings__names=["umap", "tsne", "pca"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01) config.update_server_config(**extra_server_config) config.update_dataset_config(**extra_dataset_config) config.complete_config() return config
def test_run(self): tempdir = tempfile.TemporaryDirectory(dir=f"{PROJECT_ROOT}/server") tempdirname = tempdir.name config = AppConfig() # test that eb works config.update_server_config(multi_dataset__dataroot=f"{FIXTURES_ROOT}", app__flask_secret_key="open sesame") config.complete_config() config.write_config(f"{tempdirname}/config.yaml") subprocess.check_call(f"git ls-files . | cpio -pdm {tempdirname}", cwd=f"{PROJECT_ROOT}/server/eb", shell=True) subprocess.check_call(["make", "build"], cwd=tempdirname) with run_eb_app(tempdirname) as server: session = requests.Session() r = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config") data_config = r.json() assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
def test_environment_variable_errors(self): # no name app_config = AppConfig() app_config.external_config.environment = [ dict(required=True, path=["this", "is", "a", "path"]) ] with self.assertRaises(ConfigurationError) as config_error: app_config.complete_config() self.assertEqual(config_error.exception.message, "environment: 'name' is missing") # required has wrong type app_config = AppConfig() app_config.external_config.environment = [ dict(name="myenvar", required="optional", path=["this", "is", "a", "path"]) ] with self.assertRaises(ConfigurationError) as config_error: app_config.complete_config() self.assertEqual(config_error.exception.message, "environment: 'required' must be a bool") # no path app_config = AppConfig() app_config.external_config.environment = [ dict(name="myenvar", required=True) ] with self.assertRaises(ConfigurationError) as config_error: app_config.complete_config() self.assertEqual(config_error.exception.message, "environment: 'path' is missing") # required environment variable is not set app_config = AppConfig() app_config.external_config.environment = [ dict(name="THIS_ENV_IS_NOT_SET", required=True, path=["this", "is", "a", "path"]) ] with self.assertRaises(ConfigurationError) as config_error: app_config.complete_config() self.assertEqual( config_error.exception.message, "required environment variable 'THIS_ENV_IS_NOT_SET' not set")
def test_aws_secrets_manager_error(self, mock_get_secret_key): mock_get_secret_key.return_value = { "oauth_client_secret": "mock_oauth_secret", "db_uri": "mock_db_uri", } # no region app_config = AppConfig() app_config.external_config.aws_secrets_manager__region = None app_config.external_config.aws_secrets_manager__secrets = [ dict(name="secret1", values=[ dict(key="key1", required=True, path=["this", "is", "my", "path"]) ]) ] with self.assertRaises(ConfigurationError) as config_error: app_config.complete_config() self.assertEqual( config_error.exception.message, "Invalid type for attribute: aws_secrets_manager__region, expected type str, got NoneType", ) # missing secret name app_config = AppConfig() app_config.external_config.aws_secrets_manager__region = "us-west-2" app_config.external_config.aws_secrets_manager__secrets = [ dict(values=[ dict(key="db_uri", required=True, path=["this", "is", "my", "path"]) ]) ] with self.assertRaises(ConfigurationError) as config_error: app_config.complete_config() self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' is missing") # secret name wrong type app_config = AppConfig() app_config.external_config.aws_secrets_manager__region = "us-west-2" app_config.external_config.aws_secrets_manager__secrets = [ dict(name=1, values=[ dict(key="db_uri", required=True, path=["this", "is", "my", "path"]) ]) ] with self.assertRaises(ConfigurationError) as config_error: app_config.complete_config() self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' must be a string") # missing values name app_config = AppConfig() app_config.external_config.aws_secrets_manager__region = "us-west-2" app_config.external_config.aws_secrets_manager__secrets = [ dict(name="mysecret") ] with self.assertRaises(ConfigurationError) as config_error: app_config.complete_config() self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' is missing") # values wrong type app_config = AppConfig() app_config.external_config.aws_secrets_manager__region = "us-west-2" app_config.external_config.aws_secrets_manager__secrets = [ dict(name="mysecret", values=dict(key="db_uri", required=True, path=["this", "is", "my", "path"])) ] with self.assertRaises(ConfigurationError) as config_error: app_config.complete_config() self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' must be a list") # entry missing key app_config = AppConfig() app_config.external_config.aws_secrets_manager__region = "us-west-2" app_config.external_config.aws_secrets_manager__secrets = [ dict(name="mysecret", values=[ dict(required=True, path=["this", "is", "my", "path"]) ]) ] with self.assertRaises(ConfigurationError) as config_error: app_config.complete_config() self.assertEqual(config_error.exception.message, "missing 'key' in secret values: mysecret") # entry required is wrong type app_config = AppConfig() app_config.external_config.aws_secrets_manager__region = "us-west-2" app_config.external_config.aws_secrets_manager__secrets = [ dict(name="mysecret", values=[ dict(key="db_uri", required="optional", path=["this", "is", "my", "path"]) ]) ] with self.assertRaises(ConfigurationError) as config_error: app_config.complete_config() self.assertEqual( config_error.exception.message, "wrong type for 'required' in secret values: mysecret") # entry missing path app_config = AppConfig() app_config.external_config.aws_secrets_manager__region = "us-west-2" app_config.external_config.aws_secrets_manager__secrets = [ dict(name="mysecret", values=[dict(key="db_uri", required=True)]) ] with self.assertRaises(ConfigurationError) as config_error: app_config.complete_config() self.assertEqual(config_error.exception.message, "missing 'path' in secret values: mysecret") # secret missing required key app_config = AppConfig() app_config.external_config.aws_secrets_manager__region = "us-west-2" app_config.external_config.aws_secrets_manager__secrets = [ dict( name="mysecret", values=[ dict(key="KEY_DOES_NOT_EXIST", required=True, path=["this", "is", "a", "path"]) ], ) ] with self.assertRaises(ConfigurationError) as config_error: app_config.complete_config() self.assertEqual( config_error.exception.message, "required secret 'mysecret:KEY_DOES_NOT_EXIST' not set")
class TestServerConfig(ConfigTests): def setUp(self): self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" self.config = AppConfig() self.config.update_server_config(app__flask_secret_key="secret") self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE) self.server_config = self.config.server_config self.config.complete_config() message_list = [] def noop(message): message_list.append(message) messagefn = noop self.context = dict(messagefn=messagefn, messages=message_list) def get_config(self, **kwargs): file_name = self.custom_app_config( dataset_datapath=f"{H5AD_FIXTURE}", config_file_name=self.config_file_name, **kwargs) config = AppConfig() config.update_from_config_file(file_name) return config def test_init_raises_error_if_default_config_is_invalid(self): invalid_config = self.get_config(port="not_valid") with self.assertRaises(ConfigurationError): invalid_config.complete_config() @patch( "server.common.config.server_config.BaseConfig.validate_correct_type_of_configuration_attribute" ) def test_complete_config_checks_all_attr(self, mock_check_attrs): mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute( ) self.server_config.complete_config(self.context) self.assertEqual(mock_check_attrs.call_count, 19) def test_handle_app__throws_error_if_port_doesnt_exist(self): config = self.get_config(port=99999999) with self.assertRaises(ConfigurationError): config.server_config.handle_app(self.context) @patch("server.common.config.server_config.discover_s3_region_name") def test_handle_data_locator_works_for_default_types( self, mock_discover_region_name): mock_discover_region_name.return_value = None # Default config self.assertEqual( self.config.server_config.data_locator__s3__region_name, None) # hard coded config = self.get_config() self.assertEqual(config.server_config.data_locator__s3__region_name, "us-east-1") # incorrectly formatted datapath = "s3://shouldnt/work" file_name = self.custom_app_config( dataset_datapath=datapath, config_file_name=self.config_file_name, data_locater_region_name="true") config = AppConfig() config.update_from_config_file(file_name) with self.assertRaises(ConfigurationError): config.server_config.handle_data_locator() def test_handle_app___can_use_envar_port(self): config = self.get_config(port=24) self.assertEqual(config.server_config.app__port, 24) # Note if the port is set in the config file it will NOT be overwritten by a different envvar os.environ["CXG_SERVER_PORT"] = "4008" self.config = AppConfig() self.config.update_server_config(app__flask_secret_key="secret") self.config.server_config.handle_app(self.context) self.assertEqual(self.config.server_config.app__port, 4008) del os.environ["CXG_SERVER_PORT"] def test_handle_app__can_get_secret_key_from_envvar_or_config_file_with_envvar_given_preference( self): config = self.get_config(flask_secret_key="KEY_FROM_FILE") self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_FILE") os.environ["CXG_SECRET_KEY"] = "KEY_FROM_ENV" config.external_config.handle_environment(self.context) self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_ENV") def test_config_for_single_dataset(self): file_name = self.custom_app_config( config_file_name="single_dataset.yml", dataset_datapath=f"{H5AD_FIXTURE}") config = AppConfig() config.update_from_config_file(file_name) config.server_config.handle_single_dataset(self.context) file_name = self.custom_app_config( config_file_name="single_dataset_with_about.yml", about="www.cziscience.com", dataset_datapath=f"{H5AD_FIXTURE}", ) config = AppConfig() config.update_from_config_file(file_name) with self.assertRaises(ConfigurationError): config.server_config.handle_single_dataset(self.context)
class TestDatasetConfig(ConfigTests): def setUp(self): self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" self.config = AppConfig() self.config.update_server_config(app__flask_secret_key="secret") self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE) self.dataset_config = self.config.dataset_config self.config.complete_config() message_list = [] def noop(message): message_list.append(message) messagefn = noop self.context = dict(messagefn=messagefn, messages=message_list) def get_config(self, **kwargs): file_name = self.custom_app_config(dataset_datapath=H5AD_FIXTURE, **kwargs) config = AppConfig() config.update_from_config_file(file_name) return config def test_init_datatset_config_sets_vars_from_config(self): config = AppConfig() self.assertEqual(config.dataset_config.presentation__max_categories, 1000) self.assertEqual(config.dataset_config.user_annotations__type, "local_file_csv") self.assertEqual(config.dataset_config.diffexp__lfc_cutoff, 0.01) @patch("server.common.config.dataset_config.BaseConfig.validate_correct_type_of_configuration_attribute") def test_complete_config_checks_all_attr(self, mock_check_attrs): mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute() self.dataset_config.complete_config(self.context) self.assertIsNotNone(self.config.server_config.data_adaptor) self.assertEqual(mock_check_attrs.call_count, 16) def test_app_sets_script_vars(self): config = self.get_config(scripts=["path/to/script"]) config.dataset_config.handle_app() self.assertEqual(config.dataset_config.app__scripts, [{"src": "path/to/script"}]) config = self.get_config(scripts=[{"src": "path/to/script", "more": "different/script/path"}]) config.dataset_config.handle_app() self.assertEqual( config.dataset_config.app__scripts, [{"src": "path/to/script", "more": "different/script/path"}] ) config = self.get_config(scripts=["path/to/script", "different/script/path"]) config.dataset_config.handle_app() # TODO @madison -- is this the desired functionality? self.assertEqual( config.dataset_config.app__scripts, [{"src": "path/to/script"}, {"src": "different/script/path"}] ) config = self.get_config(scripts=[{"more": "different/script/path"}]) with self.assertRaises(ConfigurationError): config.dataset_config.handle_app() def test_handle_user_annotations__instantiates_user_annotations_class_correctly(self): config = self.get_config( enable_users_annotations="true", annotation_type="local_file_csv" ) config.server_config.complete_config(self.context) config.dataset_config.handle_user_annotations(self.context) self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile) config = self.get_config( enable_users_annotations="true", annotation_type="NOT_REAL" ) config.server_config.complete_config(self.context) with self.assertRaises(ConfigurationError): config.dataset_config.handle_user_annotations(self.context) def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in(self): config = self.get_config( enable_users_annotations="true", annotation_type="local_file_csv" ) config.server_config.complete_config(self.context) config.dataset_config.handle_local_file_csv_annotations(self.context) self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile) cwd = os.getcwd() self.assertEqual(config.dataset_config.user_annotations._get_output_dir(), cwd) def test_handle_diffexp__raises_warning_for_large_datasets(self): config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15) config.server_config.complete_config(self.context) config.dataset_config.handle_diffexp(self.context) self.assertEqual(len(self.context["messages"]), 1) def test_configfile_with_specialization(self): # test that per_dataset_config config load the default config, then the specialized config with tempfile.TemporaryDirectory() as tempdir: configfile = os.path.join(tempdir, "config.yaml") with open(configfile, "w") as fconfig: config = """ server: single_dataset: datapath: fake_datapath dataset: user_annotations: enable: false type: local_file_csv local_file_csv: file: fake_file directory: fake_dir """ fconfig.write(config) app_config = AppConfig() app_config.update_from_config_file(configfile) test_config = app_config.dataset_config # test config from default self.assertEqual(test_config.user_annotations__type, "local_file_csv") self.assertEqual(test_config.user_annotations__local_file_csv__file, "fake_file")
def test_multi_dataset(self): config = AppConfig() # test for illegal url_dataroots for illegal in ("../b", "!$*", "\\n", "", "(bad)"): config.update_server_config( app__flask_secret_key="secret", multi_dataset__dataroot={ "tag": { "base_url": illegal, "dataroot": f"{PROJECT_ROOT}/example-dataset" } }, ) with self.assertRaises(ConfigurationError): config.complete_config() # test for legal url_dataroots for legal in ("d", "this.is-okay_", "a/b"): config.update_server_config( app__flask_secret_key="secret", multi_dataset__dataroot={ "tag": { "base_url": legal, "dataroot": f"{PROJECT_ROOT}/example-dataset" } }, ) config.complete_config() # test that multi dataroots work end to end config.update_server_config( app__flask_secret_key="secret", multi_dataset__dataroot=dict( s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"), s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"), s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"), ), ) # Change this default to test if the dataroot overrides below work. config.update_default_dataset_config( app__about_legal_tos="tos_default.html") # specialize the configs for set1 config.add_dataroot_config("s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html") # specialize the configs for set2 config.add_dataroot_config("s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html") # no specializations for set3 (they get the default dataset config) config.complete_config() with test_server(app_config=config) as server: session = requests.Session() response = session.get( f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config") data_config = response.json() assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" assert data_config["config"]["parameters"]["annotations"] is False assert data_config["config"]["parameters"][ "disable-diffexp"] is False assert data_config["config"]["parameters"][ "about_legal_tos"] == "tos_set1.html" response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config") data_config = response.json() assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" assert data_config["config"]["parameters"]["annotations"] is True assert data_config["config"]["parameters"][ "about_legal_tos"] == "tos_set2.html" response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config") data_config = response.json() assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" assert data_config["config"]["parameters"]["annotations"] is True assert data_config["config"]["parameters"][ "disable-diffexp"] is False assert data_config["config"]["parameters"][ "about_legal_tos"] == "tos_default.html" response = session.get(f"{server}/health") assert response.json()["status"] == "pass"
class AppConfigTest(ConfigTests): def setUp(self): self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" self.config = AppConfig() self.config.update_server_config(app__flask_secret_key="secret") self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT) self.server_config = self.config.server_config self.config.complete_config() message_list = [] def noop(message): message_list.append(message) messagefn = noop self.context = dict(messagefn=messagefn, messages=message_list) def get_config(self, **kwargs): file_name = self.custom_app_config( dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs) config = AppConfig() config.update_from_config_file(file_name) return config def test_get_default_config_correctly_reads_default_config_file(self): app_default_config = AppConfig().default_config expected_config = yaml.load(default_config, Loader=yaml.Loader) server_config = app_default_config["server"] dataset_config = app_default_config["dataset"] expected_server_config = expected_config["server"] expected_dataset_config = expected_config["dataset"] self.assertDictEqual(app_default_config, expected_config) self.assertDictEqual(server_config, expected_server_config) self.assertDictEqual(dataset_config, expected_dataset_config) def test_get_dataset_config_returns_default_dataset_config_for_single_datasets( self): datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad" file_name = self.custom_app_config( dataset_datapath=datapath, config_file_name=self.config_file_name) config = AppConfig() config.update_from_config_file(file_name) self.assertEqual(config.get_dataset_config(""), config.default_dataset_config) def test_update_server_config_updates_server_config_and_config_status( self): config = self.get_config() config.complete_config() config.check_config() config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT) with self.assertRaises(ConfigurationError): config.server_config.check_config() def test_write_config_outputs_yaml_with_all_config_vars(self): config = self.get_config() config.write_config(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml") with open(f"{FIXTURES_ROOT}/tmp_dir/{self.config_file_name}", "r") as default_config: default_config_yml = yaml.safe_load(default_config) with open(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml", "r") as output_config: output_config_yml = yaml.safe_load(output_config) self.maxDiff = None self.assertEqual(default_config_yml, output_config_yml) def test_update_app_config(self): config = AppConfig() config.update_server_config(app__verbose=True, multi_dataset__dataroot="datadir") vars = config.server_config.changes_from_default() self.assertCountEqual(vars, [("app__verbose", True, False), ("multi_dataset__dataroot", "datadir", None)]) config = AppConfig() config.update_default_dataset_config(app__scripts=(), app__inline_scripts=()) vars = config.server_config.changes_from_default() self.assertCountEqual(vars, []) config = AppConfig() config.update_default_dataset_config(app__scripts=[], app__inline_scripts=[]) vars = config.default_dataset_config.changes_from_default() self.assertCountEqual(vars, []) config = AppConfig() config.update_default_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"]) vars = config.default_dataset_config.changes_from_default() self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])]) def test_configfile_no_dataset_section(self): # test a config file without a dataset section with tempfile.TemporaryDirectory() as tempdir: configfile = os.path.join(tempdir, "config.yaml") with open(configfile, "w") as fconfig: config = """ server: app: flask_secret_key: secret multi_dataset: dataroot: test_dataroot """ fconfig.write(config) app_config = AppConfig() app_config.update_from_config_file(configfile) server_changes = app_config.server_config.changes_from_default() dataset_changes = app_config.default_dataset_config.changes_from_default( ) self.assertEqual( server_changes, [("app__flask_secret_key", "secret", None), ("multi_dataset__dataroot", "test_dataroot", None)], ) self.assertEqual(dataset_changes, []) def test_configfile_no_server_section(self): # test a config file without a dataset section with tempfile.TemporaryDirectory() as tempdir: configfile = os.path.join(tempdir, "config.yaml") with open(configfile, "w") as fconfig: config = """ dataset: user_annotations: enable: false """ fconfig.write(config) app_config = AppConfig() app_config.update_from_config_file(configfile) server_changes = app_config.server_config.changes_from_default() dataset_changes = app_config.default_dataset_config.changes_from_default( ) self.assertEqual(server_changes, []) self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)]) def test_simple_update_single_config_from_path_and_value(self): """Update a simple config parameter""" config = AppConfig() config.server_config.multi_dataset__dataroot = dict( s1=dict(dataroot="my_dataroot_s1", base_url="my_baseurl_s1"), s2=dict(dataroot="my_dataroot_s2", base_url="my_baseurl_s2"), ) config.add_dataroot_config("s1") config.add_dataroot_config("s2") # test simple value in server config.update_single_config_from_path_and_value( ["server", "app", "flask_secret_key"], "mysecret") self.assertEqual(config.server_config.app__flask_secret_key, "mysecret") # test simple value in default dataset config.update_single_config_from_path_and_value( ["dataset", "user_annotations", "hosted_tiledb_array", "db_uri"], "mydburi", ) self.assertEqual( config.default_dataset_config. user_annotations__hosted_tiledb_array__db_uri, "mydburi") self.assertEqual( config.dataroot_config["s1"]. user_annotations__hosted_tiledb_array__db_uri, "mydburi") self.assertEqual( config.dataroot_config["s2"]. user_annotations__hosted_tiledb_array__db_uri, "mydburi") # test simple value in specific dataset config.update_single_config_from_path_and_value([ "per_dataset_config", "s1", "user_annotations", "hosted_tiledb_array", "db_uri" ], "s1dburi") self.assertEqual( config.default_dataset_config. user_annotations__hosted_tiledb_array__db_uri, "mydburi") self.assertEqual( config.dataroot_config["s1"]. user_annotations__hosted_tiledb_array__db_uri, "s1dburi") self.assertEqual( config.dataroot_config["s2"]. user_annotations__hosted_tiledb_array__db_uri, "mydburi") # error checking bad_paths = [ ( ["dataset", "does", "not", "exist"], "unknown config parameter at path: '['dataset', 'does', 'not', 'exist']'", ), (["does", "not", "exist"], "path must start with 'server', 'dataset', or 'per_dataset_config'" ), ([], "path must start with 'server', 'dataset', or 'per_dataset_config'" ), (["per_dataset_config"], "missing dataroot when using per_dataset_config: got '['per_dataset_config']'" ), ( ["per_dataset_config", "unknown"], "unknown dataroot when using per_dataset_config: got '['per_dataset_config', 'unknown']'," " dataroots specified in config are ['s1', 's2']", ), ([1, 2, 3], "path must be a list of strings, got '[1, 2, 3]'"), ("string", "path must be a list of strings, got 'string'"), ] for bad_path, error_message in bad_paths: with self.assertRaises(ConfigurationError) as config_error: config.update_single_config_from_path_and_value( bad_path, "value") self.assertEqual(config_error.exception.message, error_message) def test_dict_update_single_config_from_path_and_value(self): """Update a config parameter that has a value of dict""" # the path leads to a dict config param, set the config parameter to the new value config = AppConfig() config.update_single_config_from_path_and_value( ["server", "authentication", "params_oauth", "cookie"], dict(key="mykey1", max_age=100)) self.assertEqual( config.server_config.authentication__params_oauth__cookie, dict(key="mykey1", max_age=100)) # the path leads to an entry within a dict config param, the value is simple config = AppConfig() config.server_config.authentication__params_oauth__cookie = dict( key="mykey1", max_age=100) config.update_single_config_from_path_and_value( ["server", "authentication", "params_oauth", "cookie", "httponly"], True, ) self.assertEqual( config.server_config.authentication__params_oauth__cookie, dict(key="mykey1", max_age=100, httponly=True))
class TestDatasetConfig(ConfigTests): def setUp(self): self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" self.config = AppConfig() self.config.update_server_config(app__flask_secret_key="secret") self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT) self.dataset_config = self.config.default_dataset_config self.config.complete_config() message_list = [] def noop(message): message_list.append(message) messagefn = noop self.context = dict(messagefn=messagefn, messages=message_list) def get_config(self, **kwargs): file_name = self.custom_app_config( dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs) config = AppConfig() config.update_from_config_file(file_name) return config def test_init_datatset_config_sets_vars_from_default_config(self): config = AppConfig() self.assertEqual( config.default_dataset_config.presentation__max_categories, 1000) self.assertEqual(config.default_dataset_config.user_annotations__type, "local_file_csv") self.assertEqual(config.default_dataset_config.diffexp__lfc_cutoff, 0.01) self.assertIsNone(config.default_dataset_config. user_annotations__ontology__obo_location) @patch( "server.common.config.dataset_config.BaseConfig.validate_correct_type_of_configuration_attribute" ) def test_complete_config_checks_all_attr(self, mock_check_attrs): mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute( ) self.dataset_config.complete_config(self.context) self.assertEqual(mock_check_attrs.call_count, 21) def test_app_sets_script_vars(self): config = self.get_config(scripts=["path/to/script"]) config.default_dataset_config.handle_app() self.assertEqual(config.default_dataset_config.app__scripts, [{ "src": "path/to/script" }]) config = self.get_config(scripts=[{ "src": "path/to/script", "more": "different/script/path" }]) config.default_dataset_config.handle_app() self.assertEqual(config.default_dataset_config.app__scripts, [{ "src": "path/to/script", "more": "different/script/path" }]) config = self.get_config( scripts=["path/to/script", "different/script/path"]) config.default_dataset_config.handle_app() # TODO @madison -- is this the desired functionality? self.assertEqual(config.default_dataset_config.app__scripts, [{ "src": "path/to/script" }, { "src": "different/script/path" }]) config = self.get_config(scripts=[{"more": "different/script/path"}]) with self.assertRaises(ConfigurationError): config.default_dataset_config.handle_app() def test_handle_user_annotations_ensures_auth_is_enabled_with_valid_auth_type( self): config = self.get_config(enable_users_annotations="true", authentication_enable="false") config.server_config.complete_config(self.context) with self.assertRaises(ConfigurationError): config.default_dataset_config.handle_user_annotations(self.context) config = self.get_config(enable_users_annotations="true", authentication_enable="true", auth_type="pretend") with self.assertRaises(ConfigurationError): config.server_config.complete_config(self.context) def test_handle_user_annotations__adds_warning_message_if_annotation_vars_set_when_annotations_disabled( self): config = self.get_config(enable_users_annotations="false", authentication_enable="false", db_uri="shouldnt/be/set") config.default_dataset_config.handle_user_annotations(self.context) self.assertEqual( self.context["messages"], ["Warning: db_uri ignored as annotations are disabled."]) @patch("server.common.config.dataset_config.DbUtils") def test_handle_user_annotations__instantiates_user_annotations_class_correctly( self, mock_db_utils): mock_db_utils.return_value = "123" config = self.get_config(enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv") config.server_config.complete_config(self.context) config.default_dataset_config.handle_user_annotations(self.context) self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile) config = self.get_config( enable_users_annotations="true", authentication_enable="true", annotation_type="hosted_tiledb_array", db_uri="gotta/set/this", hosted_file_directory="and/this", ) config.server_config.complete_config(self.context) config.default_dataset_config.handle_user_annotations(self.context) self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsHostedTileDB) config = self.get_config(enable_users_annotations="true", authentication_enable="true", annotation_type="NOT_REAL") config.server_config.complete_config(self.context) with self.assertRaises(ConfigurationError): config.default_dataset_config.handle_user_annotations(self.context) def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in( self): config = self.get_config(enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv") config.server_config.complete_config(self.context) config.default_dataset_config.handle_local_file_csv_annotations() self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile) cwd = os.getcwd() self.assertEqual( config.default_dataset_config.user_annotations._get_output_dir(), cwd) def test_handle_embeddings__checks_data_file_types(self): file_name = self.custom_app_config( embedding_names=["name1", "name2"], enable_reembedding="true", dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", anndata_backed="true", config_file_name=self.config_file_name, ) config = AppConfig() config.update_from_config_file(file_name) config.server_config.complete_config(self.context) with self.assertRaises(ConfigurationError): config.default_dataset_config.handle_embeddings() def test_handle_diffexp__raises_warning_for_large_datasets(self): config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15) config.server_config.complete_config(self.context) config.default_dataset_config.handle_diffexp(self.context) self.assertEqual(len(self.context["messages"]), 0) def test_multi_dataset(self): config = AppConfig() # test for illegal url_dataroots for illegal in ("../b", "!$*", "\\n", "", "(bad)"): config.update_server_config( app__flask_secret_key="secret", multi_dataset__dataroot={ "tag": { "base_url": illegal, "dataroot": f"{PROJECT_ROOT}/example-dataset" } }, ) with self.assertRaises(ConfigurationError): config.complete_config() # test for legal url_dataroots for legal in ("d", "this.is-okay_", "a/b"): config.update_server_config( app__flask_secret_key="secret", multi_dataset__dataroot={ "tag": { "base_url": legal, "dataroot": f"{PROJECT_ROOT}/example-dataset" } }, ) config.complete_config() # test that multi dataroots work end to end config.update_server_config( app__flask_secret_key="secret", multi_dataset__dataroot=dict( s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"), s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"), s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"), ), ) # Change this default to test if the dataroot overrides below work. config.update_default_dataset_config( app__about_legal_tos="tos_default.html") # specialize the configs for set1 config.add_dataroot_config("s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html") # specialize the configs for set2 config.add_dataroot_config("s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html") # no specializations for set3 (they get the default dataset config) config.complete_config() with test_server(app_config=config) as server: session = requests.Session() response = session.get( f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config") data_config = response.json() assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" assert data_config["config"]["parameters"]["annotations"] is False assert data_config["config"]["parameters"][ "disable-diffexp"] is False assert data_config["config"]["parameters"][ "about_legal_tos"] == "tos_set1.html" response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config") data_config = response.json() assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" assert data_config["config"]["parameters"]["annotations"] is True assert data_config["config"]["parameters"][ "about_legal_tos"] == "tos_set2.html" response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config") data_config = response.json() assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" assert data_config["config"]["parameters"]["annotations"] is True assert data_config["config"]["parameters"][ "disable-diffexp"] is False assert data_config["config"]["parameters"][ "about_legal_tos"] == "tos_default.html" response = session.get(f"{server}/health") assert response.json()["status"] == "pass" def test_configfile_with_specialization(self): # test that per_dataset_config config load the default config, then the specialized config with tempfile.TemporaryDirectory() as tempdir: configfile = os.path.join(tempdir, "config.yaml") with open(configfile, "w") as fconfig: config = """ server: multi_dataset: dataroot: test: base_url: test dataroot: fake_dataroot dataset: user_annotations: enable: false type: hosted_tiledb_array hosted_tiledb_array: db_uri: fake_db_uri hosted_file_directory: fake_dir per_dataset_config: test: user_annotations: enable: true """ fconfig.write(config) app_config = AppConfig() app_config.update_from_config_file(configfile) test_config = app_config.dataroot_config["test"] # test config from default self.assertEqual(test_config.user_annotations__type, "hosted_tiledb_array") self.assertEqual( test_config.user_annotations__hosted_tiledb_array__db_uri, "fake_db_uri") # test config from specialization self.assertTrue(test_config.user_annotations__enable)
logging.critical("No config file found") sys.exit(1) dataroot = os.getenv("CXG_DATAROOT") if dataroot: logging.info("Configuration from CXG_DATAROOT") app_config.update_server_config(multi_dataset__dataroot=dataroot) # overwrite configuration for the eb app app_config.update_default_dataset_config( embeddings__enable_reembedding=False, ) app_config.update_server_config( multi_dataset__allowed_matrix_types=["cxg"], ) # complete config app_config.complete_config(logging.info) server = WSGIServer(app_config) debug = False application = server.app except Exception: logging.critical("Caught exception during initialization", exc_info=True) sys.exit(1) if app_config.is_multi_dataset(): logging.info( f"starting server with multi_dataset__dataroot={app_config.server_config.multi_dataset__dataroot}" ) else: logging.info(
def launch( datapath, verbose, debug, open_browser, port, host, embedding, obs_names, var_names, max_category_items, disable_custom_colors, diffexp_lfc_cutoff, title, scripts, about, disable_annotations, annotations_file, user_generated_data_dir, gene_sets_file, disable_gene_sets_save, backed, disable_diffexp, config_file, dump_default_config, x_approximate_distribution, ): """Launch the cellxgene data viewer. This web app lets you explore single-cell expression data. Data must be in a format that cellxgene expects. Read the "getting started" guide to learn more: https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/README.md Examples: > cellxgene launch example-dataset/pbmc3k.h5ad --title pbmc3k > cellxgene launch <your data file> --title <your title> > cellxgene launch <url>""" if dump_default_config: print(default_config) sys.exit(0) # Startup message click.echo("[cellxgene] Starting the CLI...") # app config app_config = AppConfig() server_config = app_config.server_config try: if config_file: app_config.update_from_config_file(config_file) # Determine which config options were give on the command line. # Those will override the ones provided in the config file (if provided). cli_config = AppConfig() cli_config.update_server_config( app__verbose=verbose, app__debug=debug, app__host=host, app__port=port, app__open_browser=open_browser, single_dataset__datapath=datapath, single_dataset__title=title, single_dataset__about=about, single_dataset__obs_names=obs_names, single_dataset__var_names=var_names, adaptor__anndata_adaptor__backed=backed, ) cli_config.update_dataset_config( app__scripts=scripts, user_annotations__enable=not disable_annotations, user_annotations__local_file_csv__file=annotations_file, user_annotations__local_file_csv__directory=user_generated_data_dir, user_annotations__local_file_csv__gene_sets_file=gene_sets_file, user_annotations__gene_sets__readonly=disable_gene_sets_save, presentation__max_categories=max_category_items, presentation__custom_colors=not disable_custom_colors, embeddings__names=embedding, diffexp__enable=not disable_diffexp, diffexp__lfc_cutoff=diffexp_lfc_cutoff, X_approximate_distribution=x_approximate_distribution, ) diff = cli_config.server_config.changes_from_default() changes = {key: val for key, val, _ in diff} app_config.update_server_config(**changes) diff = cli_config.dataset_config.changes_from_default() changes = {key: val for key, val, _ in diff} app_config.update_dataset_config(**changes) # process the configuration # any errors will be thrown as an exception. # any info messages will be passed to the messagefn function. def messagefn(message): click.echo("[cellxgene] " + message) # Use a default secret if one is not provided if not server_config.app__flask_secret_key: app_config.update_server_config( app__flask_secret_key="SparkleAndShine") app_config.complete_config(messagefn) except (ConfigurationError, DatasetAccessError) as e: raise click.ClickException(e) handle_scripts(scripts) # create the server server = CliLaunchServer(app_config) if not server_config.app__verbose: log = logging.getLogger("werkzeug") log.setLevel(logging.ERROR) cellxgene_url = f"http://{app_config.server_config.app__host}:{app_config.server_config.app__port}" if server_config.app__open_browser: click.echo( f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now." ) webbrowser.open(cellxgene_url) else: click.echo( f"[cellxgene] Launching! Please go to {cellxgene_url} in your browser." ) click.echo("[cellxgene] Type CTRL-C at any time to exit.") if not server_config.app__verbose: f = open(os.devnull, "w") sys.stdout = f try: server.app.run( host=server_config.app__host, debug=server_config.app__debug, port=server_config.app__port, threaded=not server_config.app__debug, use_debugger=False, use_reloader=False, ) except OSError as e: if e.errno == errno.EADDRINUSE: raise click.ClickException( "Port is in use, please specify an open port using the --port flag." ) from e raise
class TestServerConfig(ConfigTests): def setUp(self): self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" self.config = AppConfig() self.config.update_server_config(app__flask_secret_key="secret") self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT) self.server_config = self.config.server_config self.config.complete_config() message_list = [] def noop(message): message_list.append(message) messagefn = noop self.context = dict(messagefn=messagefn, messages=message_list) def get_config(self, **kwargs): file_name = self.custom_app_config( dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs) config = AppConfig() config.update_from_config_file(file_name) return config def test_init_raises_error_if_default_config_is_invalid(self): invalid_config = self.get_config(port="not_valid") with self.assertRaises(ConfigurationError): invalid_config.complete_config() @patch( "server.common.config.server_config.BaseConfig.validate_correct_type_of_configuration_attribute" ) def test_complete_config_checks_all_attr(self, mock_check_attrs): mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute( ) self.server_config.complete_config(self.context) self.assertEqual(mock_check_attrs.call_count, 40) def test_handle_app__throws_error_if_port_doesnt_exist(self): config = self.get_config(port=99999999) with self.assertRaises(ConfigurationError): config.server_config.handle_app(self.context) @patch("server.common.config.server_config.discover_s3_region_name") def test_handle_data_locator_works_for_default_types( self, mock_discover_region_name): mock_discover_region_name.return_value = None # Default config self.assertEqual( self.config.server_config.data_locator__s3__region_name, None) # hard coded config = self.get_config() self.assertEqual(config.server_config.data_locator__s3__region_name, "us-east-1") # incorrectly formatted dataroot = { "d1": { "base_url": "set1", "dataroot": "/path/to/set1_datasets/" }, "d2": { "base_url": "set2/subdir", "dataroot": "s3://shouldnt/work" }, } file_name = self.custom_app_config( dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true") config = AppConfig() config.update_from_config_file(file_name) with self.assertRaises(ConfigurationError): config.server_config.handle_data_locator() @patch("server.common.config.server_config.discover_s3_region_name") def test_handle_data_locator_can_read_from_dataroot( self, mock_discover_region_name): mock_discover_region_name.return_value = "us-west-2" dataroot = { "d1": { "base_url": "set1", "dataroot": "/path/to/set1_datasets/" }, "d2": { "base_url": "set2/subdir", "dataroot": "s3://hosted-cellxgene-dev" }, } file_name = self.custom_app_config( dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true") config = AppConfig() config.update_from_config_file(file_name) config.server_config.handle_data_locator() self.assertEqual(config.server_config.data_locator__s3__region_name, "us-west-2") mock_discover_region_name.assert_called_once_with( "s3://hosted-cellxgene-dev") def test_handle_app___can_use_envar_port(self): config = self.get_config(port=24) self.assertEqual(config.server_config.app__port, 24) # Note if the port is set in the config file it will NOT be overwritten by a different envvar os.environ["CXG_SERVER_PORT"] = "4008" self.config = AppConfig() self.config.update_server_config(app__flask_secret_key="secret") self.config.server_config.handle_app(self.context) self.assertEqual(self.config.server_config.app__port, 4008) del os.environ["CXG_SERVER_PORT"] def test_handle_app__can_get_secret_key_from_envvar_or_config_file_with_envvar_given_preference( self): config = self.get_config(flask_secret_key="KEY_FROM_FILE") self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_FILE") os.environ["CXG_SECRET_KEY"] = "KEY_FROM_ENV" config.external_config.handle_environment(self.context) self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_ENV") def test_handle_app__sets_web_base_url(self): config = self.get_config(web_base_url="anything.com") self.assertEqual(config.server_config.app__web_base_url, "anything.com") def test_handle_auth__gets_client_secret_from_envvars_or_config_with_envvars_given_preference( self): config = self.get_config(client_secret="KEY_FROM_FILE") config.server_config.handle_authentication() self.assertEqual( config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_FILE") os.environ["CXG_OAUTH_CLIENT_SECRET"] = "KEY_FROM_ENV" config.external_config.handle_environment(self.context) self.assertEqual( config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_ENV") def test_handle_data_source__errors_when_passed_zero_or_two_dataroots( self): file_name = self.custom_app_config( dataroot=f"{FIXTURES_ROOT}", config_file_name="two_data_roots.yml", dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", ) config = AppConfig() config.update_from_config_file(file_name) with self.assertRaises(ConfigurationError): config.server_config.handle_data_source() file_name = self.custom_app_config(config_file_name="zero_roots.yml") config = AppConfig() config.update_from_config_file(file_name) with self.assertRaises(ConfigurationError): config.server_config.handle_data_source() def test_get_api_base_url_works(self): # test the api_base_url feature, and that it can contain a path config = AppConfig() backend_port = find_available_port("localhost", 10000) config.update_server_config( app__flask_secret_key="secret", app__api_base_url= f"http://*****:*****@patch("server.common.config.server_config.diffexp_tiledb.set_config") def test_handle_diffexp(self, mock_tiledb_config): custom_config_file = self.custom_app_config( dataroot=f"{FIXTURES_ROOT}", cpu_multiplier=3, diffexp_max_workers=1, target_workunit=4, config_file_name=self.config_file_name, ) config = AppConfig() config.update_from_config_file(custom_config_file) config.server_config.handle_diffexp() # called with the min of diffexp_max_workers and cpus*cpu_multiplier mock_tiledb_config.assert_called_once_with(1, 4) @patch("server.data_cxg.cxg_adaptor.CxgAdaptor.set_tiledb_context") def test_handle_adaptor(self, mock_tiledb_context): custom_config = self.custom_app_config(dataroot=f"{FIXTURES_ROOT}", cxg_tile_cache_size=10, cxg_num_reader_threads=2) config = AppConfig() config.update_from_config_file(custom_config) config.server_config.handle_adaptor() mock_tiledb_context.assert_called_once_with({ "sm.tile_cache_size": 10, "sm.num_reader_threads": 2, "vfs.s3.region": "us-east-1" })
class AppConfigTest(ConfigTests): def setUp(self): self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" self.config = AppConfig() self.config.update_server_config(app__flask_secret_key="secret") self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE) self.server_config = self.config.server_config self.config.complete_config() message_list = [] def noop(message): message_list.append(message) messagefn = noop self.context = dict(messagefn=messagefn, messages=message_list) def get_config(self, **kwargs): file_name = self.custom_app_config( dataset_datapath=H5AD_FIXTURE, config_file_name=self.config_file_name, **kwargs) config = AppConfig() config.update_from_config_file(file_name) return config def test_get_default_config_correctly_reads_default_config_file(self): app_default_config = AppConfig().default_config expected_config = yaml.load(default_config, Loader=yaml.Loader) server_config = app_default_config["server"] dataset_config = app_default_config["dataset"] expected_server_config = expected_config["server"] expected_dataset_config = expected_config["dataset"] self.assertDictEqual(app_default_config, expected_config) self.assertDictEqual(server_config, expected_server_config) self.assertDictEqual(dataset_config, expected_dataset_config) def test_get_dataset_config_returns_dataset_config_for_single_datasets( self): datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad" file_name = self.custom_app_config( dataset_datapath=datapath, config_file_name=self.config_file_name) config = AppConfig() config.update_from_config_file(file_name) self.assertEqual(config.get_dataset_config(), config.dataset_config) def test_update_server_config_updates_server_config_and_config_status( self): config = self.get_config() config.complete_config() config.check_config() config.update_server_config(single_dataset__datapath=H5AD_FIXTURE) with self.assertRaises(ConfigurationError): config.server_config.check_config() def test_write_config_outputs_yaml_with_all_config_vars(self): config = self.get_config() config.write_config(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml") with open(f"{FIXTURES_ROOT}/tmp_dir/{self.config_file_name}", "r") as default_config: default_config_yml = yaml.safe_load(default_config) with open(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml", "r") as output_config: output_config_yml = yaml.safe_load(output_config) self.maxDiff = None self.assertEqual(default_config_yml, output_config_yml) def test_update_app_config(self): config = AppConfig() config.update_server_config(app__verbose=True, single_dataset__datapath="datapath") vars = config.server_config.changes_from_default() self.assertCountEqual(vars, [("app__verbose", True, False), ("single_dataset__datapath", "datapath", None)]) config = AppConfig() config.update_dataset_config(app__scripts=(), app__inline_scripts=()) vars = config.server_config.changes_from_default() self.assertCountEqual(vars, []) config = AppConfig() config.update_dataset_config(app__scripts=[], app__inline_scripts=[]) vars = config.dataset_config.changes_from_default() self.assertCountEqual(vars, []) config = AppConfig() config.update_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"]) vars = config.dataset_config.changes_from_default() self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])]) def test_configfile_no_server_section(self): # test a config file without a dataset section with tempfile.TemporaryDirectory() as tempdir: configfile = os.path.join(tempdir, "config.yaml") with open(configfile, "w") as fconfig: config = """ dataset: user_annotations: enable: false """ fconfig.write(config) app_config = AppConfig() app_config.update_from_config_file(configfile) server_changes = app_config.server_config.changes_from_default() dataset_changes = app_config.dataset_config.changes_from_default() self.assertEqual(server_changes, []) self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)]) def test_simple_update_single_config_from_path_and_value(self): """Update a simple config parameter""" config = AppConfig() config.server_config.single_dataset__datapath = "my/data/path" # test simple value in server config.update_single_config_from_path_and_value( ["server", "app", "flask_secret_key"], "mysecret") self.assertEqual(config.server_config.app__flask_secret_key, "mysecret") # test simple value in default dataset config.update_single_config_from_path_and_value( ["dataset", "user_annotations"], "dummy_location", ) # error checking bad_paths = [ ( ["dataset", "does", "not", "exist"], "unknown config parameter at path: '['dataset', 'does', 'not', 'exist']'", ), (["does", "not", "exist"], "path must start with 'server', or 'dataset'"), ([], "path must start with 'server', or 'dataset'"), ([1, 2, 3], "path must be a list of strings, got '[1, 2, 3]'"), ("string", "path must be a list of strings, got 'string'"), ] for bad_path, error_message in bad_paths: with self.assertRaises(ConfigurationError) as config_error: config.update_single_config_from_path_and_value( bad_path, "value") self.assertEqual(config_error.exception.message, error_message)
def main(): parser = argparse.ArgumentParser("A command to test diffexp") parser.add_argument("dataset", help="name of a dataset to load") parser.add_argument("-na", "--numA", type=int, help="number of rows in group A") parser.add_argument("-nb", "--numB", type=int, help="number of rows in group B") parser.add_argument("-va", "--varA", help="obs variable:value to use for group A") parser.add_argument("-vb", "--varB", help="obs variable:value to use for group B") parser.add_argument("-t", "--trials", default=1, type=int, help="number of trials") parser.add_argument("-a", "--alg", choices=("default", "generic"), default="default", help="algorithm to use") parser.add_argument("-s", "--show", default=False, action="store_true", help="show the results") parser.add_argument("-n", "--new-selection", default=False, action="store_true", help="change the selection between each trial") parser.add_argument("--seed", default=1, type=int, help="set the random seed") args = parser.parse_args() app_config = AppConfig() app_config.update_server_config(single_dataset__datapath=args.dataset) app_config.update_server_config(app__verbose=True) app_config.complete_config() loader = MatrixDataLoader(args.dataset) adaptor = loader.open(app_config) random.seed(args.seed) np.random.seed(args.seed) rows = adaptor.get_shape()[0] if args.numA: filterA = random.sample(range(rows), args.numA) elif args.varA: vname, vval = args.varA.split(":") filterA = get_filter_from_obs(adaptor, vname, vval) else: print("must supply numA or varA") sys.exit(1) if args.numB: filterB = random.sample(range(rows), args.numB) elif args.varB: vname, vval = args.varB.split(":") filterB = get_filter_from_obs(adaptor, vname, vval) else: print("must supply numB or varB") sys.exit(1) for i in range(args.trials): if args.new_selection: if args.numA: filterA = random.sample(range(rows), args.numA) if args.numB: filterB = random.sample(range(rows), args.numB) maskA = np.zeros(rows, dtype=bool) maskA[filterA] = True maskB = np.zeros(rows, dtype=bool) maskB[filterB] = True t1 = time.time() if args.alg == "default": results = adaptor.compute_diffexp_ttest(maskA, maskB) elif args.alg == "generic": results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB) t2 = time.time() print("TIME=", t2 - t1) if args.show: for res in results: print(res)
def test_auth_test(self): app_config = AppConfig() app_config.update_server_config(app__flask_secret_key="secret") app_config.update_server_config(authentication__type="test") app_config.update_server_config( authentication__insecure_test_environment=True) app_config.update_server_config(multi_dataset__dataroot=dict( a1=dict(dataroot=self.dataset_dataroot, base_url="auth"), a2=dict(dataroot=self.dataset_dataroot, base_url="no-auth"), )) # specialize the configs app_config.add_dataroot_config("a1", app__authentication_enable=True, user_annotations__enable=True) app_config.add_dataroot_config("a2", app__authentication_enable=False, user_annotations__enable=False) app_config.complete_config() with test_server(app_config=app_config) as server: session = requests.Session() # auth datasets config = session.get( f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json() userinfo = session.get( f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json() self.assertFalse(userinfo["userinfo"]["is_authenticated"]) self.assertIsNone(userinfo["userinfo"]["username"]) self.assertTrue( config["config"]["authentication"]["requires_client_login"]) self.assertTrue(config["config"]["parameters"]["annotations"]) login_uri = config["config"]["authentication"]["login"] logout_uri = config["config"]["authentication"]["logout"] self.assertEqual(login_uri, "/login?dataset=auth/pbmc3k.cxg") self.assertEqual(logout_uri, "/logout?dataset=auth/pbmc3k.cxg") r = session.get(f"{server}/{login_uri}") # check that the login redirect worked self.assertEqual(r.history[0].status_code, 302) self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg") config = session.get( f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json() userinfo = session.get( f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json() self.assertTrue(userinfo["userinfo"]["is_authenticated"]) self.assertEqual(userinfo["userinfo"]["username"], "test_account") self.assertEqual(userinfo["userinfo"]["picture"], None) self.assertTrue(config["config"]["parameters"]["annotations"]) r = session.get(f"{server}/{logout_uri}") # check that the logout redirect worked self.assertEqual(r.history[0].status_code, 302) self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg") config = session.get( f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json() userinfo = session.get( f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json() self.assertFalse(userinfo["userinfo"]["is_authenticated"]) self.assertIsNone(userinfo["userinfo"]["username"]) self.assertTrue(config["config"]["parameters"]["annotations"]) # no-auth datasets config = session.get( f"{server}/no-auth/pbmc3k.cxg/api/v0.2/config").json() userinfo = session.get( f"{server}/no-auth/pbmc3k.cxg/api/v0.2/userinfo").json() self.assertIsNone(userinfo) self.assertFalse(config["config"]["parameters"]["annotations"]) # login with a picture session.get(f"{server}/{login_uri}&picture=myimage.png") userinfo = session.get( f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json() self.assertTrue(userinfo["userinfo"]["is_authenticated"]) self.assertEqual(userinfo["userinfo"]["picture"], "myimage.png")