def test_configfile_with_specialization(self): # test that per_dataset_config config load the default config, then the specialized config with tempfile.TemporaryDirectory() as tempdir: configfile = os.path.join(tempdir, "config.yaml") with open(configfile, "w") as fconfig: config = """ server: single_dataset: datapath: fake_datapath dataset: user_annotations: enable: false type: local_file_csv local_file_csv: file: fake_file directory: fake_dir """ fconfig.write(config) app_config = AppConfig() app_config.update_from_config_file(configfile) test_config = app_config.dataset_config # test config from default self.assertEqual(test_config.user_annotations__type, "local_file_csv") self.assertEqual(test_config.user_annotations__local_file_csv__file, "fake_file")
def test_handle_data_locator_works_for_default_types( self, mock_discover_region_name): mock_discover_region_name.return_value = None # Default config self.assertEqual( self.config.server_config.data_locator__s3__region_name, None) # hard coded config = self.get_config() self.assertEqual(config.server_config.data_locator__s3__region_name, "us-east-1") # incorrectly formatted dataroot = { "d1": { "base_url": "set1", "dataroot": "/path/to/set1_datasets/" }, "d2": { "base_url": "set2/subdir", "dataroot": "s3://shouldnt/work" }, } file_name = self.custom_app_config( dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true") config = AppConfig() config.update_from_config_file(file_name) with self.assertRaises(ConfigurationError): config.server_config.handle_data_locator()
def test_configfile_no_dataset_section(self): # test a config file without a dataset section with tempfile.TemporaryDirectory() as tempdir: configfile = os.path.join(tempdir, "config.yaml") with open(configfile, "w") as fconfig: config = """ server: app: flask_secret_key: secret multi_dataset: dataroot: test_dataroot """ fconfig.write(config) app_config = AppConfig() app_config.update_from_config_file(configfile) server_changes = app_config.server_config.changes_from_default() dataset_changes = app_config.default_dataset_config.changes_from_default( ) self.assertEqual( server_changes, [("app__flask_secret_key", "secret", None), ("multi_dataset__dataroot", "test_dataroot", None)], ) self.assertEqual(dataset_changes, [])
def get_config(self, **kwargs): file_name = self.custom_app_config( dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs ) config = AppConfig() config.update_from_config_file(file_name) return config
def get_config(self, **kwargs): file_name = self.custom_app_config( dataset_datapath=f"{H5AD_FIXTURE}", config_file_name=self.config_file_name, **kwargs) config = AppConfig() config.update_from_config_file(file_name) return config
def test_get_dataset_config_returns_dataset_config_for_single_datasets( self): datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad" file_name = self.custom_app_config( dataset_datapath=datapath, config_file_name=self.config_file_name) config = AppConfig() config.update_from_config_file(file_name) self.assertEqual(config.get_dataset_config(), config.dataset_config)
def test_handle_adaptor(self, mock_tiledb_context): custom_config = self.custom_app_config( dataroot=f"{FIXTURES_ROOT}", cxg_tile_cache_size=10, cxg_num_reader_threads=2 ) config = AppConfig() config.update_from_config_file(custom_config) config.server_config.handle_adaptor() mock_tiledb_context.assert_called_once_with( {"sm.tile_cache_size": 10, "sm.num_reader_threads": 2, "vfs.s3.region": "us-east-1"} )
def test_handle_app___can_use_envar_port(self): config = self.get_config(port=24) self.assertEqual(config.server_config.app__port, 24) # Note if the port is set in the config file it will NOT be overwritten by a different envvar os.environ["CXG_SERVER_PORT"] = "4008" self.config = AppConfig() self.config.update_server_config(app__flask_secret_key="secret") self.config.server_config.handle_app(self.context) self.assertEqual(self.config.server_config.app__port, 4008) del os.environ["CXG_SERVER_PORT"]
def test_handle_embeddings__checks_data_file_types(self): file_name = self.custom_app_config( embedding_names=["name1", "name2"], enable_reembedding="true", dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", anndata_backed="true", config_file_name=self.config_file_name, ) config = AppConfig() config.update_from_config_file(file_name) config.server_config.complete_config(self.context) with self.assertRaises(ConfigurationError): config.default_dataset_config.handle_embeddings()
def test_handle_diffexp(self, mock_tiledb_config): custom_config_file = self.custom_app_config( dataroot=f"{FIXTURES_ROOT}", cpu_multiplier=3, diffexp_max_workers=1, target_workunit=4, config_file_name=self.config_file_name, ) config = AppConfig() config.update_from_config_file(custom_config_file) config.server_config.handle_diffexp() # called with the min of diffexp_max_workers and cpus*cpu_multiplier mock_tiledb_config.assert_called_once_with(1, 4)
def setUp(self): self.data_file = DataLocator(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad") config = AppConfig() config.update_server_config(single_dataset__datapath=self.data_file.path) config.update_server_config(app__flask_secret_key="secret") config.complete_config() self.data = AnndataAdaptor(self.data_file, config)
def data_with_tmp_tiledb_annotations(ext: MatrixDataType): tmp_dir = tempfile.mkdtemp() fname = { MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", MatrixDataType.CXG: "test/fixtures/pbmc3k.cxg", }[ext] data_locator = DataLocator(fname) config = AppConfig() config.update_server_config( app__flask_secret_key="secret", multi_dataset__dataroot=data_locator.path, authentication__type="test", authentication__insecure_test_environment=True, ) config.update_default_dataset_config( embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01, user_annotations__type="hosted_tiledb_array", user_annotations__hosted_tiledb_array__db_uri= "postgresql://*****:*****@localhost:5432", user_annotations__hosted_tiledb_array__hosted_file_directory=tmp_dir, ) config.complete_config() data = MatrixDataLoader(data_locator.abspath()).open(config) annotations = AnnotationsHostedTileDB( tmp_dir, DbUtils("postgresql://*****:*****@localhost:5432"), ) return data, tmp_dir, annotations
def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False): tmp_dir = tempfile.mkdtemp() annotations_file = path.join(tmp_dir, "test_annotations.csv") if annotations_fixture: shutil.copyfile( f"{PROJECT_ROOT}/server/test/fixtures/pbmc3k-annotations.csv", annotations_file) fname = { MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", MatrixDataType.CXG: "test/fixtures/pbmc3k.cxg", }[ext] data_locator = DataLocator(fname) config = AppConfig() config.update_server_config( app__flask_secret_key="secret", single_dataset__obs_names=None, single_dataset__var_names=None, single_dataset__datapath=data_locator.path, ) config.update_default_dataset_config( embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01, ) config.complete_config() data = MatrixDataLoader(data_locator.abspath()).open(config) annotations = AnnotationsLocalFile(None, annotations_file) return data, tmp_dir, annotations
def test_handle_data_locator_can_read_from_dataroot(self, mock_discover_region_name): mock_discover_region_name.return_value = "us-west-2" dataroot = { "d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"}, "d2": {"base_url": "set2/subdir", "dataroot": "s3://hosted-cellxgene-dev"}, } file_name = self.custom_app_config( dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true" ) config = AppConfig() config.update_from_config_file(file_name) config.server_config.handle_data_locator() self.assertEqual(config.server_config.data_locator__s3__region_name, "us-west-2") mock_discover_region_name.assert_called_once_with("s3://hosted-cellxgene-dev")
def setUp(self): self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" self.config = AppConfig() self.config.update_server_config(app__flask_secret_key="secret") self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE) self.dataset_config = self.config.dataset_config self.config.complete_config() message_list = [] def noop(message): message_list.append(message) messagefn = noop self.context = dict(messagefn=messagefn, messages=message_list)
def test_auth_test_single(self): app_config = AppConfig() app_config.update_server_config(app__flask_secret_key="secret") app_config.update_server_config( authentication__type="test", single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg") app_config.update_server_config( authentication__insecure_test_environment=True) app_config.complete_config() with test_server(app_config=app_config) as server: session = requests.Session() config = session.get(f"{server}/api/v0.2/config").json() userinfo = session.get(f"{server}/api/v0.2/userinfo").json() self.assertFalse(userinfo["userinfo"]["is_authenticated"]) self.assertIsNone(userinfo["userinfo"]["username"]) self.assertTrue( config["config"]["authentication"]["requires_client_login"]) self.assertTrue(config["config"]["parameters"]["annotations"]) login_uri = config["config"]["authentication"]["login"] logout_uri = config["config"]["authentication"]["logout"] self.assertEqual(login_uri, "/login") self.assertEqual(logout_uri, "/logout") response = session.get(f"{server}/{login_uri}") # check that the login redirect worked self.assertEqual(response.history[0].status_code, 302) self.assertEqual(response.url, f"{server}/") config = session.get(f"{server}/api/v0.2/config").json() userinfo = session.get(f"{server}/api/v0.2/userinfo").json() self.assertTrue(userinfo["userinfo"]["is_authenticated"]) self.assertEqual(userinfo["userinfo"]["username"], "test_account") self.assertTrue(config["config"]["parameters"]["annotations"]) response = session.get(f"{server}/{logout_uri}") # check that the logout redirect worked self.assertEqual(response.history[0].status_code, 302) self.assertEqual(response.url, f"{server}/") config = session.get(f"{server}/api/v0.2/config").json() userinfo = session.get(f"{server}/api/v0.2/userinfo").json() self.assertFalse(userinfo["userinfo"]["is_authenticated"]) self.assertIsNone(userinfo["userinfo"]["username"]) self.assertTrue(config["config"]["parameters"]["annotations"])
def get_basic_config(self): config = AppConfig() config.update_server_config( single_dataset__obs_names=None, single_dataset__var_names=None, ) config.update_server_config(app__flask_secret_key="secret") config.update_default_dataset_config( embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01, ) return config
def main(): parser = argparse.ArgumentParser( "A script to check hosted configuration files") parser.add_argument("config_file", help="the configuration file") parser.add_argument( "-s", "--show", default=False, action="store_true", help= "print the configuration. NOTE: this may print secret values to stdout", ) args = parser.parse_args() app_config = AppConfig() try: app_config.update_from_config_file(args.config_file) app_config.complete_config() except Exception as e: print(f"Error: {str(e)}") print("FAIL:", args.config_file) sys.exit(1) if args.show: yaml_config = app_config.config_to_dict() yaml.dump(yaml_config, sys.stdout) print("PASS:", args.config_file) sys.exit(0)
def test_mapping_creation_returns_map_of_server_and_dataset_config(self): config = AppConfig() mapping = config.default_dataset_config.create_mapping( config.default_config) self.assertIsNotNone(mapping["server__app__verbose"]) self.assertIsNotNone(mapping["dataset__presentation__max_categories"]) self.assertIsNotNone( mapping["dataset__user_annotations__ontology__obo_location"]) self.assertIsNotNone( mapping["server__multi_dataset__allowed_matrix_types"])
def test_auth_none(self): app_config = AppConfig() app_config.update_server_config(app__flask_secret_key="secret") app_config.update_server_config( authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot) app_config.update_default_dataset_config( user_annotations__enable=False) app_config.complete_config() with test_server(app_config=app_config) as server: session = requests.Session() config = session.get( f"{server}/d/pbmc3k.cxg/api/v0.2/config").json() userinfo = session.get( f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json() self.assertNotIn("authentication", config["config"]) self.assertIsNone(userinfo)
def test_init_datatset_config_sets_vars_from_default_config(self): config = AppConfig() self.assertEqual( config.default_dataset_config.presentation__max_categories, 1000) self.assertEqual(config.default_dataset_config.user_annotations__type, "local_file_csv") self.assertEqual(config.default_dataset_config.diffexp__lfc_cutoff, 0.01) self.assertIsNone(config.default_dataset_config. user_annotations__ontology__obo_location)
def test_configfile_with_specialization(self): # test that per_dataset_config config load the default config, then the specialized config with tempfile.TemporaryDirectory() as tempdir: configfile = os.path.join(tempdir, "config.yaml") with open(configfile, "w") as fconfig: config = """ server: multi_dataset: dataroot: test: base_url: test dataroot: fake_dataroot dataset: user_annotations: enable: false type: hosted_tiledb_array hosted_tiledb_array: db_uri: fake_db_uri hosted_file_directory: fake_dir per_dataset_config: test: user_annotations: enable: true """ fconfig.write(config) app_config = AppConfig() app_config.update_from_config_file(configfile) test_config = app_config.dataroot_config["test"] # test config from default self.assertEqual(test_config.user_annotations__type, "hosted_tiledb_array") self.assertEqual( test_config.user_annotations__hosted_tiledb_array__db_uri, "fake_db_uri") # test config from specialization self.assertTrue(test_config.user_annotations__enable)
def test_auth_session(self): app_config = AppConfig() app_config.update_server_config(app__flask_secret_key="secret") app_config.update_server_config( authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot) app_config.update_default_dataset_config(user_annotations__enable=True) app_config.complete_config() with test_server(app_config=app_config) as server: session = requests.Session() config = session.get( f"{server}/d/pbmc3k.cxg/api/v0.2/config").json() userinfo = session.get( f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json() self.assertFalse( config["config"]["authentication"]["requires_client_login"]) self.assertTrue(userinfo["userinfo"]["is_authenticated"]) self.assertEqual(userinfo["userinfo"]["username"], "anonymous")
def test_handle_data_locator_works_for_default_types( self, mock_discover_region_name): mock_discover_region_name.return_value = None # Default config self.assertEqual( self.config.server_config.data_locator__s3__region_name, None) # hard coded config = self.get_config() self.assertEqual(config.server_config.data_locator__s3__region_name, "us-east-1") # incorrectly formatted datapath = "s3://shouldnt/work" file_name = self.custom_app_config( dataset_datapath=datapath, config_file_name=self.config_file_name, data_locater_region_name="true") config = AppConfig() config.update_from_config_file(file_name) with self.assertRaises(ConfigurationError): config.server_config.handle_data_locator()
def test_configfile_no_server_section(self): # test a config file without a dataset section with tempfile.TemporaryDirectory() as tempdir: configfile = os.path.join(tempdir, "config.yaml") with open(configfile, "w") as fconfig: config = """ dataset: user_annotations: enable: false """ fconfig.write(config) app_config = AppConfig() app_config.update_from_config_file(configfile) server_changes = app_config.server_config.changes_from_default() dataset_changes = app_config.dataset_config.changes_from_default() self.assertEqual(server_changes, []) self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)])
def test_get_api_base_url_works(self): # test the api_base_url feature, and that it can contain a path config = AppConfig() backend_port = find_available_port("localhost", 10000) config.update_server_config( app__flask_secret_key="secret", app__api_base_url= f"http://localhost:{backend_port}/additional/path", multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset", ) config.complete_config() with test_server(["-p", str(backend_port)], app_config=config) as server: session = requests.Session() self.assertEqual(server, f"http://localhost:{backend_port}") response = session.get( f"{server}/additional/path/d/pbmc3k.h5ad/api/v0.2/config") self.assertEqual(response.status_code, 200) data_config = response.json() self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k") # test the health check at the correct url response = session.get(f"{server}/additional/path/health") assert response.json()["status"] == "pass"
def test_config(self): check_config_script = os.path.join(PROJECT_ROOT, "server", "eb", "check_config.py") with tempfile.TemporaryDirectory() as tempdir: configfile = os.path.join(tempdir, "config.yaml") app_config = AppConfig() app_config.update_server_config( multi_dataset__dataroot=f"{FIXTURES_ROOT}") app_config.write_config(configfile) command = ["python", check_config_script, configfile] # test failure mode (flask_secret_key not set) env = os.environ.copy() env.pop("CXG_SECRET_KEY", None) with self.assertRaises( subprocess.CalledProcessError) as exception_context: subprocess.check_output(command, env=env) output = str(exception_context.exception.stdout, "utf-8") self.assertTrue( output.startswith( "Error: Invalid type for attribute: app__flask_secret_key, expected type str, got NoneType" )) self.assertEqual(exception_context.exception.returncode, 1) # test passing case env = os.environ.copy() env["CXG_SECRET_KEY"] = "secret" output = subprocess.check_output(command, env=env) output = str(output, "utf-8") self.assertTrue(output.startswith("PASS"))
def test_handle_data_source__errors_when_passed_zero_or_two_dataroots(self): file_name = self.custom_app_config( dataroot=f"{FIXTURES_ROOT}", config_file_name="two_data_roots.yml", dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", ) config = AppConfig() config.update_from_config_file(file_name) with self.assertRaises(ConfigurationError): config.server_config.handle_data_source() file_name = self.custom_app_config(config_file_name="zero_roots.yml") config = AppConfig() config.update_from_config_file(file_name) with self.assertRaises(ConfigurationError): config.server_config.handle_data_source()
def test_get_default_config_correctly_reads_default_config_file(self): app_default_config = AppConfig().default_config expected_config = yaml.load(default_config, Loader=yaml.Loader) server_config = app_default_config["server"] dataset_config = app_default_config["dataset"] expected_server_config = expected_config["server"] expected_dataset_config = expected_config["dataset"] self.assertDictEqual(app_default_config, expected_config) self.assertDictEqual(server_config, expected_server_config) self.assertDictEqual(dataset_config, expected_dataset_config)
class BaseConfigTest(ConfigTests): def setUp(self): self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" self.config = AppConfig() self.config.update_server_config(app__flask_secret_key="secret") self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE) self.server_config = self.config.server_config self.config.complete_config() message_list = [] def noop(message): message_list.append(message) messagefn = noop self.context = dict(messagefn=messagefn, messages=message_list) def get_config(self, **kwargs): file_name = self.custom_app_config( dataset_datapath=f"{H5AD_FIXTURE}", config_file_name=self.config_file_name, **kwargs) config = AppConfig() config.update_from_config_file(file_name) return config def test_mapping_creation_returns_map_of_server_and_dataset_config(self): config = AppConfig() mapping = config.dataset_config.create_mapping(config.default_config) self.assertIsNotNone(mapping["server__app__verbose"]) self.assertIsNotNone(mapping["dataset__presentation__max_categories"]) def test_changes_from_default_returns_list_of_nondefault_config_values( self): config = self.get_config(verbose="true", lfc_cutoff=0.05) server_changes = config.server_config.changes_from_default() dataset_changes = config.dataset_config.changes_from_default() self.assertEqual( server_changes, [ ("app__verbose", True, False), ("app__flask_secret_key", "secret", None), ("single_dataset__datapath", H5AD_FIXTURE, None), ("data_locator__s3__region_name", "us-east-1", True), ], ) self.assertEqual(dataset_changes, [("diffexp__lfc_cutoff", 0.05, 0.01)]) def test_check_config_throws_error_if_attr_has_not_been_checked(self): config = self.get_config(verbose="true") config.complete_config() config.check_config() config.update_server_config(app__verbose=False) with self.assertRaises(ConfigurationError): config.check_config()