def get_config(self, **kwargs): file_name = self.custom_app_config( dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs ) config = AppConfig() config.update_from_config_file(file_name) return config
def test_configfile_no_dataset_section(self): # test a config file without a dataset section with tempfile.TemporaryDirectory() as tempdir: configfile = os.path.join(tempdir, "config.yaml") with open(configfile, "w") as fconfig: config = """ server: app: flask_secret_key: secret multi_dataset: dataroot: test_dataroot """ fconfig.write(config) app_config = AppConfig() app_config.update_from_config_file(configfile) server_changes = app_config.server_config.changes_from_default() dataset_changes = app_config.default_dataset_config.changes_from_default( ) self.assertEqual( server_changes, [("app__flask_secret_key", "secret", None), ("multi_dataset__dataroot", "test_dataroot", None)], ) self.assertEqual(dataset_changes, [])
def main(): parser = argparse.ArgumentParser( "A script to check hosted configuration files") parser.add_argument("config_file", help="the configuration file") parser.add_argument( "-s", "--show", default=False, action="store_true", help= "print the configuration. NOTE: this may print secret values to stdout", ) args = parser.parse_args() app_config = AppConfig() try: app_config.update_from_config_file(args.config_file) app_config.complete_config() except Exception as e: print(f"Error: {str(e)}") print("FAIL:", args.config_file) sys.exit(1) if args.show: yaml_config = app_config.config_to_dict() yaml.dump(yaml_config, sys.stdout) print("PASS:", args.config_file) sys.exit(0)
def test_get_dataset_config_returns_default_dataset_config_for_single_datasets( self): datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad" file_name = self.custom_app_config( dataset_datapath=datapath, config_file_name=self.config_file_name) config = AppConfig() config.update_from_config_file(file_name) self.assertEqual(config.get_dataset_config(""), config.default_dataset_config)
def test_handle_adaptor(self, mock_tiledb_context): custom_config = self.custom_app_config( dataroot=f"{FIXTURES_ROOT}", cxg_tile_cache_size=10, cxg_num_reader_threads=2 ) config = AppConfig() config.update_from_config_file(custom_config) config.server_config.handle_adaptor() mock_tiledb_context.assert_called_once_with( {"sm.tile_cache_size": 10, "sm.num_reader_threads": 2, "vfs.s3.region": "us-east-1"} )
def test_aws_secrets_manager(self, mock_get_secret_key): mock_get_secret_key.return_value = { "oauth_client_secret": "mock_oauth_secret", "db_uri": "mock_db_uri", } configfile = self.custom_external_config( aws_secrets_manager_region="us-west-2", aws_secrets_manager_secrets=[ dict( name="my_secret", values=[ dict(key="flask_secret_key", path=["server", "app", "flask_secret_key"], required=False), dict( key="db_uri", path=[ "dataset", "user_annotations", "hosted_tiledb_array", "db_uri" ], required=True, ), dict( key="oauth_client_secret", path=[ "server", "authentication", "params_oauth", "client_secret" ], required=True, ), ], ) ], config_file_name="secret_external_config.yaml", ) app_config = AppConfig() app_config.update_from_config_file(configfile) app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k.cxg" app_config.server_config.app__flask_secret_key = "original" app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k.cxg" app_config.complete_config() self.assertEqual(app_config.server_config.app__flask_secret_key, "original") self.assertEqual( app_config.server_config. authentication__params_oauth__client_secret, "mock_oauth_secret") self.assertEqual( app_config.default_dataset_config. user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")
def test_handle_embeddings__checks_data_file_types(self): file_name = self.custom_app_config( embedding_names=["name1", "name2"], enable_reembedding="true", dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", anndata_backed="true", config_file_name=self.config_file_name, ) config = AppConfig() config.update_from_config_file(file_name) config.server_config.complete_config(self.context) with self.assertRaises(ConfigurationError): config.default_dataset_config.handle_embeddings()
def test_handle_diffexp(self, mock_tiledb_config): custom_config_file = self.custom_app_config( dataroot=f"{FIXTURES_ROOT}", cpu_multiplier=3, diffexp_max_workers=1, target_workunit=4, config_file_name=self.config_file_name, ) config = AppConfig() config.update_from_config_file(custom_config_file) config.server_config.handle_diffexp() # called with the min of diffexp_max_workers and cpus*cpu_multiplier mock_tiledb_config.assert_called_once_with(1, 4)
def test_handle_data_locator_can_read_from_dataroot(self, mock_discover_region_name): mock_discover_region_name.return_value = "us-west-2" dataroot = { "d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"}, "d2": {"base_url": "set2/subdir", "dataroot": "s3://hosted-cellxgene-dev"}, } file_name = self.custom_app_config( dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true" ) config = AppConfig() config.update_from_config_file(file_name) config.server_config.handle_data_locator() self.assertEqual(config.server_config.data_locator__s3__region_name, "us-west-2") mock_discover_region_name.assert_called_once_with("s3://hosted-cellxgene-dev")
def test_handle_data_source__errors_when_passed_zero_or_two_dataroots(self): file_name = self.custom_app_config( dataroot=f"{FIXTURES_ROOT}", config_file_name="two_data_roots.yml", dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", ) config = AppConfig() config.update_from_config_file(file_name) with self.assertRaises(ConfigurationError): config.server_config.handle_data_source() file_name = self.custom_app_config(config_file_name="zero_roots.yml") config = AppConfig() config.update_from_config_file(file_name) with self.assertRaises(ConfigurationError): config.server_config.handle_data_source()
def test_config_for_single_dataset(self): file_name = self.custom_app_config( config_file_name="single_dataset.yml", dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg" ) config = AppConfig() config.update_from_config_file(file_name) config.server_config.handle_single_dataset(self.context) self.assertIsNotNone(config.server_config.matrix_data_cache_manager) file_name = self.custom_app_config( config_file_name="single_dataset_with_about.yml", about="www.cziscience.com", dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg", ) config = AppConfig() config.update_from_config_file(file_name) with self.assertRaises(ConfigurationError): config.server_config.handle_single_dataset(self.context)
def test_handle_data_locator_works_for_default_types(self, mock_discover_region_name): mock_discover_region_name.return_value = None # Default config self.assertEqual(self.config.server_config.data_locator__s3__region_name, None) # hard coded config = self.get_config() self.assertEqual(config.server_config.data_locator__s3__region_name, "us-east-1") # incorrectly formatted dataroot = { "d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"}, "d2": {"base_url": "set2/subdir", "dataroot": "s3://shouldnt/work"}, } file_name = self.custom_app_config( dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true" ) config = AppConfig() config.update_from_config_file(file_name) with self.assertRaises(ConfigurationError): config.server_config.handle_data_locator()
def test_configfile_no_server_section(self): # test a config file without a dataset section with tempfile.TemporaryDirectory() as tempdir: configfile = os.path.join(tempdir, "config.yaml") with open(configfile, "w") as fconfig: config = """ dataset: user_annotations: enable: false """ fconfig.write(config) app_config = AppConfig() app_config.update_from_config_file(configfile) server_changes = app_config.server_config.changes_from_default() dataset_changes = app_config.default_dataset_config.changes_from_default( ) self.assertEqual(server_changes, []) self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)])
def test_configfile_with_specialization(self): # test that per_dataset_config config load the default config, then the specialized config with tempfile.TemporaryDirectory() as tempdir: configfile = os.path.join(tempdir, "config.yaml") with open(configfile, "w") as fconfig: config = """ server: multi_dataset: dataroot: test: base_url: test dataroot: fake_dataroot dataset: user_annotations: enable: false type: hosted_tiledb_array hosted_tiledb_array: db_uri: fake_db_uri hosted_file_directory: fake_dir per_dataset_config: test: user_annotations: enable: true """ fconfig.write(config) app_config = AppConfig() app_config.update_from_config_file(configfile) test_config = app_config.dataroot_config["test"] # test config from default self.assertEqual(test_config.user_annotations__type, "hosted_tiledb_array") self.assertEqual(test_config.user_annotations__hosted_tiledb_array__db_uri, "fake_db_uri") # test config from specialization self.assertTrue(test_config.user_annotations__enable)
def launch( datapath, dataroot, verbose, debug, open_browser, port, host, embedding, obs_names, var_names, max_category_items, disable_custom_colors, diffexp_lfc_cutoff, title, scripts, about, disable_annotations, annotations_file, annotations_dir, backed, disable_diffexp, experimental_annotations_ontology, experimental_annotations_ontology_obo, experimental_enable_reembedding, config_file, dump_default_config, ): """Launch the cellxgene data viewer. This web app lets you explore single-cell expression data. Data must be in a format that cellxgene expects. Read the "getting started" guide to learn more: https://chanzuckerberg.github.io/cellxgene/getting-started.html Examples: > cellxgene launch example-dataset/pbmc3k.h5ad --title pbmc3k > cellxgene launch <your data file> --title <your title> > cellxgene launch <url>""" # TODO Examples to provide when "--dataroot" is unhidden # > cellxgene launch --dataroot example-dataset/ # # > cellxgene launch --dataroot <url> if dump_default_config: print(default_config) sys.exit(0) # Startup message click.echo("[cellxgene] Starting the CLI...") # app config app_config = AppConfig() server_config = app_config.server_config try: if config_file: app_config.update_from_config_file(config_file) # Determine which config options were give on the command line. # Those will override the ones provided in the config file (if provided). cli_config = AppConfig() cli_config.update_server_config( app__verbose=verbose, app__debug=debug, app__host=host, app__port=port, app__open_browser=open_browser, single_dataset__datapath=datapath, single_dataset__title=title, single_dataset__about=about, single_dataset__obs_names=obs_names, single_dataset__var_names=var_names, multi_dataset__dataroot=dataroot, adaptor__anndata_adaptor__backed=backed, ) cli_config.update_default_dataset_config( app__scripts=scripts, user_annotations__enable=not disable_annotations, user_annotations__local_file_csv__file=annotations_file, user_annotations__local_file_csv__directory=annotations_dir, user_annotations__ontology__enable= experimental_annotations_ontology, user_annotations__ontology__obo_location= experimental_annotations_ontology_obo, presentation__max_categories=max_category_items, presentation__custom_colors=not disable_custom_colors, embeddings__names=embedding, embeddings__enable_reembedding=experimental_enable_reembedding, diffexp__enable=not disable_diffexp, diffexp__lfc_cutoff=diffexp_lfc_cutoff, ) diff = cli_config.server_config.changes_from_default() changes = {key: val for key, val, _ in diff} app_config.update_server_config(**changes) diff = cli_config.default_dataset_config.changes_from_default() changes = {key: val for key, val, _ in diff} app_config.update_default_dataset_config(**changes) # process the configuration # any errors will be thrown as an exception. # any info messages will be passed to the messagefn function. def messagefn(message): click.echo("[cellxgene] " + message) # Use a default secret if one is not provided if not server_config.app__flask_secret_key: app_config.update_server_config( app__flask_secret_key="SparkleAndShine") app_config.complete_config(messagefn) except (ConfigurationError, DatasetAccessError) as e: raise click.ClickException(e) handle_scripts(scripts) # create the server server = CliLaunchServer(app_config) if not server_config.app__verbose: log = logging.getLogger("werkzeug") log.setLevel(logging.ERROR) cellxgene_url = f"http://{app_config.server_config.app__host}:{app_config.server_config.app__port}" if server_config.app__open_browser: click.echo( f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now." ) webbrowser.open(cellxgene_url) else: click.echo( f"[cellxgene] Launching! Please go to {cellxgene_url} in your browser." ) click.echo("[cellxgene] Type CTRL-C at any time to exit.") if not server_config.app__verbose: f = open(os.devnull, "w") sys.stdout = f try: server.app.run( host=server_config.app__host, debug=server_config.app__debug, port=server_config.app__port, threaded=not server_config.app__debug, use_debugger=False, use_reloader=False, ) except OSError as e: if e.errno == errno.EADDRINUSE: raise click.ClickException( "Port is in use, please specify an open port using the --port flag." ) from e raise
script_hashes = WSGIServer.load_static_csp_hashes(app) script_hashes += WSGIServer.compute_inline_csp_hashes(app, app_config) return script_hashes try: app_config = AppConfig() has_config = False # config file: look first for "config.yaml" in the current working directory config_file = "config.yaml" config_location = DataLocator(config_file) if config_location.exists(): with config_location.local_handle() as lh: logging.info(f"Configuration from {config_file}") app_config.update_from_config_file(lh) has_config = True else: # config file: second, use the CXG_CONFIG_FILE config_file = os.getenv("CXG_CONFIG_FILE") if config_file: region_name = discover_s3_region_name(config_file) config_location = DataLocator(config_file, region_name) if config_location.exists(): with config_location.local_handle() as lh: logging.info(f"Configuration from {config_file}") app_config.update_from_config_file(lh) has_config = True else: logging.critical(f"Configuration file not found {config_file}")