def test_configfile_with_specialization(self):
# test that per_dataset_config config load the default config, then the specialized config
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
with open(configfile, "w") as fconfig:
config = """
server:
single_dataset:
datapath: fake_datapath
dataset:
user_annotations:
enable: false
type: local_file_csv
local_file_csv:
file: fake_file
directory: fake_dir
"""
fconfig.write(config)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
test_config = app_config.dataset_config
# test config from default
self.assertEqual(test_config.user_annotations__type, "local_file_csv")
self.assertEqual(test_config.user_annotations__local_file_csv__file, "fake_file")
def main():
parser = argparse.ArgumentParser(
"A script to check hosted configuration files")
parser.add_argument("config_file", help="the configuration file")
parser.add_argument(
"-s",
"--show",
default=False,
action="store_true",
help=
"print the configuration. NOTE: this may print secret values to stdout",
)
args = parser.parse_args()
app_config = AppConfig()
try:
app_config.update_from_config_file(args.config_file)
app_config.complete_config()
except Exception as e:
print(f"Error: {str(e)}")
print("FAIL:", args.config_file)
sys.exit(1)
if args.show:
yaml_config = app_config.config_to_dict()
yaml.dump(yaml_config, sys.stdout)
print("PASS:", args.config_file)
sys.exit(0)
def get_config(self, **kwargs):
file_name = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
)
config = AppConfig()
config.update_from_config_file(file_name)
return config
def test_configfile_no_dataset_section(self):
# test a config file without a dataset section
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
with open(configfile, "w") as fconfig:
config = """
server:
app:
flask_secret_key: secret
multi_dataset:
dataroot: test_dataroot
"""
fconfig.write(config)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
server_changes = app_config.server_config.changes_from_default()
dataset_changes = app_config.default_dataset_config.changes_from_default(
)
self.assertEqual(
server_changes,
[("app__flask_secret_key", "secret", None),
("multi_dataset__dataroot", "test_dataroot", None)],
)
self.assertEqual(dataset_changes, [])
def test_handle_data_locator_works_for_default_types(
self, mock_discover_region_name):
mock_discover_region_name.return_value = None
# Default config
self.assertEqual(
self.config.server_config.data_locator__s3__region_name, None)
# hard coded
config = self.get_config()
self.assertEqual(config.server_config.data_locator__s3__region_name,
"us-east-1")
# incorrectly formatted
dataroot = {
"d1": {
"base_url": "set1",
"dataroot": "/path/to/set1_datasets/"
},
"d2": {
"base_url": "set2/subdir",
"dataroot": "s3://shouldnt/work"
},
}
file_name = self.custom_app_config(
dataroot=dataroot,
config_file_name=self.config_file_name,
data_locater_region_name="true")
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_data_locator()
def get_config(self, **kwargs):
file_name = self.custom_app_config(
dataset_datapath=f"{H5AD_FIXTURE}",
config_file_name=self.config_file_name,
**kwargs)
config = AppConfig()
config.update_from_config_file(file_name)
return config
def test_get_dataset_config_returns_dataset_config_for_single_datasets(
self):
datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad"
file_name = self.custom_app_config(
dataset_datapath=datapath, config_file_name=self.config_file_name)
config = AppConfig()
config.update_from_config_file(file_name)
self.assertEqual(config.get_dataset_config(), config.dataset_config)
def test_handle_adaptor(self, mock_tiledb_context):
custom_config = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}", cxg_tile_cache_size=10, cxg_num_reader_threads=2
)
config = AppConfig()
config.update_from_config_file(custom_config)
config.server_config.handle_adaptor()
mock_tiledb_context.assert_called_once_with(
{"sm.tile_cache_size": 10, "sm.num_reader_threads": 2, "vfs.s3.region": "us-east-1"}
)
def test_aws_secrets_manager(self, mock_get_secret_key):
mock_get_secret_key.return_value = {
"oauth_client_secret": "mock_oauth_secret",
"db_uri": "mock_db_uri",
}
configfile = self.custom_external_config(
aws_secrets_manager_region="us-west-2",
aws_secrets_manager_secrets=[
dict(
name="my_secret",
values=[
dict(key="flask_secret_key",
path=["server", "app", "flask_secret_key"],
required=False),
dict(
key="db_uri",
path=[
"dataset", "user_annotations",
"hosted_tiledb_array", "db_uri"
],
required=True,
),
dict(
key="oauth_client_secret",
path=[
"server", "authentication", "params_oauth",
"client_secret"
],
required=True,
),
],
)
],
config_file_name="secret_external_config.yaml",
)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k.cxg"
app_config.server_config.app__flask_secret_key = "original"
app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k.cxg"
app_config.complete_config()
self.assertEqual(app_config.server_config.app__flask_secret_key,
"original")
self.assertEqual(
app_config.server_config.
authentication__params_oauth__client_secret, "mock_oauth_secret")
self.assertEqual(
app_config.default_dataset_config.
user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")
def test_handle_diffexp(self, mock_tiledb_config):
custom_config_file = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}",
cpu_multiplier=3,
diffexp_max_workers=1,
target_workunit=4,
config_file_name=self.config_file_name,
)
config = AppConfig()
config.update_from_config_file(custom_config_file)
config.server_config.handle_diffexp()
# called with the min of diffexp_max_workers and cpus*cpu_multiplier
mock_tiledb_config.assert_called_once_with(1, 4)
def test_handle_embeddings__checks_data_file_types(self):
file_name = self.custom_app_config(
embedding_names=["name1", "name2"],
enable_reembedding="true",
dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
anndata_backed="true",
config_file_name=self.config_file_name,
)
config = AppConfig()
config.update_from_config_file(file_name)
config.server_config.complete_config(self.context)
with self.assertRaises(ConfigurationError):
config.default_dataset_config.handle_embeddings()
def test_handle_data_locator_can_read_from_dataroot(self, mock_discover_region_name):
mock_discover_region_name.return_value = "us-west-2"
dataroot = {
"d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"},
"d2": {"base_url": "set2/subdir", "dataroot": "s3://hosted-cellxgene-dev"},
}
file_name = self.custom_app_config(
dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true"
)
config = AppConfig()
config.update_from_config_file(file_name)
config.server_config.handle_data_locator()
self.assertEqual(config.server_config.data_locator__s3__region_name, "us-west-2")
mock_discover_region_name.assert_called_once_with("s3://hosted-cellxgene-dev")
def test_handle_data_source__errors_when_passed_zero_or_two_dataroots(self):
file_name = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}",
config_file_name="two_data_roots.yml",
dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
)
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_data_source()
file_name = self.custom_app_config(config_file_name="zero_roots.yml")
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_data_source()
def test_config_for_single_dataset(self):
file_name = self.custom_app_config(
config_file_name="single_dataset.yml",
dataset_datapath=f"{H5AD_FIXTURE}")
config = AppConfig()
config.update_from_config_file(file_name)
config.server_config.handle_single_dataset(self.context)
file_name = self.custom_app_config(
config_file_name="single_dataset_with_about.yml",
about="www.cziscience.com",
dataset_datapath=f"{H5AD_FIXTURE}",
)
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_single_dataset(self.context)
def test_config_for_single_dataset(self):
file_name = self.custom_app_config(
config_file_name="single_dataset.yml",
dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg")
config = AppConfig()
config.update_from_config_file(file_name)
config.server_config.handle_single_dataset(self.context)
self.assertIsNotNone(config.server_config.matrix_data_cache_manager)
file_name = self.custom_app_config(
config_file_name="single_dataset_with_about.yml",
about="www.cziscience.com",
dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg",
)
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_single_dataset(self.context)
def test_configfile_with_specialization(self):
# test that per_dataset_config config load the default config, then the specialized config
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
with open(configfile, "w") as fconfig:
config = """
server:
multi_dataset:
dataroot:
test:
base_url: test
dataroot: fake_dataroot
dataset:
user_annotations:
enable: false
type: hosted_tiledb_array
hosted_tiledb_array:
db_uri: fake_db_uri
hosted_file_directory: fake_dir
per_dataset_config:
test:
user_annotations:
enable: true
"""
fconfig.write(config)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
test_config = app_config.dataroot_config["test"]
# test config from default
self.assertEqual(test_config.user_annotations__type,
"hosted_tiledb_array")
self.assertEqual(
test_config.user_annotations__hosted_tiledb_array__db_uri,
"fake_db_uri")
# test config from specialization
self.assertTrue(test_config.user_annotations__enable)
def test_configfile_no_server_section(self):
# test a config file without a dataset section
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
with open(configfile, "w") as fconfig:
config = """
dataset:
user_annotations:
enable: false
"""
fconfig.write(config)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
server_changes = app_config.server_config.changes_from_default()
dataset_changes = app_config.dataset_config.changes_from_default()
self.assertEqual(server_changes, [])
self.assertEqual(dataset_changes,
[("user_annotations__enable", False, True)])
def test_handle_data_locator_works_for_default_types(
self, mock_discover_region_name):
mock_discover_region_name.return_value = None
# Default config
self.assertEqual(
self.config.server_config.data_locator__s3__region_name, None)
# hard coded
config = self.get_config()
self.assertEqual(config.server_config.data_locator__s3__region_name,
"us-east-1")
# incorrectly formatted
datapath = "s3://shouldnt/work"
file_name = self.custom_app_config(
dataset_datapath=datapath,
config_file_name=self.config_file_name,
data_locater_region_name="true")
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_data_locator()
def launch(
datapath,
verbose,
debug,
open_browser,
port,
host,
embedding,
obs_names,
var_names,
max_category_items,
disable_custom_colors,
diffexp_lfc_cutoff,
title,
scripts,
about,
disable_annotations,
annotations_file,
user_generated_data_dir,
gene_sets_file,
disable_gene_sets_save,
backed,
disable_diffexp,
config_file,
dump_default_config,
x_approximate_distribution,
):
"""Launch the cellxgene data viewer.
This web app lets you explore single-cell expression data.
Data must be in a format that cellxgene expects.
Read the "getting started" guide to learn more:
https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/README.md
Examples:
> cellxgene launch example-dataset/pbmc3k.h5ad --title pbmc3k
> cellxgene launch <your data file> --title <your title>
> cellxgene launch <url>"""
if dump_default_config:
print(default_config)
sys.exit(0)
# Startup message
click.echo("[cellxgene] Starting the CLI...")
# app config
app_config = AppConfig()
server_config = app_config.server_config
try:
if config_file:
app_config.update_from_config_file(config_file)
# Determine which config options were give on the command line.
# Those will override the ones provided in the config file (if provided).
cli_config = AppConfig()
cli_config.update_server_config(
app__verbose=verbose,
app__debug=debug,
app__host=host,
app__port=port,
app__open_browser=open_browser,
single_dataset__datapath=datapath,
single_dataset__title=title,
single_dataset__about=about,
single_dataset__obs_names=obs_names,
single_dataset__var_names=var_names,
adaptor__anndata_adaptor__backed=backed,
)
cli_config.update_dataset_config(
app__scripts=scripts,
user_annotations__enable=not disable_annotations,
user_annotations__local_file_csv__file=annotations_file,
user_annotations__local_file_csv__directory=user_generated_data_dir,
user_annotations__local_file_csv__gene_sets_file=gene_sets_file,
user_annotations__gene_sets__readonly=disable_gene_sets_save,
presentation__max_categories=max_category_items,
presentation__custom_colors=not disable_custom_colors,
embeddings__names=embedding,
diffexp__enable=not disable_diffexp,
diffexp__lfc_cutoff=diffexp_lfc_cutoff,
X_approximate_distribution=x_approximate_distribution,
)
diff = cli_config.server_config.changes_from_default()
changes = {key: val for key, val, _ in diff}
app_config.update_server_config(**changes)
diff = cli_config.dataset_config.changes_from_default()
changes = {key: val for key, val, _ in diff}
app_config.update_dataset_config(**changes)
# process the configuration
# any errors will be thrown as an exception.
# any info messages will be passed to the messagefn function.
def messagefn(message):
click.echo("[cellxgene] " + message)
# Use a default secret if one is not provided
if not server_config.app__flask_secret_key:
app_config.update_server_config(
app__flask_secret_key="SparkleAndShine")
app_config.complete_config(messagefn)
except (ConfigurationError, DatasetAccessError) as e:
raise click.ClickException(e)
handle_scripts(scripts)
# create the server
server = CliLaunchServer(app_config)
if not server_config.app__verbose:
log = logging.getLogger("werkzeug")
log.setLevel(logging.ERROR)
cellxgene_url = f"http://{app_config.server_config.app__host}:{app_config.server_config.app__port}"
if server_config.app__open_browser:
click.echo(
f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now."
)
webbrowser.open(cellxgene_url)
else:
click.echo(
f"[cellxgene] Launching! Please go to {cellxgene_url} in your browser."
)
click.echo("[cellxgene] Type CTRL-C at any time to exit.")
if not server_config.app__verbose:
f = open(os.devnull, "w")
sys.stdout = f
try:
server.app.run(
host=server_config.app__host,
debug=server_config.app__debug,
port=server_config.app__port,
threaded=not server_config.app__debug,
use_debugger=False,
use_reloader=False,
)
except OSError as e:
if e.errno == errno.EADDRINUSE:
raise click.ClickException(
"Port is in use, please specify an open port using the --port flag."
) from e
raise
script_hashes = WSGIServer.load_static_csp_hashes(app)
script_hashes += WSGIServer.compute_inline_csp_hashes(app, app_config)
return script_hashes
try:
app_config = AppConfig()
has_config = False
# config file: look first for "config.yaml" in the current working directory
config_file = "config.yaml"
config_location = DataLocator(config_file)
if config_location.exists():
with config_location.local_handle() as lh:
logging.info(f"Configuration from {config_file}")
app_config.update_from_config_file(lh)
has_config = True
else:
# config file: second, use the CXG_CONFIG_FILE
config_file = os.getenv("CXG_CONFIG_FILE")
if config_file:
region_name = discover_s3_region_name(config_file)
config_location = DataLocator(config_file, region_name)
if config_location.exists():
with config_location.local_handle() as lh:
logging.info(f"Configuration from {config_file}")
app_config.update_from_config_file(lh)
has_config = True
else:
logging.critical(f"Configuration file not found {config_file}")
