def handle_environment(self, context):
"""For each environment variable defined, get the value (if it is set),
and set the specified config parameter"""
self.validate_correct_type_of_configuration_attribute(
"environment", list)
for envdict in self.environment:
name = envdict.get("name")
if name is None:
raise ConfigurationError("environment: 'name' is missing")
required = envdict.get("required", False)
if type(required) != bool:
raise ConfigurationError(
"environment: 'required' must be a bool")
path = envdict.get("path")
if path is None:
raise ConfigurationError("environment: 'path' is missing")
value = os.environ.get(name)
if value is None:
if required:
raise ConfigurationError(
f"required environment variable '{name}' not set")
else:
value = convert_string_to_value(value)
self.app_config.update_single_config_from_path_and_value(
path, value)
def handle_local_file_csv_annotations(self):
dirname = self.user_annotations__local_file_csv__directory
filename = self.user_annotations__local_file_csv__file
if filename is not None and dirname is not None:
raise ConfigurationError(
"'annotations-file' and 'annotations-dir' may not be used together."
)
if filename is not None:
lf_name, lf_ext = splitext(filename)
if lf_ext and lf_ext != ".csv":
raise ConfigurationError(
f"annotation file type must be .csv: {filename}")
if dirname is not None and not isdir(dirname):
try:
os.mkdir(dirname)
except OSError:
raise ConfigurationError(
"Unable to create directory specified by --annotations-dir"
)
self.user_annotations = AnnotationsLocalFile(dirname, filename)
# if the user has specified a fixed label file, go ahead and validate it
# so that we can remove errors early in the process.
server_config = self.app_config.server_config
if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file:
data_adaptor = self.get_data_adaptor()
data_adaptor.check_new_labels(
self.user_annotations.read_labels(data_adaptor))
def update_single_config_from_path_and_value(self, path, value):
"""Update a single config parameter with the value.
Path is a list of string, that gives a path to the config parameter to be updated.
For example, path may be ["server","app","port"].
"""
self.is_complete = False
if not isinstance(path, list):
raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
for part in path:
if not isinstance(part, str):
raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
if len(path) < 1 or path[0] not in ("server", "dataset"):
raise ConfigurationError("path must start with 'server', or 'dataset'")
if path[0] == "server":
attr = "__".join(path[1:])
try:
self.update_server_config(**{attr: value})
except ConfigurationError:
raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
elif path[0] == "dataset":
attr = "__".join(path[1:])
try:
self.update_dataset_config(**{attr: value})
except ConfigurationError:
raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
def handle_authentication(self):
self.validate_correct_type_of_configuration_attribute(
"authentication__type", (type(None), str))
self.validate_correct_type_of_configuration_attribute(
"authentication__insecure_test_environment", bool)
if self.authentication__type == "test" and not self.authentication__insecure_test_environment:
raise ConfigurationError(
"Test auth can only be used in an insecure test environment")
self.auth = AuthTypeFactory.create(self.authentication__type, self)
if self.auth is None:
raise ConfigurationError(
f"Unknown authentication type: {self.authentication__type}")
def update(self, **kw):
"""Update the attributes defined in kw with their new values."""
for key, value in kw.items():
if not hasattr(self, key):
raise ConfigurationError(f"unknown config parameter {key}.")
try:
if type(value) == tuple:
# convert tuple values to list values
value = list(value)
setattr(self, key, value)
except KeyError:
raise ConfigurationError(
f"Unable to set config parameter {key}.")
self.attr_checked[key] = False
def handle_app(self, context):
self.validate_correct_type_of_configuration_attribute(
"app__verbose", bool)
self.validate_correct_type_of_configuration_attribute(
"app__debug", bool)
self.validate_correct_type_of_configuration_attribute("app__host", str)
self.validate_correct_type_of_configuration_attribute(
"app__port", (type(None), int))
self.validate_correct_type_of_configuration_attribute(
"app__open_browser", bool)
self.validate_correct_type_of_configuration_attribute(
"app__force_https", bool)
self.validate_correct_type_of_configuration_attribute(
"app__flask_secret_key", str)
if self.app__port:
try:
if not is_port_available(self.app__host, self.app__port):
raise ConfigurationError(
f"The port selected {self.app__port} is in use, please configure an open port."
)
except OverflowError:
raise ConfigurationError(f"Invalid port: {self.app__port}")
else:
try:
default_server_port = int(
os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
except ValueError:
raise ConfigurationError(
"Invalid port from environment variable CXG_SERVER_PORT: "
+ os.environ.get("CXG_SERVER_PORT"))
try:
self.app__port = find_available_port(self.app__host,
default_server_port)
except OverflowError:
raise ConfigurationError(
f"Invalid port: {default_server_port}")
if self.app__debug:
context["messagefn"](
"in debug mode, setting verbose=True and open_browser=False")
self.app__verbose = True
self.app__open_browser = False
else:
warnings.formatwarning = custom_format_warning
if not self.app__verbose:
sys.tracebacklimit = 0
def handle_app(self):
self.validate_correct_type_of_configuration_attribute(
"app__scripts", list)
self.validate_correct_type_of_configuration_attribute(
"app__inline_scripts", list)
self.validate_correct_type_of_configuration_attribute(
"app__authentication_enable", bool)
# scripts can be string (filename) or dict (attributes). Convert string to dict.
scripts = []
for script in self.app__scripts:
try:
if isinstance(script, str):
scripts.append({"src": script})
elif isinstance(script, dict) and isinstance(
script["src"], str):
scripts.append(script)
else:
raise Exception
except Exception as e:
raise ConfigurationError(
f"Scripts must be string or a dict containing an src key: {e}"
)
self.app__scripts = scripts
def check_config(self):
"""Verify all the attributes in the config have been type checked"""
if not self.is_completed:
raise ConfigurationError("The configuration has not been completed")
self.server_config.check_config()
self.dataset_config.check_config()
self.external_config.check_config()
def import_plugins(plugin_module):
"""
Load optional plugin modules from local_server.common.plugins
If you would like to customize cellxgene, you can add submodules to server.common.plugins before running the app.
This code will import each, loading the code in each. If no plugins are defined, initializing the app continues as
normal.
"""
loaded_modules = []
try:
pkg = importlib.import_module(plugin_module)
for loader, name, is_pkg in pkgutil.walk_packages(pkg.__path__):
full_name = f"{plugin_module}.{name}"
try:
module = importlib.import_module(full_name)
except Exception as e:
raise ConfigurationError(
f"Unexpected error while importing plugin: {plugin_module}.{name}: {str(e)}"
)
loaded_modules.append(module)
except ModuleNotFoundError as e:
# This exception occurs when the plugin_module does not exist (not an error).
logging.debug(f"No plugins found in module: {plugin_module}: {str(e)}")
return loaded_modules
def validate_correct_type_of_configuration_attribute(
self, attrname, vtype):
val = getattr(self, attrname)
if type(vtype) in (list, tuple):
if type(val) not in vtype:
tnames = ",".join([x.__name__ for x in vtype])
raise ConfigurationError(
f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
)
else:
if type(val) != vtype:
raise ConfigurationError(
f"Invalid type for attribute: {attrname}, "
f"expected type {vtype.__name__}, got {type(val).__name__}"
)
self.attr_checked[attrname] = True
def update_from_config_file(self, config_file):
try:
with open(config_file) as yml_file:
config = yaml.safe_load(yml_file)
except yaml.YAMLError as e:
raise ConfigurationError(f"The specified config file contained an error: {e}")
except OSError as e:
raise ConfigurationError(f"Issue retrieving the specified config file: {e}")
if config.get("server"):
self.server_config.update_from_config(config["server"], "server")
if config.get("dataset"):
self.dataset_config.update_from_config(config["dataset"], "dataset")
if config.get("external"):
self.external_config.update_from_config(config["external"], "external")
self.is_complete = False
def update_from_config(self, config, prefix):
mapping = self.create_mapping(config)
for attr, (key, value) in mapping.items():
if not hasattr(self, attr):
raise ConfigurationError(
f"Unknown key from config file: {prefix}__{attr}")
setattr(self, attr, value)
self.attr_checked[attr] = False
def handle_single_dataset(self, context):
self.validate_correct_type_of_configuration_attribute(
"single_dataset__datapath", (str, type(None)))
self.validate_correct_type_of_configuration_attribute(
"single_dataset__title", (str, type(None)))
self.validate_correct_type_of_configuration_attribute(
"single_dataset__about", (str, type(None)))
self.validate_correct_type_of_configuration_attribute(
"single_dataset__obs_names", (str, type(None)))
self.validate_correct_type_of_configuration_attribute(
"single_dataset__var_names", (str, type(None)))
# preload this data set
matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath,
app_config=self.app_config)
try:
matrix_data_loader.pre_load_validation()
except DatasetAccessError as e:
raise ConfigurationError(str(e))
file_size = matrix_data_loader.file_size()
file_basename = basename(self.single_dataset__datapath)
if file_size > BIG_FILE_SIZE_THRESHOLD:
context["messagefn"](
f"Loading data from {file_basename}, this may take a while...")
else:
context["messagefn"](f"Loading data from {file_basename}.")
if self.single_dataset__about:
def url_check(url):
try:
result = urlparse(url)
if all([result.scheme, result.netloc]):
return True
else:
return False
except ValueError:
return False
if not url_check(self.single_dataset__about):
raise ConfigurationError(
"Must provide an absolute URL for --about. (Example format: http://example.com)"
)
def handle_embeddings(self):
self.validate_correct_type_of_configuration_attribute(
"embeddings__names", list)
self.validate_correct_type_of_configuration_attribute(
"embeddings__enable_reembedding", bool)
server_config = self.app_config.server_config
if self.embeddings__enable_reembedding:
if server_config.single_dataset__datapath:
if server_config.adaptor__anndata_adaptor__backed:
raise ConfigurationError(
"enable-reembedding is not supported when run in --backed mode."
)
try:
get_scanpy_module()
except NotImplementedError:
# Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported
raise ConfigurationError(
"Please install scanpy to enable UMAP re-embedding")
def __init__(self, app_config, default_config):
super().__init__(app_config, default_config)
try:
self.environment = default_config["environment"]
self.aws_secrets_manager__region = default_config[
"aws_secrets_manager"]["region"]
self.aws_secrets_manager__secrets = default_config[
"aws_secrets_manager"]["secrets"]
except KeyError as e:
raise ConfigurationError(f"Unexpected config: {str(e)}")
def handle_user_annotations(self, context):
self.validate_correct_type_of_configuration_attribute(
"user_annotations__enable", bool)
self.validate_correct_type_of_configuration_attribute(
"user_annotations__type", str)
self.validate_correct_type_of_configuration_attribute(
"user_annotations__local_file_csv__directory", (type(None), str))
self.validate_correct_type_of_configuration_attribute(
"user_annotations__local_file_csv__file", (type(None), str))
self.validate_correct_type_of_configuration_attribute(
"user_annotations__ontology__enable", bool)
self.validate_correct_type_of_configuration_attribute(
"user_annotations__ontology__obo_location", (type(None), str))
if self.user_annotations__enable:
server_config = self.app_config.server_config
if not self.app__authentication_enable:
raise ConfigurationError(
"user annotations requires authentication to be enabled")
if not server_config.auth.is_valid_authentication_type():
auth_type = server_config.authentication__type
raise ConfigurationError(
f"authentication method {auth_type} is not compatible with user annotations"
)
if self.user_annotations__type == "local_file_csv":
self.handle_local_file_csv_annotations()
else:
raise ConfigurationError(
'The only annotation type support is "local_file_csv"')
if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
try:
self.user_annotations.load_ontology(
self.user_annotations__ontology__obo_location)
except OntologyLoadFailure as e:
raise ConfigurationError(
"Unable to load ontology terms\n" + str(e))
else:
self.check_annotation_config_vars_not_set(context)
def handle_data_locator(self):
self.validate_correct_type_of_configuration_attribute(
"data_locator__s3__region_name", (type(None), bool, str))
if self.data_locator__s3__region_name is True:
path = self.single_dataset__datapath
if path.startswith("s3://"):
region_name = discover_s3_region_name(path)
if region_name is None:
raise ConfigurationError(
f"Unable to discover s3 region name from {path}")
else:
region_name = None
self.data_locator__s3__region_name = region_name
def __init__(self, app_config, default_config):
super().__init__(app_config, default_config)
try:
self.app__verbose = default_config["app"]["verbose"]
self.app__debug = default_config["app"]["debug"]
self.app__host = default_config["app"]["host"]
self.app__port = default_config["app"]["port"]
self.app__open_browser = default_config["app"]["open_browser"]
self.app__force_https = default_config["app"]["force_https"]
self.app__flask_secret_key = default_config["app"][
"flask_secret_key"]
self.authentication__type = default_config["authentication"][
"type"]
self.authentication__insecure_test_environment = default_config[
"authentication"]["insecure_test_environment"]
self.single_dataset__datapath = default_config["single_dataset"][
"datapath"]
self.single_dataset__obs_names = default_config["single_dataset"][
"obs_names"]
self.single_dataset__var_names = default_config["single_dataset"][
"var_names"]
self.single_dataset__about = default_config["single_dataset"][
"about"]
self.single_dataset__title = default_config["single_dataset"][
"title"]
self.data_locator__s3__region_name = default_config[
"data_locator"]["s3"]["region_name"]
self.adaptor__anndata_adaptor__backed = default_config["adaptor"][
"anndata_adaptor"]["backed"]
self.limits__diffexp_cellcount_max = default_config["limits"][
"diffexp_cellcount_max"]
self.limits__column_request_max = default_config["limits"][
"column_request_max"]
except KeyError as e:
raise ConfigurationError(f"Unexpected config: {str(e)}")
self.data_adaptor = None
# The authentication object
self.auth = None
def __init__(self, tag, app_config, default_config):
super().__init__(app_config, default_config)
self.tag = tag
try:
self.app__scripts = default_config["app"]["scripts"]
self.app__inline_scripts = default_config["app"]["inline_scripts"]
self.app__authentication_enable = default_config["app"][
"authentication_enable"]
self.presentation__max_categories = default_config["presentation"][
"max_categories"]
self.presentation__custom_colors = default_config["presentation"][
"custom_colors"]
self.user_annotations__enable = default_config["user_annotations"][
"enable"]
self.user_annotations__type = default_config["user_annotations"][
"type"]
self.user_annotations__local_file_csv__directory = default_config[
"user_annotations"]["local_file_csv"]["directory"]
self.user_annotations__local_file_csv__file = default_config[
"user_annotations"]["local_file_csv"]["file"]
self.user_annotations__ontology__enable = default_config[
"user_annotations"]["ontology"]["enable"]
self.user_annotations__ontology__obo_location = default_config[
"user_annotations"]["ontology"]["obo_location"]
self.embeddings__names = default_config["embeddings"]["names"]
self.embeddings__enable_reembedding = default_config["embeddings"][
"enable_reembedding"]
self.diffexp__enable = default_config["diffexp"]["enable"]
self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
self.diffexp__top_n = default_config["diffexp"]["top_n"]
except KeyError as e:
raise ConfigurationError(f"Unexpected config: {str(e)}")
# The annotation object is created during complete_config and stored here.
self.user_annotations = None
def check_config(self):
mapping = self.create_mapping(self.default_config)
for key in mapping.keys():
if not self.attr_checked[key]:
raise ConfigurationError(
f"The attr '{key}' has not been checked")
def handle_aws_secrets_manager(self, context):
"""For each aws secret defined, get the key/values, and set the specified config parameter"""
self.validate_correct_type_of_configuration_attribute(
"aws_secrets_manager__region", (type(None), str))
self.validate_correct_type_of_configuration_attribute(
"aws_secrets_manager__secrets", list)
if not self.aws_secrets_manager__secrets:
return
self.validate_correct_type_of_configuration_attribute(
"aws_secrets_manager__region", str)
for secret in self.aws_secrets_manager__secrets:
secret_name = secret.get("name")
if secret_name is None:
raise ConfigurationError(
"aws_secrets_manager: 'name' is missing")
if not isinstance(secret_name, str):
raise ConfigurationError(
"aws_secrets_manager: 'name' must be a string")
try:
secret_dict = get_secret_key(self.aws_secrets_manager__region,
secret_name)
except SecretKeyRetrievalError as e:
raise ConfigurationError(
f"Unable to retrieve secret {secret_name}: {str(e)}")
values = secret.get("values")
if values is None:
raise ConfigurationError(
"aws_secrets_manager: 'values' is missing")
if not isinstance(values, list):
raise ConfigurationError(
"aws_secrets_manager: 'values' must be a list")
for value in values:
key = value.get("key")
if key is None:
raise ConfigurationError(
f"missing 'key' in secret values: {secret_name}")
path = value.get("path")
if path is None:
raise ConfigurationError(
f"missing 'path' in secret values: {secret_name}")
required = value.get("required", False)
if type(required) != bool:
raise ConfigurationError(
f"wrong type for 'required' in secret values: {secret_name}"
)
secret_value = secret_dict.get(key)
if secret_value is None:
if required:
raise ConfigurationError(
f"required secret '{secret_name}:{key}' not set")
else:
secret_value = convert_string_to_value(secret_value)
self.app_config.update_single_config_from_path_and_value(
path, secret_value)