def setUp(self): args = { "layout": ["umap"], "max_category_items": 100, "obs_names": None, "var_names": None, "diffexp_lfc_cutoff": 0.01, } self.data = ScanpyEngine("example-dataset/pbmc3k.h5ad", args)
def setUp(self): args = { "layout": ["umap"], "max_category_items": 100, "obs_names": None, "var_names": None, "diffexp_lfc_cutoff": 0.01, "layout_file": None } self.data = ScanpyEngine(DataLocator(self.data_locator), args)
def setUp(self): self.args = { "layout": ["umap"], "max_category_items": 100, "obs_names": None, "var_names": None, "diffexp_lfc_cutoff": 0.01, } with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UserWarning) self.data = ScanpyEngine(DataLocator("server/test/test_datasets/nan.h5ad"), self.args) self.data._create_schema()
def setUp(self): self.tmpDir = tempfile.mkdtemp() self.label_file = path.join(self.tmpDir, "labels.csv") args = { "layout": ["umap"], "max_category_items": 100, "obs_names": None, "var_names": None, "diffexp_lfc_cutoff": 0.01, "label_file": self.label_file } self.data = ScanpyEngine(DataLocator("example-dataset/pbmc3k.h5ad"), args)
def setUp(self): args = { "layout": "umap", "diffexp": "ttest", "max_category_items": 100, "obs_names": None, "var_names": None, "diffexp_lfc_cutoff": 0.01, "nan_to_num": True, } self.data = ScanpyEngine("example-dataset/pbmc3k.h5ad", args) self.data._create_schema()
def run(self): super(Worker, self).run() if not self.data_file: self.emit("finished") return from server.app.app import Server from server.app.scanpy_engine.scanpy_engine import ScanpyEngine # create server try: server = Server() server.create_app() except Exception as e: self.emit("server_error", str(e)) self.emit("finished") return # load data try: args = { "max_category_items": 100, "diffexp_lfc_cutoff": 0.01, "obs_names": None, "var_names": None, } args.update(self.engine_options) data = ScanpyEngine(self.data_file, args) server.attach_data(data, self.title) self.emit("ready") except Exception as e: self.emit("engine_error", str(e)) self.emit("finished") return # launch server try: server.app.run(host=self.host, debug=False, port=self.port, threaded=True) except Exception as e: self.emit("server_error", str(e)) finally: self.emit("finished")
class EngineTest(unittest.TestCase): def setUp(self): args = { "layout": ["umap"], "max_category_items": 100, "obs_names": None, "var_names": None, "diffexp_lfc_cutoff": 0.01, "layout_file": None, "backed": self.backed } self.data = ScanpyEngine(DataLocator(self.data_locator), args) def test_init(self): self.assertEqual(self.data.cell_count, 2638) self.assertEqual(self.data.gene_count, 1838) epsilon = 0.000_005 self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon) def test_mandatory_annotations(self): obs_index_col_name = self.data.get_schema( )["annotations"]["obs"]["index"] self.assertIn(obs_index_col_name, self.data.data.obs) self.assertEqual(list(self.data.data.obs.index), list(range(2638))) var_index_col_name = self.data.get_schema( )["annotations"]["var"]["index"] self.assertIn(var_index_col_name, self.data.data.var) self.assertEqual(list(self.data.data.var.index), list(range(1838))) @pytest.mark.filterwarnings("ignore:Scanpy data matrix") def test_data_type(self): # don't run the test on the more exotic data types, as they don't # support the astype() interface (used by this test, but not underlying app) if isinstance(self.data.data.X, np.ndarray): self.data.data.X = self.data.data.X.astype("float64") with self.assertWarns(UserWarning): self.data._validate_data_types() def test_filter_idx(self): filter_ = {"filter": {"var": {"index": [1, 99, [200, 300]]}}} fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") data = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(data["n_rows"], 2638) self.assertEqual(data["n_cols"], 102) def test_filter_complex(self): filter_ = { "filter": { "var": { "annotation_value": [{ "name": "n_cells", "min": 10 }], "index": [1, 99, [200, 300]] } } } fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") data = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(data["n_rows"], 2638) self.assertEqual(data["n_cols"], 91) def test_obs_and_var_names(self): self.assertEqual( np.sum(self.data.data.var[self.data.get_schema()["annotations"] ["var"]["index"]].isna()), 0) self.assertEqual( np.sum(self.data.data.obs[self.data.get_schema()["annotations"] ["obs"]["index"]].isna()), 0) def test_get_schema(self): with open(path.join(path.dirname(__file__), "schema.json")) as fh: schema = json.load(fh) self.assertEqual(self.data.get_schema(), schema) def test_schema_produces_error(self): self.data.data.obs["time"] = pd.Series( list([time.time() for i in range(self.data.cell_count)]), dtype="datetime64[ns]", ) with pytest.raises(TypeError): self.data._create_schema() def test_config(self): self.assertEqual( self.data.features["layout"]["obs"], { "available": True, "interactiveLimit": 50000 }, ) def test_layout(self): fbs = self.data.layout_to_fbs_matrix() layout = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(layout["n_cols"], 2) self.assertEqual(layout["n_rows"], 2638) X = layout["columns"][0] self.assertTrue((X >= 0).all() and (X <= 1).all()) Y = layout["columns"][1] self.assertTrue((Y >= 0).all() and (Y <= 1).all()) def test_annotations(self): fbs = self.data.annotation_to_fbs_matrix("obs") annotations = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(annotations["n_rows"], 2638) self.assertEqual(annotations["n_cols"], 5) obs_index_col_name = self.data.get_schema( )["annotations"]["obs"]["index"] self.assertEqual( annotations["col_idx"], [ obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain" ], ) fbs = self.data.annotation_to_fbs_matrix("var") annotations = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(annotations['n_rows'], 1838) self.assertEqual(annotations['n_cols'], 2) var_index_col_name = self.data.get_schema( )["annotations"]["var"]["index"] self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"]) def test_annotation_fields(self): fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"]) annotations = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(annotations["n_rows"], 2638) self.assertEqual(annotations['n_cols'], 2) var_index_col_name = self.data.get_schema( )["annotations"]["var"]["index"] fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name]) annotations = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(annotations['n_rows'], 1838) self.assertEqual(annotations['n_cols'], 1) def test_annotation_put(self): with self.assertRaises(DisabledFeatureError): self.data.annotation_put_fbs(None, "obs") def test_diffexp_topN(self): f1 = {"filter": {"obs": {"index": [[0, 500]]}}} f2 = {"filter": {"obs": {"index": [[500, 1000]]}}} result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"])) self.assertEqual(len(result), 10) result = json.loads( self.data.diffexp_topN(f1["filter"], f2["filter"], 20)) self.assertEqual(len(result), 20) def test_data_frame(self): f1 = {"var": {"index": [[0, 10]]}} fbs = self.data.data_frame_to_fbs_matrix(f1, "var") data = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(data["n_rows"], 2638) self.assertEqual(data["n_cols"], 10) with self.assertRaises(ValueError): self.data.data_frame_to_fbs_matrix(None, "obs") def test_filtered_data_frame(self): filter_ = { "filter": { "var": { "annotation_value": [{ "name": "n_cells", "min": 100 }] } } } fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") data = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(data["n_rows"], 2638) self.assertEqual(data["n_cols"], 1040) filter_ = { "filter": { "obs": { "annotation_value": [{ "name": "n_counts", "min": 3000 }] } } } with self.assertRaises(FilterError): self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") def test_data_named_gene(self): var_index_col_name = self.data.get_schema( )["annotations"]["var"]["index"] filter_ = { "filter": { "var": { "annotation_value": [{ "name": var_index_col_name, "values": ["RER1"] }] } } } fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") data = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(data["n_rows"], 2638) self.assertEqual(data["n_cols"], 1) self.assertEqual(data["col_idx"], [4]) filter_ = { "filter": { "var": { "annotation_value": [{ "name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"] }] } } } fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") data = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(data["n_rows"], 2638) self.assertEqual(data["n_cols"], 3) self.assertTrue((data["col_idx"] == [15, 1818, 1837]).all())
def launch(data, verbose, debug, open_browser, port, host, embedding, obs_names, var_names, max_category_items, diffexp_lfc_cutoff, title, scripts, about, experimental_label_file, backed, disable_diffexp): """Launch the cellxgene data viewer. This web app lets you explore single-cell expression data. Data must be in a format that cellxgene expects, read the "getting started" guide. Examples: > cellxgene launch example_dataset/pbmc3k.h5ad --title pbmc3k > cellxgene launch <your data file> --title <your title> > cellxgene launch <url>""" e_args = parse_engine_args(embedding, obs_names, var_names, max_category_items, diffexp_lfc_cutoff, experimental_label_file, backed, disable_diffexp) try: data_locator = DataLocator(data) except RuntimeError as re: raise click.ClickException( f"Unable to access data at {data}. {str(re)}") # Startup message click.echo("[cellxgene] Starting the CLI...") # Argument checking if data_locator.islocal(): # if data locator is local, apply file system conventions and other "cheap" # validation checks. If a URI, defer until we actually fetch the data and # try to read it. Many of these tests don't make sense for URIs (eg, extension- # based typing). if not data_locator.exists(): raise click.FileError(data, hint="file does not exist") if not data_locator.isfile(): raise click.FileError(data, hint="data is not a file") name, extension = splitext(data) if extension != ".h5ad": raise click.FileError(basename(data), hint="file type must be .h5ad") if debug: verbose = True open_browser = False else: warnings.formatwarning = custom_format_warning if not verbose: sys.tracebacklimit = 0 if scripts: click.echo(r""" / / /\ \ \__ _ _ __ _ __ (_)_ __ __ _ \ \/ \/ / _` | '__| '_ \| | '_ \ / _` | \ /\ / (_| | | | | | | | | | | (_| | \/ \/ \__,_|_| |_| |_|_|_| |_|\__, | |___/ The --scripts flag is intended for developers to include google analytics etc. You could be opening yourself to a security risk by including the --scripts flag. Make sure you trust the scripts that you are including. """) scripts_pretty = ", ".join(scripts) click.confirm( f"Are you sure you want to inject these scripts: {scripts_pretty}?", abort=True) if not title: file_parts = splitext(basename(data)) title = file_parts[0] if port: if debug: raise click.ClickException( "--port and --debug may not be used together (try --verbose for error logging)." ) if not is_port_available(host, int(port)): raise click.ClickException( f"The port selected {port} is in use, please specify an open port using the --port flag." ) else: port = find_available_port(host) if experimental_label_file: lf_name, lf_ext = splitext(experimental_label_file) if lf_ext and lf_ext != ".csv": raise click.FileError(basename(experimental_label_file), hint="label file type must be .csv") if about: def url_check(url): try: result = urlparse(url) if all([result.scheme, result.netloc]): return True else: return False except ValueError: return False if not url_check(about): raise click.ClickException( "Must provide an absolute URL for --about. (Example format: http://example.com)" ) # Setup app cellxgene_url = f"http://{host}:{port}" # Import Flask app server = Server() server.create_app() server.app.config.update(SCRIPTS=scripts) if not verbose: log = logging.getLogger("werkzeug") log.setLevel(logging.ERROR) file_size = data_locator.size() if data_locator.islocal() else 0 # if a big file, let the user know it may take a while to load. if file_size > BIG_FILE_SIZE_THRESHOLD: click.echo( f"[cellxgene] Loading data from {basename(data)}, this may take a while..." ) else: click.echo(f"[cellxgene] Loading data from {basename(data)}.") from server.app.scanpy_engine.scanpy_engine import ScanpyEngine try: server.attach_data(ScanpyEngine(data_locator, e_args), title=title, about=about) except ScanpyFileError as e: raise click.ClickException(f"{e}") if not disable_diffexp and server.app.data.config['diffexp_may_be_slow']: click.echo(f"[cellxgene] CAUTION: due to the size of your dataset, " f"running differential expression may take longer or fail.") if open_browser: click.echo( f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now." ) webbrowser.open(cellxgene_url) else: click.echo( f"[cellxgene] Launching! Please go to {cellxgene_url} in your browser." ) click.echo("[cellxgene] Type CTRL-C at any time to exit.") if not verbose: f = open(devnull, "w") sys.stdout = f try: server.app.run(host=host, debug=debug, port=port, threaded=False if debug else True, use_debugger=False) except OSError as e: if e.errno == errno.EADDRINUSE: raise click.ClickException( "Port is in use, please specify an open port using the --port flag." ) from e raise
class UtilTest(unittest.TestCase): def setUp(self): args = { "layout": "umap", "diffexp": "ttest", "max_category_items": 100, "obs_names": None, "var_names": None, "diffexp_lfc_cutoff": 0.01, } self.data = ScanpyEngine("example-dataset/pbmc3k.h5ad", args) self.data._create_schema() def test_init(self): self.assertEqual(self.data.cell_count, 2638) self.assertEqual(self.data.gene_count, 1838) epsilon = 0.000_005 self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon) def test_mandatory_annotations(self): self.assertIn("name", self.data.data.obs) self.assertEqual(list(self.data.data.obs.index), list(range(2638))) self.assertIn("name", self.data.data.var) self.assertEqual(list(self.data.data.var.index), list(range(1838))) @pytest.mark.filterwarnings("ignore:Scanpy data matrix") def test_data_type(self): self.data.data.X = self.data.data.X.astype("float64") with self.assertWarns(UserWarning): self.data._validate_data_types() def test_filter_idx(self): filter_ = { "filter": { "var": { "index": [1, 99, [200, 300]] }, "obs": { "index": [1, 99, [1000, 2000]] }, } } data = self.data.filter_dataframe(filter_["filter"]) self.assertEqual(data.shape, (1002, 102)) def test_filter_annotation(self): filter_ = { "filter": { "obs": { "annotation_value": [{ "name": "louvain", "values": ["NK cells", "CD8 T cells"] }] } } } data = self.data.filter_dataframe(filter_["filter"]) self.assertEqual(data.shape, (470, 1838)) filter_ = { "filter": { "obs": { "annotation_value": [{ "name": "n_counts", "min": 3000 }] } } } data = self.data.filter_dataframe(filter_["filter"]) self.assertEqual(data.shape, (497, 1838)) def test_filter_annotation_no_uns(self): filter_ = { "filter": { "var": { "annotation_value": [{ "name": "name", "values": ["RER1"] }] } } } data = self.data.filter_dataframe(filter_["filter"]) self.assertEqual(data.shape[1], 1) def test_filter_complex(self): filter_ = { "filter": { "var": { "index": [1, 99, [200, 300]] }, "obs": { "annotation_value": [ { "name": "louvain", "values": ["NK cells", "CD8 T cells"] }, { "name": "n_counts", "min": 3000 }, ], "index": [1, 99, [1000, 2000]], }, } } data = self.data.filter_dataframe(filter_["filter"]) self.assertEqual(data.shape, (15, 102)) def test_obs_and_var_names(self): self.assertEqual(np.sum(self.data.data.var["name"].isna()), 0) self.assertEqual(np.sum(self.data.data.obs["name"].isna()), 0) def test_schema(self): with open(path.join(path.dirname(__file__), "schema.json")) as fh: schema = json.load(fh) self.assertEqual(self.data.schema, schema) def test_schema_produces_error(self): self.data.data.obs["time"] = Series( list([time.time() for i in range(self.data.cell_count)]), dtype="datetime64[ns]", ) with pytest.raises(TypeError): self.data._create_schema() def test_config(self): self.assertEqual( self.data.features["layout"]["obs"], { "available": True, "interactiveLimit": 50000 }, ) def test_layout(self): layout = json.loads(self.data.layout(None)) self.assertEqual(layout["layout"]["ndims"], 2) self.assertEqual(len(layout["layout"]["coordinates"]), 2638) self.assertEqual(layout["layout"]["coordinates"][0][0], 0) for idx, val in enumerate(layout["layout"]["coordinates"]): self.assertLessEqual(val[1], 1) self.assertLessEqual(val[2], 1) def test_annotations(self): annotations = json.loads(self.data.annotation(None, "obs")) self.assertEqual( annotations["names"], ["name", "n_genes", "percent_mito", "n_counts", "louvain"], ) self.assertEqual(len(annotations["data"]), 2638) annotations = json.loads(self.data.annotation(None, "var")) self.assertEqual(annotations["names"], ["name", "n_cells"]) self.assertEqual(len(annotations["data"]), 1838) def test_annotation_fields(self): annotations = json.loads( self.data.annotation(None, "obs", ["n_genes", "n_counts"])) self.assertEqual(annotations["names"], ["n_genes", "n_counts"]) self.assertEqual(len(annotations["data"]), 2638) annotations = json.loads(self.data.annotation(None, "var", ["name"])) self.assertEqual(annotations["names"], ["name"]) self.assertEqual(len(annotations["data"]), 1838) def test_filtered_annotation(self): filter_ = { "filter": { "obs": { "annotation_value": [{ "name": "n_counts", "min": 3000 }] }, "var": { "annotation_value": [{ "name": "name", "values": ["ATAD3C", "RER1"] }] }, } } annotations = json.loads(self.data.annotation(filter_["filter"], "obs")) self.assertEqual( annotations["names"], ["name", "n_genes", "percent_mito", "n_counts", "louvain"], ) self.assertEqual(len(annotations["data"]), 497) annotations = json.loads(self.data.annotation(filter_["filter"], "var")) self.assertEqual(annotations["names"], ["name", "n_cells"]) self.assertEqual(len(annotations["data"]), 2) def test_filtered_layout(self): filter_ = { "filter": { "obs": { "annotation_value": [{ "name": "n_counts", "min": 3000 }] } } } layout = json.loads(self.data.layout(filter_["filter"])) self.assertEqual(len(layout["layout"]["coordinates"]), 497) def test_diffexp_topN(self): f1 = {"filter": {"obs": {"index": [[0, 500]]}}} f2 = {"filter": {"obs": {"index": [[500, 1000]]}}} result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"])) self.assertEqual(len(result), 10) result = json.loads( self.data.diffexp_topN(f1["filter"], f2["filter"], 20)) self.assertEqual(len(result), 20) def test_data_frame(self): data_frame_obs = json.loads(self.data.data_frame(None, "obs")) self.assertEqual(len(data_frame_obs["var"]), 1838) self.assertEqual(len(data_frame_obs["obs"]), 2638) data_frame_var = json.loads(self.data.data_frame(None, "var")) self.assertEqual(len(data_frame_var["var"]), 1838) self.assertEqual(len(data_frame_var["obs"]), 2638) def test_filtered_data_frame(self): filter_ = { "filter": { "obs": { "annotation_value": [{ "name": "n_counts", "min": 3000 }] } } } data_frame_obs = json.loads( self.data.data_frame(filter_["filter"], "obs")) self.assertEqual(len(data_frame_obs["var"]), 1838) self.assertEqual(len(data_frame_obs["obs"]), 497) self.assertIsInstance(data_frame_obs["obs"][0], (list, tuple)) self.assertEqual(type(data_frame_obs["var"][0]), int) data_frame_var = json.loads( self.data.data_frame(filter_["filter"], "var")) self.assertEqual(len(data_frame_var["var"]), 1838) self.assertEqual(len(data_frame_var["obs"]), 497) self.assertIsInstance(data_frame_var["var"][0], (list, tuple)) self.assertEqual(type(data_frame_var["obs"][0]), int) def test_data_single_gene(self): for axis in ["obs", "var"]: filter_ = { "filter": { "var": { "annotation_value": [{ "name": "name", "values": ["RER1"] }] } } } data_frame_var = json.loads( self.data.data_frame(filter_["filter"], axis)) if axis == "obs": self.assertEqual(type(data_frame_var["var"][0]), int) self.assertIsInstance(data_frame_var["obs"][0], (list, tuple)) elif axis == "var": self.assertEqual(type(data_frame_var["obs"][0]), int) self.assertIsInstance(data_frame_var["var"][0], (list, tuple)) if __name__ == "__main__": unittest.main()
def test_url_http(self): url = "http://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/example-dataset/pbmc3k.h5ad" locator = DataLocator(url) data = ScanpyEngine(locator, self.args) self.stdAsserts(data)
class DataLoadEngineTest(unittest.TestCase): """ Test file loading, including deferred loading/update. """ def setUp(self): self.data_file = DataLocator("example-dataset/pbmc3k.h5ad") self.data = ScanpyEngine() def test_init(self): self.assertIsNone(self.data.data) def test_delayed_load_args(self): args = { "layout": ["tsne"], "max_category_items": 1000, "obs_names": "foo", "var_names": "bar", "diffexp_lfc_cutoff": 0.1, "annotations": False, "annotations_file": None, "annotations_output_dir": None, "backed": False, "diffexp_may_be_slow": False, "disable_diffexp": False } self.data.update(args=args) self.assertEqual(args, self.data.config) def test_requires_data(self): with self.assertRaises(DriverError): self.data._create_schema() def test_delayed_load_data(self): self.data.update(data_locator=self.data_file) self.data._create_schema() self.assertEqual(self.data.cell_count, 2638) self.assertEqual(self.data.gene_count, 1838) epsilon = 0.000_005 self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon) def test_diffexp_topN(self): self.data.update(data_locator=self.data_file) f1 = {"filter": {"obs": {"index": [[0, 500]]}}} f2 = {"filter": {"obs": {"index": [[500, 1000]]}}} result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"])) self.assertEqual(len(result), 10) result = json.loads( self.data.diffexp_topN(f1["filter"], f2["filter"], 20)) self.assertEqual(len(result), 20)
def test_posix_file(self): locator = DataLocator("example-dataset/pbmc3k.h5ad") data = ScanpyEngine(locator, self.args) self.stdAsserts(data)
def setUp(self): self.data_file = DataLocator("example-dataset/pbmc3k.h5ad") self.data = ScanpyEngine()
def test_load(self): with self.assertWarns(UserWarning): ScanpyEngine("server/test/test_datasets/nan.h5ad", self.args)
class DataLoadEngineTest(unittest.TestCase): def setUp(self): self.data_file = "example-dataset/pbmc3k.h5ad" self.data = ScanpyEngine() def test_init(self): self.assertIsNone(self.data.data) def test_delayed_load_args(self): args = { "layout": ["tsne"], "max_category_items": 1000, "obs_names": "foo", "var_names": "bar", "diffexp_lfc_cutoff": 0.1, } self.data.update(args=args) self.assertEqual(args, self.data.config) def test_requires_data(self): with self.assertRaises(DriverError): self.data._create_schema() def test_delayed_load_data(self): self.data.update(data=self.data_file) self.data._create_schema() self.assertEqual(self.data.cell_count, 2638) self.assertEqual(self.data.gene_count, 1838) epsilon = 0.000_005 self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon) def test_diffexp_topN(self): self.data.update(data=self.data_file) f1 = {"filter": {"obs": {"index": [[0, 500]]}}} f2 = {"filter": {"obs": {"index": [[500, 1000]]}}} result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"])) self.assertEqual(len(result), 10) result = json.loads( self.data.diffexp_topN(f1["filter"], f2["filter"], 20)) self.assertEqual(len(result), 20) if __name__ == "__main__": unittest.main()
class UtilTest(unittest.TestCase): def setUp(self): self.data = ScanpyEngine("example-dataset/", schema="data_schema.json") def test_init(self): self.assertEqual(self.data.cell_count, 2638) self.assertEqual(self.data.gene_count, 1838) epsilon = 0.000005 self.assertTrue(self.data.data.X[0, 0] - -0.17146951 < epsilon) def test_schema(self): self.assertEqual( self.data.schema, { 'CellName': { 'type': 'string', 'variabletype': 'categorical', 'displayname': 'Name', 'include': True }, 'n_genes': { 'type': 'int', 'variabletype': 'continuous', 'displayname': 'Num Genes', 'include': True }, 'percent_mito': { 'type': 'float', 'variabletype': 'continuous', 'displayname': 'Mitochondrial Percentage', 'include': True }, 'n_counts': { 'type': 'float', 'variabletype': 'continuous', 'displayname': 'Num Counts', 'include': True }, 'louvain': { 'type': 'string', 'variabletype': 'categorical', 'displayname': 'Louvain Cluster', 'include': True } }) def test_cells(self): cells = self.data.cells() self.assertIn("AAACATACAACCAC-1", cells) self.assertEqual(len(cells), 2638) def test_genes(self): genes = self.data.genes() self.assertIn("SEPT4", genes) self.assertEqual(len(genes), 1838) def test_filter_categorical(self): filter = { "louvain": { "variable_type": "categorical", "value_type": "string", "query": ["B cells"] } } filtered_data = self.data.filter_cells(filter) self.assertEqual(filtered_data.shape, (342, 1838)) louvain_vals = filtered_data.obs['louvain'].tolist() self.assertIn("B cells", louvain_vals) self.assertNotIn("NK cells", louvain_vals) def test_filter_continuous(self): # print(self.data.data.obs["n_genes"].tolist()) filter = { "n_genes": { "variable_type": "continuous", "value_type": "int", "query": { "min": 300, "max": 400 } } } filtered_data = self.data.filter_cells(filter) self.assertEqual(filtered_data.shape, (71, 1838)) n_genes_vals = filtered_data.obs['n_genes'].tolist() for val in n_genes_vals: self.assertTrue(300 <= val <= 400) def test_metadata(self): metadata = self.data.metadata(df=self.data.data) self.assertEqual(len(metadata), 2638) self.assertIn('louvain', metadata[0]) @unittest.skip( "Umap not producing the same graph on different systems, even with the same seed. Skipping for now" ) def test_create_graph(self): graph = self.data.create_graph(df=self.data.data) self.assertEqual(graph[0][1], 0.5545382653143183) self.assertEqual(graph[0][2], 0.6021833809031731) def test_diffexp(self): diffexp = self.data.diffexp(["AAACATACAACCAC-1", "AACCGATGGTCATG-1"], ["CCGATAGACCTAAG-1", "GGTGGAGAAGTAGA-1"], 0.5, 7) self.assertEqual(diffexp["celllist1"]["topgenes"], [ 'EBNA1BP2', 'DIAPH1', 'SLC25A11', 'SNRNP27', 'COMMD8', 'COTL1', 'GTF3A' ]) def test_expression(self): expression = self.data.expression(cells=["AAACATACAACCAC-1"]) data_exp = self.data.data[["AAACATACAACCAC-1"], :].X for idx in range(len(expression["cells"][0]["e"])): self.assertEqual(expression["cells"][0]["e"][idx], data_exp[idx])
class WritableAnnotationTest(unittest.TestCase): def setUp(self): self.tmpDir = tempfile.mkdtemp() self.label_file = path.join(self.tmpDir, "labels.csv") args = { "layout": ["umap"], "max_category_items": 100, "obs_names": None, "var_names": None, "diffexp_lfc_cutoff": 0.01, "label_file": self.label_file } self.data = ScanpyEngine(DataLocator("example-dataset/pbmc3k.h5ad"), args) def tearDown(self): shutil.rmtree(self.tmpDir) def make_fbs(self, data): df = pd.DataFrame(data) return encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns) def test_error_checks(self): # verify that the expected errors are generated n_rows = self.data.data.obs.shape[0] fbs_bad = self.make_fbs({ 'louvain': pd.Series(['undefined' for l in range(0, n_rows)], dtype='category') }) # ensure attempt to change VAR annotation with self.assertRaises(ValueError): self.data.annotation_put_fbs("var", fbs_bad) # ensure we catch attempt to overwrite non-writable data with self.assertRaises(KeyError): self.data.annotation_put_fbs("obs", fbs_bad) def test_write_to_file(self): # verify the file is written as expected n_rows = self.data.data.obs.shape[0] fbs = self.make_fbs({ 'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'), 'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category') }) res = self.data.annotation_put_fbs("obs", fbs) self.assertEqual(res, json.dumps({"status": "OK"})) self.assertTrue(path.exists(self.label_file)) df = pd.read_csv(self.label_file, index_col=0) self.assertEqual(df.shape, (n_rows, 2)) self.assertEqual(set(df.columns), set(['cat_A', 'cat_B'])) self.assertTrue(self.data.original_obs_index.equals(df.index)) self.assertTrue( np.all(df['cat_A'] == ['label_A' for l in range(0, n_rows)])) self.assertTrue( np.all(df['cat_B'] == ['label_B' for l in range(0, n_rows)])) # verify complete overwrite on second attempt, AND rotation occurs fbs = self.make_fbs({ 'cat_A': pd.Series(['label_A1' for l in range(0, n_rows)], dtype='category'), 'cat_C': pd.Series(['label_C' for l in range(0, n_rows)], dtype='category') }) res = self.data.annotation_put_fbs("obs", fbs) self.assertEqual(res, json.dumps({"status": "OK"})) self.assertTrue(path.exists(self.label_file)) df = pd.read_csv(self.label_file, index_col=0) self.assertEqual(set(df.columns), set(['cat_A', 'cat_C'])) self.assertTrue( np.all(df['cat_A'] == ['label_A1' for l in range(0, n_rows)])) self.assertTrue( np.all(df['cat_C'] == ['label_C' for l in range(0, n_rows)])) # rotation name, ext = path.splitext(self.label_file) self.assertTrue(path.exists(f"{name}-1{ext}")) def test_file_rotation_to_max_9(self): # verify we stop rotation at 9 n_rows = self.data.data.obs.shape[0] fbs = self.make_fbs({ 'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'), 'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category') }) for i in range(0, 11): res = self.data.annotation_put_fbs("obs", fbs) self.assertEqual(res, json.dumps({"status": "OK"})) name, ext = path.splitext(self.label_file) expected_files = [self.label_file ] + [f"{name}-{i}{ext}" for i in range(1, 10)] found_files = [path.join(self.tmpDir, p) for p in listdir(self.tmpDir)] self.assertEqual(set(expected_files), set(found_files)) def test_put_get_roundtrip(self): # verify that OBS PUTs (annotation_put_fbs) are accessible via # GET (annotation_to_fbs_matrix) n_rows = self.data.data.obs.shape[0] fbs = self.make_fbs({ 'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'), 'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category') }) # put res = self.data.annotation_put_fbs("obs", fbs) self.assertEqual(res, json.dumps({"status": "OK"})) # get fbsAll = self.data.annotation_to_fbs_matrix("obs") schema = self.data.get_schema() annotations = decode_fbs.decode_matrix_FBS(fbsAll) obs_index_col_name = schema["annotations"]["obs"]["index"] self.assertEqual(annotations["n_rows"], n_rows) self.assertEqual(annotations["n_cols"], 7) self.assertIsNone(annotations["row_idx"]) self.assertEqual(annotations["col_idx"], [ obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B" ]) col_idx = annotations["col_idx"] self.assertEqual(annotations["columns"][col_idx.index('cat_A')], ['label_A' for l in range(0, n_rows)]) self.assertEqual(annotations["columns"][col_idx.index('cat_B')], ['label_B' for l in range(0, n_rows)]) # verify the schema was updated all_col_schema = { c["name"]: c for c in schema["annotations"]["obs"]["columns"] } self.assertEqual( all_col_schema["cat_A"], { "name": "cat_A", "type": "categorical", "categories": ["label_A"], "writable": True }) self.assertEqual( all_col_schema["cat_B"], { "name": "cat_B", "type": "categorical", "categories": ["label_B"], "writable": True })
def launch(data, verbose, debug, open_browser, port, host, layout, obs_names, var_names, max_category_items, diffexp_lfc_cutoff, title, scripts): """Launch the cellxgene data viewer. This web app lets you explore single-cell expression data. Data must be in a format that cellxgene expects, read the "getting started" guide. Examples: > cellxgene launch example_dataset/pbmc3k.h5ad --title pbmc3k > cellxgene launch <your data file> --title <your title>""" e_args = parse_engine_args(layout, obs_names, var_names, max_category_items, diffexp_lfc_cutoff) # Startup message click.echo("[cellxgene] Starting the CLI...") # Argument checking name, extension = splitext(data) if extension != ".h5ad": raise click.FileError(basename(data), hint="file type must be .h5ad") if debug: verbose = True open_browser = False else: warnings.formatwarning = custom_format_warning if not verbose: sys.tracebacklimit = 0 if scripts: click.echo(r""" / / /\ \ \__ _ _ __ _ __ (_)_ __ __ _ \ \/ \/ / _` | '__| '_ \| | '_ \ / _` | \ /\ / (_| | | | | | | | | | | (_| | \/ \/ \__,_|_| |_| |_|_|_| |_|\__, | |___/ The --scripts flag is intended for developers to include google analytics etc. You could be opening yourself to a security risk by including the --scripts flag. Make sure you trust the scripts that you are including. """) scripts_pretty = ", ".join(scripts) click.confirm( f"Are you sure you want to inject these scripts: {scripts_pretty}?", abort=True) if not title: file_parts = splitext(basename(data)) title = file_parts[0] if port: if debug: raise click.ClickException( "--port and --debug may not be used together (try --verbose for error logging)." ) if not is_port_available(host, int(port)): raise click.ClickException( f"The port selected {port} is in use, please specify an open port using the --port flag." ) else: port = find_available_port(host) # Setup app cellxgene_url = f"http://{host}:{port}" # Import Flask app server = Server() server.create_app() server.app.config.update(SCRIPTS=scripts) if not verbose: log = logging.getLogger("werkzeug") log.setLevel(logging.ERROR) file_size = getsize(data) # if a big file, let the user know it may take a while to load. if file_size > BIG_FILE_SIZE_THRESHOLD: click.echo( f"[cellxgene] Loading data from {basename(data)}, this may take awhile..." ) else: click.echo(f"[cellxgene] Loading data from {basename(data)}.") # Fix for anaconda python. matplotlib typically expects python to be installed as a framework TKAgg is usually # available and fixes this issue. See https://matplotlib.org/faq/virtualenv_faq.html import matplotlib as mpl mpl.use("TkAgg") from server.app.scanpy_engine.scanpy_engine import ScanpyEngine try: server.attach_data(ScanpyEngine(data, e_args), title=title) except ScanpyFileError as e: raise click.ClickException(f"{e}") if open_browser: click.echo( f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now." ) webbrowser.open(cellxgene_url) else: click.echo( f"[cellxgene] Launching! Please go to {cellxgene_url} in your browser." ) click.echo("[cellxgene] Type CTRL-C at any time to exit.") if not verbose: f = open(devnull, "w") sys.stdout = f try: server.app.run(host=host, debug=debug, port=port, threaded=True, use_debugger=False) except OSError as e: if e.errno == errno.EADDRINUSE: raise click.ClickException( "Port is in use, please specify an open port using the --port flag." ) from e raise
def launch( data, layout, diffexp, title, verbose, debug, obs_names, var_names, open_browser, port, host, max_category_items, diffexp_lfc_cutoff, ): """Launch the cellxgene data viewer. This web app lets you explore single-cell expression data. Data must be in a format that cellxgene expects, read the "getting started" guide. Examples: > cellxgene launch example_dataset/pbmc3k.h5ad --title pbmc3k > cellxgene launch <your data file> --title <your title>""" # Startup message click.echo("[cellxgene] Starting the CLI...") # Argument checking name, extension = splitext(data) if extension != ".h5ad": raise click.FileError(basename(data), hint="file type must be .h5ad") if debug: verbose = True open_browser = False else: warnings.formatwarning = custom_format_warning if not verbose: sys.tracebacklimit = 0 if not title: file_parts = splitext(basename(data)) title = file_parts[0] # Setup app cellxgene_url = f"http://{host}:{port}" # Import Flask app from server.app.app import app app.config.update(DATASET_TITLE=title) if not verbose: log = logging.getLogger("werkzeug") log.setLevel(logging.ERROR) click.echo( f"[cellxgene] Loading data from {basename(data)}, this may take awhile..." ) # Fix for anaconda python. matplotlib typically expects python to be installed as a framework TKAgg is usually # available and fixes this issue. See https://matplotlib.org/faq/virtualenv_faq.html import matplotlib as mpl mpl.use("TkAgg") from server.app.scanpy_engine.scanpy_engine import ScanpyEngine args = { "layout": layout, "diffexp": diffexp, "max_category_items": max_category_items, "diffexp_lfc_cutoff": diffexp_lfc_cutoff, "obs_names": obs_names, "var_names": var_names, } try: app.data = ScanpyEngine(data, args) except ScanpyFileError as e: raise click.ClickException(f"{e}") if open_browser: click.echo( f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now." ) webbrowser.open(cellxgene_url) else: click.echo( f"[cellxgene] Launching! Please go to {cellxgene_url} in your browser." ) click.echo("[cellxgene] Type CTRL-C at any time to exit.") if not verbose: f = open(devnull, "w") sys.stdout = f app.run(host=host, debug=debug, port=port, threaded=True)
def setUp(self): self.data_file = "example-dataset/pbmc3k.h5ad" self.data = ScanpyEngine()
def setUp(self): self.data = ScanpyEngine("example-dataset/", schema="data_schema.json")
class NaNTest(unittest.TestCase): def setUp(self): self.args = { "layout": "umap", "diffexp": "ttest", "max_category_items": 100, "obs_names": None, "var_names": None, "diffexp_lfc_cutoff": 0.01, } with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UserWarning) self.data = ScanpyEngine("server/test/test_datasets/nan.h5ad", self.args) self.data._create_schema() def test_load(self): with self.assertWarns(UserWarning): ScanpyEngine("server/test/test_datasets/nan.h5ad", self.args) def test_init(self): self.assertEqual(self.data.cell_count, 100) self.assertEqual(self.data.gene_count, 100) epsilon = 0.000_005 self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon) def test_dataframe(self): data_frame_var = decode_fbs.decode_matrix_FBS( self.data.data_frame_to_fbs_matrix(None, "var")) self.assertIsNotNone(data_frame_var) self.assertEqual(data_frame_var["n_rows"], 100) self.assertEqual(data_frame_var["n_cols"], 100) self.assertTrue(math.isnan(data_frame_var["columns"][3][3])) with pytest.raises(JSONEncodingValueError): json.loads(self.data.data_frame(None, "obs")) with pytest.raises(JSONEncodingValueError): json.loads(self.data.data_frame(None, "var")) def test_dataframe_obs_not_implemented(self): with self.assertRaises(ValueError) as cm: decode_fbs.decode_matrix_FBS( self.data.data_frame_to_fbs_matrix(None, "obs")) self.assertIsNotNone(cm.exception) def test_annotation(self): annotations = decode_fbs.decode_matrix_FBS( self.data.annotation_to_fbs_matrix("obs")) self.assertEqual( annotations["col_idx"], ["name", "n_genes", "percent_mito", "n_counts", "louvain"]) self.assertEqual(annotations["n_rows"], 100) self.assertTrue(math.isnan(annotations["columns"][2][0])) annotations = decode_fbs.decode_matrix_FBS( self.data.annotation_to_fbs_matrix("var")) self.assertEqual(annotations["col_idx"], ["name", "n_cells", "var_with_nans"]) self.assertEqual(annotations["n_rows"], 100) self.assertTrue(math.isnan(annotations["columns"][2][0])) with pytest.raises(JSONEncodingValueError): json.loads(self.data.annotation(None, "obs")) with pytest.raises(JSONEncodingValueError): json.loads(self.data.annotation(None, "var"))