Beispiel #1
0
    def test_update(self):
        c = AppConfig()
        c.update_server_config(app__verbose=True,
                               multi_dataset__dataroot="datadir")
        v = c.server_config.changes_from_default()
        self.assertCountEqual(v,
                              [("app__verbose", True, False),
                               ("multi_dataset__dataroot", "datadir", None)])

        c = AppConfig()
        c.update_default_dataset_config(app__scripts=(),
                                        app__inline_scripts=())
        v = c.server_config.changes_from_default()
        self.assertCountEqual(v, [])

        c = AppConfig()
        c.update_default_dataset_config(app__scripts=[],
                                        app__inline_scripts=[])
        v = c.default_dataset_config.changes_from_default()
        self.assertCountEqual(v, [])

        c = AppConfig()
        c.update_default_dataset_config(app__scripts=("a", "b"),
                                        app__inline_scripts=["c", "d"])
        v = c.default_dataset_config.changes_from_default()
        self.assertCountEqual(v, [("app__scripts", ["a", "b"], []),
                                  ("app__inline_scripts", ["c", "d"], [])])
Beispiel #2
0
def data_with_tmp_tiledb_annotations(ext: MatrixDataType):
    tmp_dir = tempfile.mkdtemp()
    fname = {
        MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
        MatrixDataType.CXG: "test/fixtures/pbmc3k.cxg",
    }[ext]
    data_locator = DataLocator(fname)
    config = AppConfig()
    config.update_server_config(
        multi_dataset__dataroot=data_locator.path,
        authentication__type="test",
    )
    config.update_default_dataset_config(
        embeddings__names=["umap"],
        presentation__max_categories=100,
        diffexp__lfc_cutoff=0.01,
        user_annotations__type="hosted_tiledb_array",
        user_annotations__hosted_tiledb_array__db_uri=
        "postgresql://*****:*****@localhost:5432",
        user_annotations__hosted_tiledb_array__hosted_file_directory=tmp_dir)

    config.complete_config()

    data = MatrixDataLoader(data_locator.abspath()).open(config)
    annotations = AnnotationsHostedTileDB(
        tmp_dir,
        DbUtils("postgresql://*****:*****@localhost:5432"),
    )
    return data, tmp_dir, annotations
Beispiel #3
0
    def test_auth_test(self):
        c = AppConfig()
        c.update_server_config(authentication__type="test")
        c.update_server_config(
            multi_dataset__dataroot=dict(
                a1=dict(dataroot=self.dataset_dataroot, base_url="auth"),
                a2=dict(dataroot=self.dataset_dataroot, base_url="no-auth"),
            )
        )

        # specialize the configs
        c.add_dataroot_config("a1", app__authentication_enable=True, user_annotations__enable=True)
        c.add_dataroot_config("a2", app__authentication_enable=False, user_annotations__enable=False)

        c.complete_config()

        with test_server(app_config=c) as server:
            session = requests.Session()

            # auth datasets
            config = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json()
            userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()

            self.assertFalse(userinfo["userinfo"]["is_authenticated"])
            self.assertIsNone(userinfo["userinfo"]["username"])
            self.assertTrue(config["config"]["authentication"]["requires_client_login"])
            self.assertTrue(config["config"]["parameters"]["annotations"])

            login_uri = config["config"]["authentication"]["login"]
            logout_uri = config["config"]["authentication"]["logout"]

            self.assertEqual(login_uri, "/login?dataset=auth/pbmc3k.cxg")
            self.assertEqual(logout_uri, "/logout?dataset=auth/pbmc3k.cxg")

            r = session.get(f"{server}/{login_uri}")
            # check that the login redirect worked
            self.assertEqual(r.history[0].status_code, 302)
            self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg/")

            config = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json()
            userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
            self.assertTrue(userinfo["userinfo"]["is_authenticated"])
            self.assertEqual(userinfo["userinfo"]["username"], "test_account")
            self.assertTrue(config["config"]["parameters"]["annotations"])

            r = session.get(f"{server}/{logout_uri}")
            # check that the logout redirect worked
            self.assertEqual(r.history[0].status_code, 302)
            self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg/")
            config = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json()
            userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
            self.assertFalse(userinfo["userinfo"]["is_authenticated"])
            self.assertIsNone(userinfo["userinfo"]["username"])
            self.assertTrue(config["config"]["parameters"]["annotations"])

            # no-auth datasets
            config = session.get(f"{server}/no-auth/pbmc3k.cxg/api/v0.2/config").json()
            userinfo = session.get(f"{server}/no-auth/pbmc3k.cxg/api/v0.2/userinfo").json()
            self.assertIsNone(userinfo)
            self.assertFalse(config["config"]["parameters"]["annotations"])
Beispiel #4
0
    def test_run(self):

        tempdir = tempfile.TemporaryDirectory(dir=f"{PROJECT_ROOT}/server")
        tempdirname = tempdir.name

        c = AppConfig()
        # test that eb works
        c.update_server_config(
            multi_dataset__dataroot=f"{PROJECT_ROOT}/server/test/test_datasets",
            app__flask_secret_key="open sesame")

        c.complete_config()
        c.write_config(f"{tempdirname}/config.yaml")

        subprocess.check_call(f"git ls-files . | cpio -pdm {tempdirname}",
                              cwd=f"{PROJECT_ROOT}/server/eb",
                              shell=True)
        subprocess.check_call(["make", "build"], cwd=tempdirname)

        with run_eb_app(tempdirname) as server:
            session = requests.Session()

            r = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config")
            data_config = r.json()
            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
 def test_url_http(self):
     url = "http://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/example-dataset/pbmc3k.h5ad"
     locator = DataLocator(url)
     config = AppConfig()
     config.update(**self.args)
     data = AnndataAdaptor(locator, config)
     self.stdAsserts(data)
Beispiel #6
0
    def test_get_config_vars_from_aws_secrets(self, mock_get_secret_key):
        mock_get_secret_key.return_value = {
            "flask_secret_key": "mock_flask_secret",
            "oauth_client_secret": "mock_oauth_secret",
            "db_uri": "mock_db_uri"
        }

        config = AppConfig()

        with self.assertLogs(level="INFO") as logger:
            from server.common.aws_secret_utils import handle_config_from_secret
            # should not throw error
            # "AttributeError: 'XConfig' object has no attribute 'x'"
            handle_config_from_secret(config)

            # should log 3 lines (one for each var set from a secret)
            self.assertEqual(len(logger.output), 3)
            self.assertIn('INFO:root:set app__flask_secret_key from secret',
                          logger.output[0])
            self.assertIn(
                'INFO:root:set authentication__params_oauth__client_secret from secret',
                logger.output[1])
            self.assertIn(
                'INFO:root:set user_annotations__hosted_tiledb_array__db_uri from secret',
                logger.output[2])
            self.assertEqual(config.server_config.app__flask_secret_key,
                             "mock_flask_secret")
            self.assertEqual(
                config.server_config.
                authentication__params_oauth__client_secret,
                "mock_oauth_secret")
            self.assertEqual(
                config.default_dataset_config.
                user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")
Beispiel #7
0
def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
    tmp_dir = tempfile.mkdtemp()
    annotations_file = path.join(tmp_dir, "test_annotations.csv")
    if annotations_fixture:
        shutil.copyfile(
            f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-annotations.csv",
            annotations_file)
    args = {
        "embeddings__names": ["umap"],
        "presentation__max_categories": 100,
        "single_dataset__obs_names": None,
        "single_dataset__var_names": None,
        "diffexp__lfc_cutoff": 0.01,
    }
    fname = {
        MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
        MatrixDataType.CXG: "test/test_datasets/pbmc3k.cxg",
    }[ext]
    data_locator = DataLocator(fname)
    config = AppConfig()
    config.update(**args)
    config.update(single_dataset__datapath=data_locator.path)
    config.complete_config()
    data = MatrixDataLoader(data_locator.abspath()).open(config)
    annotations = AnnotationsLocalFile(None, annotations_file)
    return data, tmp_dir, annotations
 def test_posix_file(self):
     locator = DataLocator("../example-dataset/pbmc3k.h5ad")
     config = AppConfig()
     config.update(**self.args)
     config.update(single_dataset__datapath=locator.path)
     config.complete_config()
     data = AnndataAdaptor(locator, config)
     self.stdAsserts(data)
Beispiel #9
0
    def test_update(self):
        c = AppConfig()
        c.update(server__verbose=True, multi_dataset__dataroot="datadir")

        v = c.changes_from_default()
        self.assertCountEqual(v,
                              [("server__verbose", True, False),
                               ("multi_dataset__dataroot", "datadir", None)]),
 def setUp(self):
     self.data_file = DataLocator(
         f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
     config = AppConfig()
     config.update_server_config(
         single_dataset__datapath=self.data_file.path)
     config.complete_config()
     self.data = AnndataAdaptor(self.data_file, config)
 def get_basic_config(self):
     config = AppConfig()
     config.update_server_config(
         single_dataset__obs_names=None, single_dataset__var_names=None,
     )
     config.update_default_dataset_config(
         embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01,
     )
     return config
Beispiel #12
0
    def test_auth_none(self):
        c = AppConfig()
        c.update_server_config(
            authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot
        )
        c.update_default_dataset_config(user_annotations__enable=False)

        c.complete_config()

        with test_server(app_config=c) as server:
            session = requests.Session()
            config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
            userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
            self.assertNotIn("authentication", config["config"])
            self.assertIsNone(userinfo)
Beispiel #13
0
    def test_access_error(self):
        with tempfile.TemporaryDirectory() as dirname:
            self.make_temporay_datasets(dirname, 1)

            app_config = AppConfig()
            m = MatrixDataCacheManager(max_cached=3, timelimit_s=1)

            # use the 0 datasets
            self.use_dataset(m, dirname, app_config, 0)
            self.check_datasets(m, dirname, [0])

            # use the 0 datasets, but this time a DatasetAccessError is raised.
            # verify that dataset is removed from the cache.
            self.use_dataset_with_error(m, dirname, app_config, 0)
            self.check_datasets(m, dirname, [])
Beispiel #14
0
    def test_auth_session(self):
        c = AppConfig()
        c.update_server_config(
            authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot
        )
        c.update_default_dataset_config(user_annotations__enable=True)
        c.complete_config()

        with test_server(app_config=c) as server:
            session = requests.Session()
            config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
            userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()

            self.assertFalse(config["config"]["authentication"]["requires_client_login"])
            self.assertTrue(userinfo["userinfo"]["is_authenticated"])
            self.assertEqual(userinfo["userinfo"]["username"], "anonymous")
Beispiel #15
0
def app_config(data_locator, backed=False):
    args = {
        "embeddings__names": ["umap", "tsne", "pca"],
        "presentation__max_categories": 100,
        "single_dataset__obs_names": None,
        "single_dataset__var_names": None,
        "diffexp__lfc_cutoff": 0.01,
        "adaptor__anndata_adaptor__backed": backed,
        "single_dataset__datapath": data_locator,
        "limits__diffexp_cellcount_max": None,
        "limits__column_request_max": None,
    }
    config = AppConfig()
    config.update(**args)
    config.complete_config()
    return config
Beispiel #16
0
    def test_auth_oauth_session(self):
        # test with session cookies
        app_config = AppConfig()
        app_config.update_server_config(
            authentication__type="oauth",
            authentication__params_oauth__api_base_url=
            f"http://localhost:{PORT}",
            authentication__params_oauth__client_id="mock_client_id",
            authentication__params_oauth__client_secret="mock_client_secret",
            authentication__params_oauth__session_cookie=True,
        )

        app_config.update_server_config(
            multi_dataset__dataroot=self.dataset_dataroot)
        app_config.complete_config()

        self.auth_flow(app_config)
Beispiel #17
0
    def test_auth_test_single(self):
        c = AppConfig()
        c.update_server_config(
            authentication__type="test",
            single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg")

        c.complete_config()

        with test_server(app_config=c) as server:
            session = requests.Session()
            config = session.get(f"{server}/api/v0.2/config").json()
            userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
            self.assertFalse(userinfo["userinfo"]["is_authenticated"])
            self.assertIsNone(userinfo["userinfo"]["username"])
            self.assertTrue(config["config"]["authentication"]["requires_client_login"])
            self.assertTrue(config["config"]["parameters"]["annotations"])

            login_uri = config["config"]["authentication"]["login"]
            logout_uri = config["config"]["authentication"]["logout"]

            self.assertEqual(login_uri, "/login")
            self.assertEqual(logout_uri, "/logout")

            r = session.get(f"{server}/{login_uri}")
            # check that the login redirect worked
            self.assertEqual(r.history[0].status_code, 302)
            self.assertEqual(r.url, f"{server}/")

            config = session.get(f"{server}/api/v0.2/config").json()
            userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
            self.assertTrue(userinfo["userinfo"]["is_authenticated"])
            self.assertEqual(userinfo["userinfo"]["username"], "test_account")
            self.assertTrue(config["config"]["parameters"]["annotations"])

            r = session.get(f"{server}/{logout_uri}")
            # check that the logout redirect worked
            self.assertEqual(r.history[0].status_code, 302)
            self.assertEqual(r.url, f"{server}/")
            config = session.get(f"{server}/api/v0.2/config").json()
            userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
            self.assertFalse(userinfo["userinfo"]["is_authenticated"])
            self.assertIsNone(userinfo["userinfo"]["username"])
            self.assertTrue(config["config"]["parameters"]["annotations"])
Beispiel #18
0
    def test_auth_oauth_cookie(self):
        # test with specified cookie
        app_config = AppConfig()
        app_config.update_server_config(
            authentication__type="oauth",
            authentication__params_oauth__api_base_url=
            f"http://localhost:{PORT}",
            authentication__params_oauth__client_id="mock_client_id",
            authentication__params_oauth__client_secret="mock_client_secret",
            authentication__params_oauth__session_cookie=False,
            authentication__params_oauth__cookie=dict(key="test_cxguser",
                                                      httponly=True,
                                                      max_age=60),
        )

        app_config.update_server_config(
            multi_dataset__dataroot=self.dataset_dataroot)
        app_config.complete_config()

        self.auth_flow(app_config, "test_cxguser")
Beispiel #19
0
def app_config(data_locator,
               backed=False,
               extra_server_config={},
               extra_dataset_config={}):
    config = AppConfig()
    config.update_server_config(
        single_dataset__obs_names=None,
        single_dataset__var_names=None,
        adaptor__anndata_adaptor__backed=backed,
        single_dataset__datapath=data_locator,
        limits__diffexp_cellcount_max=None,
        limits__column_request_max=None,
    )
    config.update_default_dataset_config(
        embeddings__names=["umap", "tsne", "pca"],
        presentation__max_categories=100,
        diffexp__lfc_cutoff=0.01)
    config.update_server_config(**extra_server_config)
    config.update_default_dataset_config(**extra_dataset_config)
    config.complete_config()
    return config
Beispiel #20
0
    def test_timelimit(self):
        with tempfile.TemporaryDirectory() as dirname:
            self.make_temporay_datasets(dirname, 2)

            app_config = AppConfig()
            m = MatrixDataCacheManager(max_cached=3, timelimit_s=1)

            adaptor = self.use_dataset(m, dirname, app_config, 0)
            adaptor1 = self.use_dataset(m, dirname, app_config, 0)
            self.assertTrue(adaptor is adaptor1)

            # wait until the timelimit expires and check that there is a new adaptor
            time.sleep(1.1)
            adaptor2 = self.use_dataset(m, dirname, app_config, 0)
            self.assertTrue(adaptor is not adaptor2)
            self.check_datasets(m, dirname, [0])

            # now load a different dataset and see if dataset 0 gets evicted
            time.sleep(1.1)
            self.use_dataset(m, dirname, app_config, 1)
            self.check_datasets(m, dirname, [1])
Beispiel #21
0
    def test_basic(self):
        with tempfile.TemporaryDirectory() as dirname:
            self.make_temporay_datasets(dirname, 5)
            app_config = AppConfig()
            m = MatrixDataCacheManager(max_cached=3, timelimit_s=None)

            # should have only dataset 0
            self.use_dataset(m, dirname, app_config, 0)
            self.check_datasets(m, dirname, [0])

            # should have datasets 0, 1
            self.use_dataset(m, dirname, app_config, 1)
            self.check_datasets(m, dirname, [0, 1])

            # should have datasets 0, 1, 2
            self.use_dataset(m, dirname, app_config, 2)
            self.check_datasets(m, dirname, [0, 1, 2])

            # should have datasets 1, 2, 3
            self.use_dataset(m, dirname, app_config, 3)
            self.check_datasets(m, dirname, [1, 2, 3])

            # use dataset 1, making is more recent than dataset 2
            self.use_dataset(m, dirname, app_config, 1)
            self.check_datasets(m, dirname, [1, 2, 3])

            # use dataset 4, should have 1,3,4
            self.use_dataset(m, dirname, app_config, 4)
            self.check_datasets(m, dirname, [1, 3, 4])

            # use dataset 4 a few more times, get the count to 3
            self.use_dataset(m, dirname, app_config, 4)
            self.use_dataset(m, dirname, app_config, 4)

            datasets = self.get_datasets(m, dirname)
            self.assertEqual(datasets[1].num_access, 2)
            self.assertEqual(datasets[3].num_access, 1)
            self.assertEqual(datasets[4].num_access, 3)
Beispiel #22
0
def main():
    parser = argparse.ArgumentParser("A command to test diffexp")
    parser.add_argument("dataset", help="name of a dataset to load")
    parser.add_argument("-na",
                        "--numA",
                        type=int,
                        required=True,
                        help="number of rows in group A")
    parser.add_argument("-nb",
                        "--numB",
                        type=int,
                        required=True,
                        help="number of rows in group B")
    parser.add_argument("-t",
                        "--trials",
                        default=1,
                        type=int,
                        help="number of trials")
    parser.add_argument("-a",
                        "--alg",
                        choices=("default", "generic", "cxg"),
                        default="default",
                        help="algorithm to use")
    parser.add_argument("-s",
                        "--show",
                        default=False,
                        action="store_true",
                        help="show the results")
    parser.add_argument("-n",
                        "--new-selection",
                        default=False,
                        action="store_true",
                        help="change the selection between each trial")
    parser.add_argument("--seed",
                        default=1,
                        type=int,
                        help="set the random seed")

    args = parser.parse_args()

    app_config = AppConfig()
    app_config.single_dataset__datapath = args.dataset
    app_config.server__verbose = True
    app_config.complete_config()

    loader = MatrixDataLoader(args.dataset)
    adaptor = loader.open(app_config)

    if args.show:
        if isinstance(adaptor, CxgAdaptor):
            adaptor.open_array("X").schema.dump()

    numA = args.numA
    numB = args.numB
    rows = adaptor.get_shape()[0]

    random.seed(args.seed)

    if not args.new_selection:
        samples = random.sample(range(rows), numA + numB)
        filterA = samples[:numA]
        filterB = samples[numA:]

    for i in range(args.trials):
        if args.new_selection:
            samples = random.sample(range(rows), numA + numB)
            filterA = samples[:numA]
            filterB = samples[numA:]

        maskA = np.zeros(rows, dtype=bool)
        maskA[filterA] = True
        maskB = np.zeros(rows, dtype=bool)
        maskB[filterB] = True

        t1 = time.time()
        if args.alg == "default":
            results = adaptor.compute_diffexp_ttest(maskA, maskB)
        elif args.alg == "generic":
            results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB)
        elif args.alg == "cxg":
            if not isinstance(adaptor, CxgAdaptor):
                print("cxg only works with CxgAdaptor")
                sys.exit(1)
            results = diffexp_cxg.diffexp_ttest(adaptor, maskA, maskB)

        t2 = time.time()
        print("TIME=", t2 - t1)

    if args.show:
        for res in results:
            print(res)
Beispiel #23
0
                    # we use jinja2 template include, which trims final newline if present.
                    if content[-1] == 0x0A:
                        content = content[0:-1]
                    hash = base64.b64encode(hashlib.sha256(content).digest())
                    hashes.append(f"'sha256-{hash.decode('utf-8')}'")
        return hashes

    @staticmethod
    def get_csp_hashes(app, app_config):
        script_hashes, style_hashes = WSGIServer.load_static_csp_hashes(app)
        script_hashes += WSGIServer.compute_inline_scp_hashes(app, app_config)
        return (script_hashes, style_hashes)


try:
    app_config = AppConfig()

    has_config = False
    # config file: look first for "config.yaml" in the current working directory
    config_file = "config.yaml"
    config_location = DataLocator(config_file)
    if config_location.exists():
        with config_location.local_handle() as lh:
            logging.info(f"Configuration from {config_file}")
            app_config.update_from_config_file(lh)
            has_config = True

    else:
        # config file: second, use the CXG_CONFIG_FILE
        config_file = os.getenv("CXG_CONFIG_FILE")
        if config_file:
    def test_multi_dataset(self):

        c = AppConfig()
        # test for illegal url_dataroots
        for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
            c.update_server_config(
                multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
            )
            with self.assertRaises(ConfigurationError):
                c.complete_config()

        # test for legal url_dataroots
        for legal in ("d", "this.is-okay_", "a/b"):
            c.update_server_config(
                multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
            )
            c.complete_config()

        # test that multi dataroots work end to end
        c.update_server_config(
            multi_dataset__dataroot=dict(
                s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
                s2=dict(dataroot=f"{PROJECT_ROOT}/server/test/test_datasets", base_url="set2"),
                s3=dict(dataroot=f"{PROJECT_ROOT}/server/test/test_datasets", base_url="set3"),
            )
        )

        # Change this default to test if the dataroot overrides below work.
        c.update_default_dataset_config(app__about_legal_tos="tos_default.html")

        # specialize the configs for set1
        c.add_dataroot_config(
            "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
        )

        # specialize the configs for set2
        c.add_dataroot_config(
            "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
        )

        # no specializations for set3 (they get the default dataset config)
        c.complete_config()

        with test_server(app_config=c) as server:
            session = requests.Session()

            r = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
            data_config = r.json()
            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
            assert data_config["config"]["parameters"]["annotations"] is False
            assert data_config["config"]["parameters"]["disable-diffexp"] is False
            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"

            r = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
            data_config = r.json()
            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
            assert data_config["config"]["parameters"]["annotations"] is True
            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"

            r = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
            data_config = r.json()
            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
            assert data_config["config"]["parameters"]["annotations"] is True
            assert data_config["config"]["parameters"]["disable-diffexp"] is False
            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"

            r = session.get(f"{server}/health")
            assert r.json()["status"] == "pass"
Beispiel #25
0
def main():
    parser = argparse.ArgumentParser("A command to test diffexp")
    parser.add_argument("dataset", help="name of a dataset to load")
    parser.add_argument("-na",
                        "--numA",
                        type=int,
                        help="number of rows in group A")
    parser.add_argument("-nb",
                        "--numB",
                        type=int,
                        help="number of rows in group B")
    parser.add_argument("-va",
                        "--varA",
                        help="obs variable:value to use for group A")
    parser.add_argument("-vb",
                        "--varB",
                        help="obs variable:value to use for group B")
    parser.add_argument("-t",
                        "--trials",
                        default=1,
                        type=int,
                        help="number of trials")
    parser.add_argument("-a",
                        "--alg",
                        choices=("default", "generic", "cxg"),
                        default="default",
                        help="algorithm to use")
    parser.add_argument("-s",
                        "--show",
                        default=False,
                        action="store_true",
                        help="show the results")
    parser.add_argument("-n",
                        "--new-selection",
                        default=False,
                        action="store_true",
                        help="change the selection between each trial")
    parser.add_argument("--seed",
                        default=1,
                        type=int,
                        help="set the random seed")

    args = parser.parse_args()

    app_config = AppConfig()
    app_config.update_server_config(single_dataset__datapath=args.dataset)
    app_config.update_server_config(app__verbose=True)
    app_config.complete_config()

    loader = MatrixDataLoader(args.dataset)
    adaptor = loader.open(app_config)

    if args.show:
        if isinstance(adaptor, CxgAdaptor):
            adaptor.open_array("X").schema.dump()

    random.seed(args.seed)
    np.random.seed(args.seed)
    rows = adaptor.get_shape()[0]

    if args.numA:
        filterA = random.sample(range(rows), args.numA)
    elif args.varA:
        vname, vval = args.varA.split(":")
        filterA = get_filter_from_obs(adaptor, vname, vval)
    else:
        print("must supply numA or varA")
        sys.exit(1)

    if args.numB:
        filterB = random.sample(range(rows), args.numB)
    elif args.varB:
        vname, vval = args.varB.split(":")
        filterB = get_filter_from_obs(adaptor, vname, vval)
    else:
        print("must supply numB or varB")
        sys.exit(1)

    for i in range(args.trials):
        if args.new_selection:
            if args.numA:
                filterA = random.sample(range(rows), args.numA)
            if args.numB:
                filterB = random.sample(range(rows), args.numB)

        maskA = np.zeros(rows, dtype=bool)
        maskA[filterA] = True
        maskB = np.zeros(rows, dtype=bool)
        maskB[filterB] = True

        t1 = time.time()
        if args.alg == "default":
            results = adaptor.compute_diffexp_ttest(maskA, maskB)
        elif args.alg == "generic":
            results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB)
        elif args.alg == "cxg":
            if not isinstance(adaptor, CxgAdaptor):
                print("cxg only works with CxgAdaptor")
                sys.exit(1)
            results = diffexp_cxg.diffexp_ttest(adaptor, maskA, maskB)

        t2 = time.time()
        print("TIME=", t2 - t1)

    if args.show:
        for res in results:
            print(res)
Beispiel #26
0
import logging
import sys
import webbrowser
from os import devnull

import click
from flask_compress import Compress
from flask_cors import CORS

from server.app.app import Server
from server.common.app_config import AppConfig
from server.common.default_config import default_config
from server.common.errors import DatasetAccessError, ConfigurationError
from server.common.utils.utils import sort_options

DEFAULT_CONFIG = AppConfig()


def annotation_args(func):
    @click.option(
        "--disable-annotations",
        is_flag=True,
        default=not DEFAULT_CONFIG.default_dataset_config.user_annotations__enable,
        show_default=True,
        help="Disable user annotation of data.",
    )
    @click.option(
        "--annotations-file",
        default=DEFAULT_CONFIG.default_dataset_config.user_annotations__local_file_csv__file,
        show_default=True,
        multiple=False,
Beispiel #27
0
def launch(
        datapath,
        dataroot,
        verbose,
        debug,
        open_browser,
        port,
        host,
        embedding,
        obs_names,
        var_names,
        max_category_items,
        disable_custom_colors,
        diffexp_lfc_cutoff,
        title,
        scripts,
        about,
        disable_annotations,
        annotations_file,
        annotations_dir,
        backed,
        disable_diffexp,
        experimental_annotations_ontology,
        experimental_annotations_ontology_obo,
        experimental_enable_reembedding,
        config_file,
        dump_default_config,
):
    """Launch the cellxgene data viewer.
    This web app lets you explore single-cell expression data.
    Data must be in a format that cellxgene expects.
    Read the "getting started" guide to learn more:
    https://chanzuckerberg.github.io/cellxgene/getting-started.html

    Examples:

    > cellxgene launch example-dataset/pbmc3k.h5ad --title pbmc3k

    > cellxgene launch <your data file> --title <your title>

    > cellxgene launch <url>"""

    # TODO Examples to provide when "--dataroot" is unhidden
    # > cellxgene launch --dataroot example-dataset/
    #
    # > cellxgene launch --dataroot <url>

    if dump_default_config:
        print(default_config)
        sys.exit(0)

    # Startup message
    click.echo("[cellxgene] Starting the CLI...")

    # app config
    app_config = AppConfig()
    server_config = app_config.server_config

    try:
        if config_file:
            app_config.update_from_config_file(config_file)

        # Determine which config options were give on the command line.
        # Those will override the ones provided in the config file (if provided).
        cli_config = AppConfig()
        cli_config.update_server_config(
            app__verbose=verbose,
            app__debug=debug,
            app__host=host,
            app__port=port,
            app__open_browser=open_browser,
            single_dataset__datapath=datapath,
            single_dataset__title=title,
            single_dataset__about=about,
            single_dataset__obs_names=obs_names,
            single_dataset__var_names=var_names,
            multi_dataset__dataroot=dataroot,
            adaptor__anndata_adaptor__backed=backed,
        )
        cli_config.update_default_dataset_config(
            app__scripts=scripts,
            user_annotations__enable=not disable_annotations,
            user_annotations__local_file_csv__file=annotations_file,
            user_annotations__local_file_csv__directory=annotations_dir,
            user_annotations__ontology__enable=experimental_annotations_ontology,
            user_annotations__ontology__obo_location=experimental_annotations_ontology_obo,
            presentation__max_categories=max_category_items,
            presentation__custom_colors=not disable_custom_colors,
            embeddings__names=embedding,
            embeddings__enable_reembedding=experimental_enable_reembedding,
            diffexp__enable=not disable_diffexp,
            diffexp__lfc_cutoff=diffexp_lfc_cutoff,
        )

        diff = cli_config.server_config.changes_from_default()
        changes = {key: val for key, val, _ in diff}
        app_config.update_server_config(**changes)

        diff = cli_config.default_dataset_config.changes_from_default()
        changes = {key: val for key, val, _ in diff}
        app_config.update_default_dataset_config(**changes)

        # process the configuration
        #  any errors will be thrown as an exception.
        #  any info messages will be passed to the messagefn function.

        def messagefn(message):
            click.echo("[cellxgene] " + message)

        # Use a default secret if one is not provided
        if not server_config.app__flask_secret_key:
            app_config.update_server_config(app__flask_secret_key="SparkleAndShine")

        app_config.complete_config(messagefn)

    except (ConfigurationError, DatasetAccessError) as e:
        raise click.ClickException(e)

    handle_scripts(scripts)

    # create the server
    server = CliLaunchServer(app_config)

    if not server_config.app__verbose:
        log = logging.getLogger("werkzeug")
        log.setLevel(logging.ERROR)

    cellxgene_url = f"http://{app_config.server_config.app__host}:{app_config.server_config.app__port}"
    if server_config.app__open_browser:
        click.echo(f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now.")
        webbrowser.open(cellxgene_url)
    else:
        click.echo(f"[cellxgene] Launching! Please go to {cellxgene_url} in your browser.")

    click.echo("[cellxgene] Type CTRL-C at any time to exit.")

    if not server_config.app__verbose:
        f = open(devnull, "w")
        sys.stdout = f

    try:
        server.app.run(
            host=server_config.app__host,
            debug=server_config.app__debug,
            port=server_config.app__port,
            threaded=not server_config.app__debug,
            use_debugger=False,
            use_reloader=False,
        )
    except OSError as e:
        if e.errno == errno.EADDRINUSE:
            raise click.ClickException("Port is in use, please specify an open port using the --port flag.") from e
        raise