Beispiel #1
0
def login(key, host, cloud, relogin, anonymously, no_offline=False):
    # TODO: handle no_offline
    anon_mode = "must" if anonymously else "never"

    if host and not host.startswith("http"):
        raise ClickException("host must start with http(s)://")

    _api = InternalApi()
    if host == "https://api.wandb.ai" or (host is None and cloud):
        _api.clear_setting("base_url", globally=True, persist=True)
        # To avoid writing an empty local settings file, we only clear if it exists
        if os.path.exists(Settings._local_path()):
            _api.clear_setting("base_url", persist=True)
    elif host:
        # force relogin if host is specified
        _api.set_setting("base_url", host.strip("/"), globally=True, persist=True)
    key = key[0] if len(key) > 0 else None
    if host or cloud or key:
        relogin = True

    wandb.setup(
        settings=wandb.Settings(
            _cli_only_mode=True,
            _disable_viewer=relogin,
            anonymous=anon_mode,
            base_url=host,
        )
    )
    wandb.login(relogin=relogin, key=key, anonymous=anon_mode, host=host, force=True)
Beispiel #2
0
def test_get_url(git_repo, loggedin):
    api = InternalApi({"entity": "cool"})
    api.set_setting("anonymous", "true")
    run = wandb_run.Run(api=api)
    assert run.get_url() == "https://app.wandb.ai/cool/uncategorized/runs/"+run.id+"?apiKey="+"X"*40
    assert run.get_project_url() == "https://app.wandb.ai/cool/uncategorized?apiKey="+"X"*40
    api.set_setting("entity", "")
    with pytest.raises(CommError):
        run.get_url()
Beispiel #3
0
def off():
    wandb.ensure_configured()
    api = InternalApi()
    try:
        api.set_setting('disabled', 'true')
        click.echo(
            "W&B disabled, running your script from this directory will only write metadata locally.")
    except configparser.Error as e:
        click.echo(
            'Unable to write config, copy and paste the following in your terminal to turn off W&B:\nexport WANDB_MODE=dryrun')
Beispiel #4
0
def fake_run_manager(mocker,
                     run=None,
                     cloud=True,
                     rm_class=wandb.run_manager.RunManager):
    # NOTE: This will create a run directory so make sure it's called in an isolated file system
    # We have an optional rm_class object because we mock it above so we need it before it's mocked
    api = InternalApi(load_settings=False)
    api.set_setting('project', 'testing')

    if wandb.run is None:
        wandb.run = run or Run()
        wandb.config = wandb.run.config
    wandb.run._api = api
    wandb.run._mkdir()
    wandb.run.socket = wandb_socket.Server()
    api.set_current_run_id(wandb.run.id)
    mocker.patch('wandb.apis.internal.FileStreamApi')
    api._file_stream_api = mocker.MagicMock()
    run_manager = rm_class(wandb.run, cloud=cloud, port=wandb.run.socket.port)

    class FakeProc(object):
        def poll(self):
            return None

        def exit(self, code=0):
            return None

    run_manager.proc = FakeProc()
    run_manager._meta = mocker.MagicMock()
    run_manager._stdout_tee = mocker.MagicMock()
    run_manager._stderr_tee = mocker.MagicMock()
    run_manager._output_log = mocker.MagicMock()
    run_manager._stdout_stream = mocker.MagicMock()
    run_manager._stderr_stream = mocker.MagicMock()
    run_manager.mirror_stdout_stderr = mocker.MagicMock()
    run_manager.unmirror_stdout_stderr = mocker.MagicMock()
    socket_thread = threading.Thread(target=wandb.run.socket.listen)
    socket_thread.start()
    run_manager._socket.ready()
    thread = threading.Thread(target=run_manager._sync_etc)
    thread.daemon = True
    thread.start()

    def test_shutdown():
        if wandb.run and wandb.run.socket:
            wandb.run.socket.done()
            # TODO: is this needed?
            socket_thread.join()
            thread.join()

    run_manager.test_shutdown = test_shutdown
    run_manager._unblock_file_observer()
    run_manager._file_pusher._push_function = mocker.MagicMock()
    return run_manager
Beispiel #5
0
def off():
    api = InternalApi()
    try:
        api.set_setting("disabled", "true", persist=True)
        click.echo(
            "W&B disabled, running your script from this directory will only write metadata locally."
        )
    except configparser.Error:
        click.echo(
            "Unable to write config, copy and paste the following in your terminal to turn off W&B:\nexport WANDB_MODE=dryrun"
        )
Beispiel #6
0
    def wrapper(mocker, run, status_code=200, body_match='', error=None):
        api = InternalApi()
        api.set_setting("project", "new-project")
        api.set_setting("entity", "bagsy")

        def match_body(request):
            return body_match in (request.text or '')

        api._current_run_id = run
        url = api.get_file_stream_api()._endpoint
        print("Mocked %s" % url)
        return mocker.register_uri("POST", url, [{'json': {'limits': {}}, 'status_code': status_code}],
                                   additional_matcher=match_body)
Beispiel #7
0
def test_anonymous_redaction(mocker):
    mocker.patch.object(sys, 'argv', ["foo", "bar"])
    api = InternalApi()
    api.set_setting('anonymous', 'true')

    meta = Meta(api, "wandb")
    meta.write()

    print(meta.data)
    assert "host" not in meta.data
    assert "username" not in meta.data
    assert "executable" not in meta.data
    assert "email" not in meta.data
    assert "root" not in meta.data
Beispiel #8
0
def local(ctx, port, env, daemon, upgrade, edge):
    api = InternalApi()
    if not find_executable('docker'):
        raise ClickException(
            "Docker not installed, install it from https://docker.com")
    if wandb.docker.image_id("wandb/local") != wandb.docker.image_id_from_registry("wandb/local"):
        if upgrade:
            subprocess.call(["docker", "pull", "wandb/local"])
        else:
            wandb.termlog("A new version of W&B local is available, upgrade by calling `wandb local --upgrade`")
    running = subprocess.check_output(["docker", "ps", "--filter", "name=wandb-local", "--format", "{{.ID}}"])
    if running != b"":
        if upgrade:
            subprocess.call(["docker", "stop", "wandb-local"])
        else:
            wandb.termerror("A container named wandb-local is already running, run `docker stop wandb-local` if you want to start a new instance")
            exit(1)
    image = "docker.pkg.github.com/wandb/core/local" if edge else "wandb/local"
    username = getpass.getuser()
    env_vars = ['-e', 'LOCAL_USERNAME=%s' % username]
    for e in env:
        env_vars.append('-e')
        env_vars.append(e)
    command = ['docker', 'run', '--rm', '-v', 'wandb:/vol', '-p', port + ':8080', '--name', 'wandb-local'] + env_vars
    host = "http://localhost:%s" % port
    api.set_setting("base_url", host, globally=True, persist=True)
    if daemon:
        command += ["-d"]
    command += [image]

    # DEVNULL is only in py3
    try:
        from subprocess import DEVNULL
    except ImportError:
        DEVNULL = open(os.devnull, 'wb')  # noqa: N806
    code = subprocess.call(command, stdout=DEVNULL)
    if daemon:
        if code != 0:
            wandb.termerror("Failed to launch the W&B local container, see the above error.")
            exit(1)
        else:
            wandb.termlog("W&B local started at http://localhost:%s \U0001F680" % port)
            wandb.termlog("You can stop the server by running `docker stop wandb-local`")
            if not api.api_key:
                # Let the server start before potentially launching a browser
                time.sleep(2)
                ctx.invoke(login, host=host)
Beispiel #9
0
def put(path, name, description, type, alias):
    if name is None:
        name = os.path.basename(path)
    public_api = PublicApi()
    entity, project, artifact_name = public_api._parse_artifact_path(name)
    if project is None:
        project = click.prompt("Enter the name of the project you want to use")
    # TODO: settings nightmare...
    api = InternalApi()
    api.set_setting("entity", entity)
    api.set_setting("project", project)
    artifact = wandb.Artifact(name=artifact_name, type=type, description=description)
    artifact_path = "{entity}/{project}/{name}:{alias}".format(entity=entity,
                                                               project=project, name=artifact_name, alias=alias[0])
    if os.path.isdir(path):
        wandb.termlog("Uploading directory {path} to: \"{artifact_path}\" ({type})".format(
            path=path, type=type, artifact_path=artifact_path))
        artifact.add_dir(path)
    elif os.path.isfile(path):
        wandb.termlog("Uploading file {path} to: \"{artifact_path}\" ({type})".format(
            path=path, type=type, artifact_path=artifact_path))
        artifact.add_file(path)
    elif "://" in path:
        wandb.termlog("Logging reference artifact from {path} to: \"{artifact_path}\" ({type})".format(
            path=path, type=type, artifact_path=artifact_path))
        artifact.add_reference(path)
    else:
        raise ClickException("Path argument must be a file or directory")

    run = wandb.init(entity=entity, project=project, config={"path": path}, job_type="cli_put")
    # We create the artifact manually to get the current version
    res = api.create_artifact(type, artifact_name, artifact.digest,
                              entity_name=entity, project_name=project, run_name=run.id, description=description,
                              aliases=[{"artifactCollectionName": artifact_name, "alias": a} for a in alias])
    artifact_path = artifact_path.split(":")[0] + ":" + res.get("version", "latest")
    # Re-create the artifact and actually upload any files needed
    run.log_artifact(artifact, aliases=alias)
    wandb.termlog("Artifact uploaded, use this artifact in a run by adding:\n", prefix=False)

    wandb.termlog("    artifact = run.use_artifact(\"{path}\")\n".format(
        path=artifact_path,
    ), prefix=False)
Beispiel #10
0
def init(ctx, project, entity, reset):
    from wandb.old.core import _set_stage_dir, __stage_dir__, wandb_dir

    if __stage_dir__ is None:
        _set_stage_dir("wandb")

    # non interactive init
    if reset or project or entity:
        api = InternalApi()
        if reset:
            api.clear_setting("entity", persist=True)
            api.clear_setting("project", persist=True)
            # TODO(jhr): clear more settings?
        if entity:
            api.set_setting("entity", entity, persist=True)
        if project:
            api.set_setting("project", project, persist=True)
        return

    if os.path.isdir(wandb_dir()) and os.path.exists(
        os.path.join(wandb_dir(), "settings")
    ):
        click.confirm(
            click.style(
                "This directory has been configured previously, should we re-configure it?",
                bold=True,
            ),
            abort=True,
        )
    else:
        click.echo(
            click.style("Let's setup this directory for W&B!", fg="green", bold=True)
        )
    api = InternalApi()
    if api.api_key is None:
        ctx.invoke(login)

    viewer = api.viewer()

    # Viewer can be `None` in case your API information became invalid, or
    # in testing if you switch hosts.
    if not viewer:
        click.echo(
            click.style(
                "Your login information seems to be invalid: can you log in again please?",
                fg="red",
                bold=True,
            )
        )
        ctx.invoke(login)

    # This shouldn't happen.
    viewer = api.viewer()
    if not viewer:
        click.echo(
            click.style(
                "We're sorry, there was a problem logging you in. Please send us a note at [email protected] and tell us how this happened.",
                fg="red",
                bold=True,
            )
        )
        sys.exit(1)

    # At this point we should be logged in successfully.
    if len(viewer["teams"]["edges"]) > 1:
        team_names = [e["node"]["name"] for e in viewer["teams"]["edges"]]
        question = {
            "type": "list",
            "name": "team_name",
            "message": "Which team should we use?",
            "choices": team_names
            # TODO(jhr): disabling manual entry for cling
            # 'choices': team_names + ["Manual Entry"]
        }
        result = whaaaaat.prompt([question])
        # result can be empty on click
        if result:
            entity = result["team_name"]
        else:
            entity = "Manual Entry"
        if entity == "Manual Entry":
            entity = click.prompt("Enter the name of the team you want to use")
    else:
        entity = viewer.get("entity") or click.prompt(
            "What username or team should we use?"
        )

    # TODO: this error handling sucks and the output isn't pretty
    try:
        project = prompt_for_project(ctx, entity)
    except ClickWandbException:
        raise ClickException("Could not find team: %s" % entity)

    api.set_setting("entity", entity, persist=True)
    api.set_setting("project", project, persist=True)
    api.set_setting("base_url", api.settings().get("base_url"), persist=True)

    util.mkdir_exists_ok(wandb_dir())
    with open(os.path.join(wandb_dir(), ".gitignore"), "w") as file:
        file.write("*\n!settings")

    click.echo(
        click.style("This directory is configured!  Next, track a run:\n", fg="green")
        + textwrap.dedent(
            """\
        * In your training script:
            {code1}
            {code2}
        * then `{run}`.
        """
        ).format(
            code1=click.style("import wandb", bold=True),
            code2=click.style('wandb.init(project="%s")' % project, bold=True),
            run=click.style("python <train.py>", bold=True),
        )
    )
Beispiel #11
0
def test_set_setting_no_persist_by_default(git_repo):
    os.remove("wandb/settings")
    api = InternalApi({"entity": "cool"})
    api.set_setting("rad", "true")
    assert not os.path.exists("wandb/settings")
    assert api.settings("rad") == "true"