def launch_command( master: str, endpoint: str, config: Dict[str, Any], template: str, context_path: Optional[Path] = None, data: Optional[Dict[str, Any]] = None, preview: Optional[bool] = False, ) -> Any: user_files = [] # type: List[Dict[str, Any]] if context_path: user_files, _ = context.read_context(context_path) body = {"config": config} # type: Dict[str, Any] if template: body["template_name"] = template if len(user_files) > 0: body["files"] = user_files if data is not None: message_bytes = json.dumps(data).encode("utf-8") base64_bytes = base64.b64encode(message_bytes) body["data"] = base64_bytes if preview: body["preview"] = preview return api.post( master, endpoint, body, ).json()
def test_read_context_with_detignore(tmp_path: Path) -> None: with FileTree(tmp_path, {"A.py": "", "B.py": "", "C.py": ""}) as tree: model_def, _ = context.read_context(tree) assert {f["path"] for f in model_def} == {"A.py", "B.py", "C.py"} with FileTree(tmp_path, { "A.py": "", "B.py": "", "C.py": "", ".detignore": "\nA.py\n" }) as tree: model_def, size = context.read_context(tree) assert {f["path"] for f in model_def} == {"B.py", "C.py"} with FileTree(tmp_path, { "A.py": "", "B.py": "", "C.py": "", ".detignore": "\n*.py\n" }) as tree: model_def, size = context.read_context(tree) assert model_def == []
def test_read_context_ignore_pycaches(tmp_path: Path) -> None: with FileTree( tmp_path, { "__pycache__/A.cpython-37.pyc": "", "A.py": "", "subdir/A.py": "", "subdir/__pycache__/A.cpython-37.pyc": "", }, ) as tree: model_def, _ = context.read_context(tree) assert {f["path"] for f in model_def} == {"A.py", "subdir", "subdir/A.py"}
def _path_to_files(path: pathlib.Path) -> List[V1File]: files = [] for item in context.read_context(path)[0]: content = item["content"].decode("utf-8") file = V1File( path=item["path"], type=item["type"], content=content, mtime=item["mtime"], uid=item["uid"], gid=item["gid"], mode=item["mode"], ) files.append(file) return files
def start_tensorboard(args: Namespace) -> None: if not (args.trial_ids or args.experiment_ids): print("Either experiment_ids or trial_ids must be specified.") sys.exit(1) config = parse_config(args.config_file, None, args.config, []) req_body = { "config": config, "trial_ids": args.trial_ids, "experiment_ids": args.experiment_ids, } if args.context is not None: req_body["files"], _ = context.read_context(args.context, constants.MAX_CONTEXT_SIZE) resp = api.post(args.master, "api/v1/tensorboards", json=req_body).json()["tensorboard"] if args.detach: print(resp["id"]) return url = "tensorboard/{}/events".format(resp["id"]) with api.ws(args.master, url) as ws: for msg in ws: if msg["log_event"] is not None: # TensorBoard will print a url by default. The URL is incorrect since # TensorBoard is not aware of the master proxy address it is assigned. if "http" in msg["log_event"]: continue if msg["service_ready_event"]: if args.no_browser: url = api.make_url(args.master, resp["serviceAddress"]) else: url = api.browser_open(args.master, resp["serviceAddress"]) print( colored("TensorBoard is running at: {}".format(url), "green")) render_event_stream(msg) break render_event_stream(msg)
def create_experiment( self, config: Union[str, pathlib.Path, Dict], model_dir: Union[str, pathlib.Path], ) -> experiment.ExperimentReference: """ Create an experiment with config parameters and model directory. The function returns :class:`~determined.experimental.ExperimentReference` of the experiment. Arguments: config(string, pathlib.Path, dictionary): experiment config filename (.yaml) or a dict. model_dir(string): directory containing model definition. """ check.is_instance( config, (str, pathlib.Path, dict), "config parameter must be dictionary or path" ) if isinstance(config, str): with open(config) as f: experiment_config = util.safe_load_yaml_with_exceptions(f) elif isinstance(config, pathlib.Path): with config.open() as f: experiment_config = util.safe_load_yaml_with_exceptions(f) elif isinstance(config, Dict): experiment_config = config if isinstance(model_dir, str): model_dir = pathlib.Path(model_dir) model_context, _ = context.read_context(model_dir) resp = self._session.post( "/api/v1/experiments", body={ "config": yaml.safe_dump(experiment_config), "model_definition": model_context, }, ) exp_id = _CreateExperimentResponse(resp.json()).id exp = experiment.ExperimentReference(exp_id, self._session) exp.activate() return exp
def launch_command( master: str, endpoint: str, config: Dict[str, Any], template: str, context_path: Optional[Path] = None, data: Optional[Dict[str, Any]] = None, ) -> Any: user_files = [] # type: List[Dict[str, Any]] if context_path: user_files, _ = context.read_context(context_path) return api.post( master, endpoint, body={ "config": config, "template": template, "user_files": user_files, "data": data }, ).json()
def test_read_context_with_detignore_subdirs(tmp_path: Path) -> None: with FileTree( tmp_path, { "A.py": "", "B.py": "", Path("subdir").joinpath("A.py"): "", Path("subdir").joinpath("B.py"): "", }, ) as tree: model_def, _ = context.read_context(tree) assert {f["path"] for f in model_def} == { "A.py", "B.py", "subdir", "subdir/A.py", "subdir/B.py", } with FileTree( tmp_path, { "A.py": "", "B.py": "", ".detignore": "\nA.py\n", Path("subdir").joinpath("A.py"): "", Path("subdir").joinpath("B.py"): "", }, ) as tree: model_def, size = context.read_context(tree) assert {f["path"] for f in model_def} == {"B.py", "subdir", "subdir/B.py"} with FileTree( tmp_path, { "A.py": "", "B.py": "", Path("subdir").joinpath("A.py"): "", Path("subdir").joinpath("B.py"): "", ".detignore": "\nsubdir/A.py\n", }, ) as tree: model_def, size = context.read_context(tree) assert {f["path"] for f in model_def } == {"A.py", "B.py", "subdir", "subdir/B.py"} with FileTree( tmp_path, { "A.py": "", "B.py": "", Path("subdir").joinpath("A.py"): "", Path("subdir").joinpath("B.py"): "", ".detignore": "\n*.py\n", }, ) as tree: model_def, size = context.read_context(tree) assert len(model_def) == 1 with FileTree( tmp_path, { "A.py": "", "B.py": "", "subdir/A.py": "", "subdir/B.py": "", ".detignore": "\nsubdir\n" }, ) as tree: model_def, size = context.read_context(tree) assert {f["path"] for f in model_def} == {"A.py", "B.py"} with FileTree( tmp_path, { "A.py": "", "B.py": "", "subdir/A.py": "", "subdir/B.py": "", ".detignore": "\nsubdir/\n", }, ) as tree: model_def, size = context.read_context(tree) assert {f["path"] for f in model_def} == {"A.py", "B.py"}
def test_read_context(tmp_path: Path) -> None: with FileTree(tmp_path, {"A.py": "", "B.py": "", "C.py": ""}) as tree: model_def, _ = context.read_context(tree) assert {f["path"] for f in model_def} == {"A.py", "B.py", "C.py"}