def launch_command(
master: str,
endpoint: str,
config: Dict[str, Any],
template: str,
context_path: Optional[Path] = None,
data: Optional[Dict[str, Any]] = None,
preview: Optional[bool] = False,
) -> Any:
user_files = [] # type: List[Dict[str, Any]]
if context_path:
user_files, _ = context.read_context(context_path)
body = {"config": config} # type: Dict[str, Any]
if template:
body["template_name"] = template
if len(user_files) > 0:
body["files"] = user_files
if data is not None:
message_bytes = json.dumps(data).encode("utf-8")
base64_bytes = base64.b64encode(message_bytes)
body["data"] = base64_bytes
if preview:
body["preview"] = preview
return api.post(
master,
endpoint,
body,
).json()
def test_read_context_with_detignore(tmp_path: Path) -> None:
with FileTree(tmp_path, {"A.py": "", "B.py": "", "C.py": ""}) as tree:
model_def, _ = context.read_context(tree)
assert {f["path"] for f in model_def} == {"A.py", "B.py", "C.py"}
with FileTree(tmp_path, {
"A.py": "",
"B.py": "",
"C.py": "",
".detignore": "\nA.py\n"
}) as tree:
model_def, size = context.read_context(tree)
assert {f["path"] for f in model_def} == {"B.py", "C.py"}
with FileTree(tmp_path, {
"A.py": "",
"B.py": "",
"C.py": "",
".detignore": "\n*.py\n"
}) as tree:
model_def, size = context.read_context(tree)
assert model_def == []
def test_read_context_ignore_pycaches(tmp_path: Path) -> None:
with FileTree(
tmp_path,
{
"__pycache__/A.cpython-37.pyc": "",
"A.py": "",
"subdir/A.py": "",
"subdir/__pycache__/A.cpython-37.pyc": "",
},
) as tree:
model_def, _ = context.read_context(tree)
assert {f["path"]
for f in model_def} == {"A.py", "subdir", "subdir/A.py"}
def _path_to_files(path: pathlib.Path) -> List[V1File]:
files = []
for item in context.read_context(path)[0]:
content = item["content"].decode("utf-8")
file = V1File(
path=item["path"],
type=item["type"],
content=content,
mtime=item["mtime"],
uid=item["uid"],
gid=item["gid"],
mode=item["mode"],
)
files.append(file)
return files
def start_tensorboard(args: Namespace) -> None:
if not (args.trial_ids or args.experiment_ids):
print("Either experiment_ids or trial_ids must be specified.")
sys.exit(1)
config = parse_config(args.config_file, None, args.config, [])
req_body = {
"config": config,
"trial_ids": args.trial_ids,
"experiment_ids": args.experiment_ids,
}
if args.context is not None:
req_body["files"], _ = context.read_context(args.context,
constants.MAX_CONTEXT_SIZE)
resp = api.post(args.master, "api/v1/tensorboards",
json=req_body).json()["tensorboard"]
if args.detach:
print(resp["id"])
return
url = "tensorboard/{}/events".format(resp["id"])
with api.ws(args.master, url) as ws:
for msg in ws:
if msg["log_event"] is not None:
# TensorBoard will print a url by default. The URL is incorrect since
# TensorBoard is not aware of the master proxy address it is assigned.
if "http" in msg["log_event"]:
continue
if msg["service_ready_event"]:
if args.no_browser:
url = api.make_url(args.master, resp["serviceAddress"])
else:
url = api.browser_open(args.master, resp["serviceAddress"])
print(
colored("TensorBoard is running at: {}".format(url),
"green"))
render_event_stream(msg)
break
render_event_stream(msg)
def create_experiment(
self,
config: Union[str, pathlib.Path, Dict],
model_dir: Union[str, pathlib.Path],
) -> experiment.ExperimentReference:
"""
Create an experiment with config parameters and model directory. The function
returns :class:`~determined.experimental.ExperimentReference` of the experiment.
Arguments:
config(string, pathlib.Path, dictionary): experiment config filename (.yaml)
or a dict.
model_dir(string): directory containing model definition.
"""
check.is_instance(
config, (str, pathlib.Path, dict), "config parameter must be dictionary or path"
)
if isinstance(config, str):
with open(config) as f:
experiment_config = util.safe_load_yaml_with_exceptions(f)
elif isinstance(config, pathlib.Path):
with config.open() as f:
experiment_config = util.safe_load_yaml_with_exceptions(f)
elif isinstance(config, Dict):
experiment_config = config
if isinstance(model_dir, str):
model_dir = pathlib.Path(model_dir)
model_context, _ = context.read_context(model_dir)
resp = self._session.post(
"/api/v1/experiments",
body={
"config": yaml.safe_dump(experiment_config),
"model_definition": model_context,
},
)
exp_id = _CreateExperimentResponse(resp.json()).id
exp = experiment.ExperimentReference(exp_id, self._session)
exp.activate()
return exp
def launch_command(
master: str,
endpoint: str,
config: Dict[str, Any],
template: str,
context_path: Optional[Path] = None,
data: Optional[Dict[str, Any]] = None,
) -> Any:
user_files = [] # type: List[Dict[str, Any]]
if context_path:
user_files, _ = context.read_context(context_path)
return api.post(
master,
endpoint,
body={
"config": config,
"template": template,
"user_files": user_files,
"data": data
},
).json()
def test_read_context_with_detignore_subdirs(tmp_path: Path) -> None:
with FileTree(
tmp_path,
{
"A.py": "",
"B.py": "",
Path("subdir").joinpath("A.py"): "",
Path("subdir").joinpath("B.py"): "",
},
) as tree:
model_def, _ = context.read_context(tree)
assert {f["path"]
for f in model_def} == {
"A.py",
"B.py",
"subdir",
"subdir/A.py",
"subdir/B.py",
}
with FileTree(
tmp_path,
{
"A.py": "",
"B.py": "",
".detignore": "\nA.py\n",
Path("subdir").joinpath("A.py"): "",
Path("subdir").joinpath("B.py"): "",
},
) as tree:
model_def, size = context.read_context(tree)
assert {f["path"]
for f in model_def} == {"B.py", "subdir", "subdir/B.py"}
with FileTree(
tmp_path,
{
"A.py": "",
"B.py": "",
Path("subdir").joinpath("A.py"): "",
Path("subdir").joinpath("B.py"): "",
".detignore": "\nsubdir/A.py\n",
},
) as tree:
model_def, size = context.read_context(tree)
assert {f["path"]
for f in model_def
} == {"A.py", "B.py", "subdir", "subdir/B.py"}
with FileTree(
tmp_path,
{
"A.py": "",
"B.py": "",
Path("subdir").joinpath("A.py"): "",
Path("subdir").joinpath("B.py"): "",
".detignore": "\n*.py\n",
},
) as tree:
model_def, size = context.read_context(tree)
assert len(model_def) == 1
with FileTree(
tmp_path,
{
"A.py": "",
"B.py": "",
"subdir/A.py": "",
"subdir/B.py": "",
".detignore": "\nsubdir\n"
},
) as tree:
model_def, size = context.read_context(tree)
assert {f["path"] for f in model_def} == {"A.py", "B.py"}
with FileTree(
tmp_path,
{
"A.py": "",
"B.py": "",
"subdir/A.py": "",
"subdir/B.py": "",
".detignore": "\nsubdir/\n",
},
) as tree:
model_def, size = context.read_context(tree)
assert {f["path"] for f in model_def} == {"A.py", "B.py"}
def test_read_context(tmp_path: Path) -> None:
with FileTree(tmp_path, {"A.py": "", "B.py": "", "C.py": ""}) as tree:
model_def, _ = context.read_context(tree)
assert {f["path"] for f in model_def} == {"A.py", "B.py", "C.py"}