def list_versions(args: Namespace) -> None:
if args.json:
r = api.get(args.master, "models/{}/versions".format(args.name))
data = r.json()
print(json.dumps(data, indent=2))
else:
model = Determined(args.master).get_model(args.name)
render_model(model)
print("\n")
headers = [
"Version #",
"Trial ID",
"Batch #",
"Checkpoint UUID",
"Validation Metrics",
"Metadata",
]
values = [
[
ckpt.model_version,
ckpt.trial_id,
ckpt.batch_number,
ckpt.uuid,
json.dumps(ckpt.validation, indent=2),
json.dumps(ckpt.metadata, indent=2),
]
for ckpt in model.get_versions()
]
render.tabulate_or_csv(headers, values, False)
def create(args: Namespace) -> None:
model = Determined(args.master, None).create_model(args.name, args.description)
if args.json:
print(json.dumps(model.to_json(), indent=2))
else:
render_model(model)
def download(args: Namespace) -> None:
checkpoint = Determined(args.master, None).get_checkpoint(args.uuid)
path = checkpoint.download(path=args.output_dir)
if args.quiet:
print(path)
else:
render_checkpoint(checkpoint, path)
def my_experiment(cfg: DictConfig) -> None:
config = OmegaConf.to_container(cfg, resolve=True)
# We use a helper function now to check for missing values.
# In the next version of omegaconf, we will be able to check for missing values by
# passing throw_on_missing to the OmegaConf.to_container call above.
check_for_missing(config)
master = Determined()
exp = master.create_experiment(config, CONTEXT_DIR)
exp.activate()
def describe(args: Namespace) -> None:
model = Determined(args.master, None).get_model(args.name)
checkpoint = model.get_version(args.version)
if args.json:
print(json.dumps(model.to_json(), indent=2))
else:
render_model(model)
if checkpoint is not None:
print("\n")
render_model_version(checkpoint)
def download(args: Namespace) -> None:
exp = Determined(args.master, args.user).get_experiment(args.experiment_id)
checkpoints = exp.top_n_checkpoints(
args.top_n, sort_by=args.sort_by, smaller_is_better=args.smaller_is_better
)
top_level = pathlib.Path(args.output_dir)
top_level.mkdir(parents=True, exist_ok=True)
for ckpt in checkpoints:
path = ckpt.download(str(top_level.joinpath(ckpt.uuid)))
if args.quiet:
print(path)
else:
render_checkpoint(ckpt, path)
print()
def register_version(args: Namespace) -> None:
if args.json:
resp = api.post(
args.master,
"/api/v1/models/{}/versions".format(args.name),
body={"checkpoint_uuid": args.uuid},
)
print(json.dumps(resp.json(), indent=2))
else:
model = Determined(args.master, None).get_model(args.name)
checkpoint = model.register_version(args.uuid)
render_model(model)
print("\n")
render_model_version(checkpoint)
def list_models(args: Namespace) -> None:
models = Determined(args.master, None).get_models(
sort_by=ModelSortBy[args.sort_by.upper()], order_by=ModelOrderBy[args.order_by.upper()]
)
if args.json:
print(json.dumps([m.to_json() for m in models], indent=2))
else:
headers = ["Name", "Creation Time", "Last Updated Time", "Metadata"]
values = [
[m.name, m.creation_time, m.last_updated_time, json.dumps(m.metadata or {}, indent=2)]
for m in models
]
render.tabulate_or_csv(headers, values, False)
def download(args: Namespace) -> None:
checkpoint = (Determined(args.master,
None).get_trial(args.trial_id).select_checkpoint(
latest=args.latest,
best=args.best,
uuid=args.uuid,
sort_by=args.sort_by,
smaller_is_better=args.smaller_is_better,
))
path = checkpoint.download(path=args.output_dir)
if args.quiet:
print(path)
else:
render_checkpoint(checkpoint, path)
def describe(args: Namespace) -> None:
checkpoint = Determined(args.master, None).get_checkpoint(args.uuid)
render_checkpoint(checkpoint)
def export_model(experiment_id: int, master_url: str) -> tf.keras.Model:
checkpoint = (
Determined(master=master_url).get_experiment(experiment_id).top_checkpoint()
)
model = checkpoint.load()
return model