# ---------------------------------
print_header("test_env_to_path")
assert env_to_path("$HOME") == os.environ["HOME"]
assert env_to_path("$HOME/") == os.environ["HOME"] + "/"
assert env_to_path("$HOME/Documents") == str(
Path(os.environ["HOME"]) / "Documents")
print("ok.")
# ----------------------------------------
# ----- Testing get_increased_path -----
# ----------------------------------------
print_header("test_get_increased_path")
uid = str(uuid.uuid4())
p = Path() / uid
p.mkdir()
get_increased_path(p).mkdir()
get_increased_path(p).mkdir()
get_increased_path(p).mkdir()
paths = {str(d) for d in Path().glob(uid + "*")}
target = {str(p), str(p) + " (1)", str(p) + " (2)", str(p) + " (3)"}
assert paths == target
print("ok.")
for d in Path().glob(uid + "*"):
d.rmdir()
# ----------------------------------
# ----- Testing flatten_opts -----
# ----------------------------------
print_header("test_flatten_opts")
d = addict.Dict()
d.a.b.c = 2
def main(opts):
# -----------------------------
# ----- Parse arguments -----
# -----------------------------
opts = Dict(OmegaConf.to_container(opts))
args = opts.args
# -----------------------
# ----- Load opts -----
# -----------------------
opts = load_opts(args.config, default=opts)
if args.resume:
opts.train.resume = True
opts.output_path = env_to_path(opts.output_path)
if not opts.train.resume:
opts.output_path = get_increased_path(opts.output_path)
pprint("Running model in", opts.output_path)
exp = None
if not args.dev:
# -------------------------------
# ----- Check output_path -----
# -------------------------------
if opts.train.resume:
Path(opts.output_path).mkdir(exist_ok=True)
else:
assert not Path(opts.output_path).exists()
Path(opts.output_path).mkdir()
# Save config file
# TODO what if resuming? re-dump?
with Path(opts.output_path / "opts.yaml").open("w") as f:
yaml.safe_dump(opts.to_dict())
if not args.no_comet:
# ----------------------------------
# ----- Set Comet Experiment -----
# ----------------------------------
exp = Experiment(project_name="omnigan", auto_metric_logging=False)
exp.log_parameters(flatten_opts(opts))
if args.note:
exp.log_parameter("note", args.note)
with open(Path(opts.output_path) / "comet_url.txt", "w") as f:
f.write(exp.url)
else:
# ----------------------
# ----- Dev Mode -----
# ----------------------
pprint("> /!\ Development mode ON")
print("Cropping data to 32")
opts.data.transforms += [
Dict({
"name": "crop",
"ignore": False,
"height": 32,
"width": 32
})
]
# -------------------
# ----- Train -----
# -------------------
trainer = Trainer(opts, comet_exp=exp)
trainer.logger.time.start_time = time()
trainer.setup()
trainer.train()
# -----------------------------
# ----- End of training -----
# -----------------------------
pprint("Done training")
)
return parser.parse_args()
if __name__ == "__main__":
args = parsed_args()
assert Path(args.experiment).exists()
assert Path(args.template).exists()
xopts = load_exp(args.experiment)
for i in range(len(xopts.runs)):
# setup run path
p = get_increased_path(xopts.runs[i].trainer.output_path)
xopts.runs[i].trainer.output_path = str(p)
p.mkdir(parents=True)
# write data files
with open(p / "config.yaml", "w") as f:
yaml.safe_dump(xopts.runs[i].trainer.to_dict(), f)
copyfile(args.experiment, p / f"exp_{i}.yaml")
write_run_template(xopts, i, args.template, p / "exp.sh")
write_hash(p / "hash.txt")
# launch sbatch job
if not xopts.experiment.dev_mode:
print(
subprocess.check_output(f"sbatch { str(p / 'exp.sh')}",
shell=True))