def test_create_and_get_experiment(self):
with tempfile.TemporaryDirectory() as tempdir:
rootdir = Path(tempdir)
store = LogStore(rootdir)
experiment_id = store.create_experiment(self.experiment)
experiment = store.get_experiment(experiment_id)
assert experiment.name == self.experiment_name
def run(self, args: argparse.Namespace) -> None:
settings = Settings()
if args.config_file is not None:
settings.load(args.config_file)
store_path = args.store or settings.logstore_storepath
store = LogStore(store_path)
runinfo = store.load_run(args.run)
print(json.dumps(runinfo.to_json(), indent=2))
def run(self, args: argparse.Namespace) -> None:
settings = Settings()
if args.config_file is not None:
settings.load(args.config_file)
store_path = args.store or settings.logstore_storepath
store = LogStore(store_path)
runinfos = store.get_runs(args.experiment, args.worker)
runinfo_dicts = [x.to_json() for x in runinfos]
print(json.dumps(runinfo_dicts))
def __init__(
self,
rootdir: tp.Union[Path, str],
module: str,
execution_path: tp.Union[Path, str],
) -> None:
self._store = LogStore(Path(rootdir))
self._module = module
self._execution_path = Path(execution_path).absolute()
if execution_path is not None:
sys.path.append(str(execution_path))
import_submodules(module)
def run(self, args: argparse.Namespace) -> None:
settings = Settings()
if args.config_file is not None:
settings.load(args.config_file)
if args.store is None:
store_path = settings.logstore_storepath
else:
store_path = Path(args.store)
store = LogStore(store_path)
runinfo = store.load_run(args.run)
runinfo.name = args.name
store.save_run(runinfo)
def run(self, args: argparse.Namespace) -> None:
settings = Settings()
if args.config_file is not None:
settings.load(args.config_file)
store_path = args.store or settings.logstore_storepath
store = LogStore(store_path)
runinfo = store.load_run(args.run)
if args.force:
choise = "y"
else:
choise = input(f"Do you delete this run: {runinfo.uuid} ? [y/N] ")
if choise.strip().lower() == "y":
store.delete_run(runinfo.uuid)
def run(self, args: argparse.Namespace) -> None:
settings = Settings()
if args.config_file is not None:
settings.load(args.config_file)
store_path = args.store or settings.logstore_storepath
store = LogStore(store_path)
runinfos = store.get_runs(args.experiment, args.worker)
table = _get_runinfo_table(runinfos, args.max_column_width)
if args.sort is not None:
table.sort(args.sort, args.desc)
if args.columns is not None:
columns = args.columns.split(",")
table = table[columns]
table.print()
def run(self, args: argparse.Namespace) -> None:
settings = Settings()
if args.config_file is not None:
settings.load(args.config_file)
if args.store is None:
store_path = settings.logstore_storepath
else:
store_path = Path(args.store)
store = LogStore(store_path)
runinfo = store.load_run(args.run)
note = edit(
editor=settings.logexp_editor,
filename="note.txt",
text=runinfo.note,
)
runinfo.note = note
store.save_run(runinfo)
def run(self, args: argparse.Namespace) -> None:
settings = Settings()
if args.config_file is not None:
settings.load(args.config_file)
store_path = args.store or settings.logstore_storepath
store = LogStore(store_path)
runinfos = [
x for x in store.get_runs(args.experiment, args.worker)
if x.status == Status.FAILED or x.status == Status.INTERRUPTED
]
if args.force:
choise = "y"
else:
print("Following runs will be removed:")
for runinfo in runinfos:
print(f" {runinfo.uuid}")
choise = input("Are you sure you want to continue? [y/N] ")
if choise.strip().lower() == "y":
for runinfo in runinfos:
store.delete_run(runinfo.uuid)
def test_delete_run(self):
with tempfile.TemporaryDirectory() as tempdir:
rootdir = Path(tempdir)
store = LogStore(rootdir)
experiment_id = store.create_experiment(self.experiment)
run_id = store.create_run(experiment_id, self.worker_name)
run_info = RunInfo(
version=VERSION,
uuid=run_id,
name="test run",
module="test_module",
execution_path=Path("test/path"),
experiment_id=experiment_id,
experiment_name=self.experiment_name,
worker_name=self.worker_name,
status=Status.FINISHED,
params=Params({"test": "params"}),
report=Report({"test": "report"}),
storage=store.get_storage(experiment_id, run_id),
platform=get_platform_info(),
git=get_git_info(),
note="test note",
stdout="test stdout",
stderr="test stderr",
start_time=datetime.datetime.now(),
end_time=datetime.datetime.now(),
)
store.save_run(run_info)
assert (rootdir / str(experiment_id) / self.worker_name /
run_id).exists()
runinfos = store.delete_run(run_id)
assert not (rootdir / str(experiment_id) / self.worker_name /
run_id).exists()
class Executor:
def __init__(
self,
rootdir: tp.Union[Path, str],
module: str,
execution_path: tp.Union[Path, str],
) -> None:
self._store = LogStore(Path(rootdir))
self._module = module
self._execution_path = Path(execution_path).absolute()
if execution_path is not None:
sys.path.append(str(execution_path))
import_submodules(module)
@staticmethod
def _load_params(path: Path):
with open(path, "r") as f:
params_dict = json.load(f)
return Params.from_json(params_dict)
def _build_runinfo(
self,
experiment_id: int,
experiment: Experiment,
worker: BaseWorker,
run_id: str,
name: str,
status: Status,
params: Params,
report: Report = None,
note: str = None,
stdout: str = None,
stderr: str = None,
) -> RunInfo:
storage = self._store.get_storage(experiment_id, run_id)
gitinfo: tp.Optional[GitInfo] = None
try:
gitinfo = get_git_info()
except (error.GitCommandNotFoundError,
error.GitRepositoryNotFoundError):
gitinfo = None
runinfo = RunInfo(
version=VERSION,
uuid=run_id,
name=name,
module=self._module,
execution_path=self._execution_path,
experiment_id=experiment_id,
experiment_name=experiment.name,
worker_name=worker.name,
status=status,
params=params,
report=report,
storage=storage,
platform=get_platform_info(),
git=gitinfo,
note=note,
stdout=stdout,
stderr=stderr,
start_time=datetime.datetime.now(),
end_time=datetime.datetime.now(),
)
return runinfo
def init(
self,
experiment_name: str,
) -> int:
experiment = Experiment.get_experiment(experiment_name)
experiment_id = self._store.create_experiment(experiment)
return experiment_id
def run(
self,
experiment_id: int,
worker_name: str,
params_path: Path = None,
name: str = None,
note: str = None,
) -> RunInfo:
name = name or ""
experiment = self._store.get_experiment(experiment_id)
worker = experiment.get_worker(worker_name)
params = (self._load_params(params_path)
if params_path else worker.params)
# make sure that worker_name exists before create_run
run_id = self._store.create_run(experiment_id, worker_name)
runinfo = self._build_runinfo(
experiment_id=experiment_id,
experiment=experiment,
worker=worker,
run_id=run_id,
name=name,
status=Status.RUNNING,
params=params,
note=note,
)
self._store.save_run(runinfo)
report: tp.Optional[Report] = None
try:
with capture() as captured_out:
report = worker(
params=params,
storage=runinfo.storage,
)
except KeyboardInterrupt:
runinfo.status = Status.INTERRUPTED
except Exception as e: # pylint: disable=broad-except
runinfo.status = Status.FAILED
raise e
else:
runinfo.status = Status.FINISHED
finally:
runinfo.report = report
runinfo.stdout = captured_out["stdout"]
runinfo.stderr = captured_out["stderr"]
runinfo.end_time = datetime.datetime.now()
self._store.save_run(runinfo)
return runinfo