def _build_job_meta(cls, job_dir):
"""Build meta file for job.
Args:
job_dir (str): Directory path of the job.
Return:
A dict of job meta info.
"""
meta_file = os.path.join(job_dir, JOB_META_FILE)
meta = parse_json(meta_file)
if not meta:
job_name = job_dir.split('/')[-1]
user = os.environ.get("USER", None)
meta = {
"job_id": job_name,
"job_name": job_name,
"user": user,
"type": "ray",
"start_time": os.path.getctime(job_dir),
"end_time": None,
"best_trial_id": None,
}
if meta.get("start_time", None):
meta["start_time"] = timestamp2date(meta["start_time"])
return meta
def _build_job_meta(cls, job_dir):
"""Build meta file for job.
Args:
job_dir: Directory path of the job.
Return:
A dict of job meta info.
"""
meta_file = os.path.join(job_dir, JOB_META_FILE)
meta = parse_json(meta_file)
if not meta:
job_name = job_dir.split("/")[-1]
user = os.environ.get("USER", None)
meta = {
"job_id": job_name,
"job_name": job_name,
"user": user,
"type": "ray",
"start_time": os.path.getctime(job_dir),
"end_time": None,
"best_trial_id": None,
}
if meta.get("start_time", None):
meta["start_time"] = timestamp2date(meta["start_time"])
return meta
def _build_trial_meta(cls, expr_dir):
"""Build meta file for trial.
Args:
expr_dir: Directory path of the experiment.
Return:
A dict of trial meta info.
"""
meta_file = os.path.join(expr_dir, EXPR_META_FILE)
meta = parse_json(meta_file)
if not meta:
job_id = expr_dir.split("/")[-2]
trial_id = expr_dir[-8:]
params = parse_json(os.path.join(expr_dir, EXPR_PARAM_FILE))
meta = {
"trial_id": trial_id,
"job_id": job_id,
"status": "RUNNING",
"type": "TUNE",
"start_time": os.path.getctime(expr_dir),
"end_time": None,
"progress_offset": 0,
"result_offset": 0,
"params": params,
}
if not meta.get("start_time", None):
meta["start_time"] = os.path.getctime(expr_dir)
if isinstance(meta["start_time"], float):
meta["start_time"] = timestamp2date(meta["start_time"])
if meta.get("end_time", None):
meta["end_time"] = timestamp2date(meta["end_time"])
meta["params"] = parse_json(os.path.join(expr_dir, EXPR_PARAM_FILE))
return meta
def _build_trial_meta(cls, expr_dir):
"""Build meta file for trial.
Args:
expr_dir (str): Directory path of the experiment.
Return:
A dict of trial meta info.
"""
meta_file = os.path.join(expr_dir, EXPR_META_FILE)
meta = parse_json(meta_file)
if not meta:
job_id = expr_dir.split('/')[-2]
trial_id = expr_dir[-8:]
params = parse_json(os.path.join(expr_dir, EXPR_PARARM_FILE))
meta = {
"trial_id": trial_id,
"job_id": job_id,
"status": "RUNNING",
"type": "TUNE",
"start_time": os.path.getctime(expr_dir),
"end_time": None,
"progress_offset": 0,
"result_offset": 0,
"params": params
}
if not meta.get("start_time", None):
meta["start_time"] = os.path.getctime(expr_dir)
if isinstance(meta["start_time"], float):
meta["start_time"] = timestamp2date(meta["start_time"])
if meta.get("end_time", None):
meta["end_time"] = timestamp2date(meta["end_time"])
meta["params"] = parse_json(os.path.join(expr_dir, EXPR_PARARM_FILE))
return meta
def _update_job_info(cls, job_dir):
"""Update information for given job.
Meta file will be loaded if exists, and the job information in
in db backend will be updated.
Args:
job_dir: Directory path of the job.
Return:
Updated dict of job meta info
"""
meta_file = os.path.join(job_dir, JOB_META_FILE)
meta = parse_json(meta_file)
if meta:
logging.debug("Update job info for %s" % meta["job_id"])
JobRecord.objects.filter(job_id=meta["job_id"]).update(
end_time=timestamp2date(meta["end_time"]))
def _update_job_info(cls, job_dir):
"""Update information for given job.
Meta file will be loaded if exists, and the job information in
in db backend will be updated.
Args:
job_dir (str): Directory path of the job.
Return:
Updated dict of job meta info
"""
meta_file = os.path.join(job_dir, JOB_META_FILE)
meta = parse_json(meta_file)
if meta:
logging.debug("Update job info for %s" % meta["job_id"])
JobRecord.objects \
.filter(job_id=meta["job_id"]) \
.update(end_time=timestamp2date(meta["end_time"]))
def _update_trial_info(self, expr_dir):
"""Update information for given trial.
Meta file will be loaded if exists, and the trial information
in db backend will be updated.
Args:
expr_dir(str)
"""
trial_id = expr_dir[-8:]
meta_file = os.path.join(expr_dir, EXPR_META_FILE)
meta = parse_json(meta_file)
result_file = os.path.join(expr_dir, EXPR_RESULT_FILE)
offset = self._result_offsets.get(trial_id, 0)
results, new_offset = parse_multiple_json(result_file, offset)
self._add_results(results, trial_id)
self._result_offsets[trial_id] = new_offset
if meta:
TrialRecord.objects \
.filter(trial_id=trial_id) \
.update(trial_status=meta["status"],
end_time=timestamp2date(meta.get("end_time", None)))
elif len(results) > 0:
metrics = {
"episode_reward": results[-1].get("episode_reward_mean", None),
"accuracy": results[-1].get("mean_accuracy", None),
"loss": results[-1].get("loss", None)
}
if results[-1].get("done"):
TrialRecord.objects \
.filter(trial_id=trial_id) \
.update(trial_status="TERMINATED",
end_time=results[-1].get("date", None),
metrics=str(metrics))
else:
TrialRecord.objects \
.filter(trial_id=trial_id) \
.update(metrics=str(metrics))
def _update_trial_info(self, expr_dir):
"""Update information for given trial.
Meta file will be loaded if exists, and the trial information
in db backend will be updated.
Args:
expr_dir(str)
"""
trial_id = expr_dir[-8:]
meta_file = os.path.join(expr_dir, EXPR_META_FILE)
meta = parse_json(meta_file)
result_file = os.path.join(expr_dir, EXPR_RESULT_FILE)
offset = self._result_offsets.get(trial_id, 0)
results, new_offset = parse_multiple_json(result_file, offset)
self._add_results(results, trial_id)
self._result_offsets[trial_id] = new_offset
if meta:
TrialRecord.objects \
.filter(trial_id=trial_id) \
.update(trial_status=meta["status"],
end_time=timestamp2date(meta.get("end_time", None)))
elif len(results) > 0:
metrics = {
"episode_reward": results[-1].get("episode_reward_mean", None),
"accuracy": results[-1].get("mean_accuracy", None),
"loss": results[-1].get("loss", None)
}
if results[-1].get("done"):
TrialRecord.objects \
.filter(trial_id=trial_id) \
.update(trial_status="TERMINATED",
end_time=results[-1].get("date", None),
metrics=str(metrics))
else:
TrialRecord.objects \
.filter(trial_id=trial_id) \
.update(metrics=str(metrics))
