def __init__(self, split, datadir, exp_dict):
self.split = split
self.exp_dict = exp_dict
self.n_classes = 3#2
if split == "train":
fname = os.path.join(datadir, 'image_sets', 'training.txt')
elif split == "val":
fname = os.path.join(datadir, 'image_sets', 'validation.txt')
elif split == "test":
fname = os.path.join(datadir, 'image_sets', 'test.txt')
self.img_names = [name.replace(".jpg\n","") for name in hu.read_text(fname)]
self.path = os.path.join(datadir, 'images')
def get_summary(self, failed_only=False, columns=None, max_lines=200):
"""[summary]
Returns
-------
[type]
[description]
"""
add_job_utils()
import haven_jobs_utils as hju
# get job ids
job_id_list = []
for exp_dict in self.exp_list:
exp_id = hu.hash_dict(exp_dict)
savedir = os.path.join(self.savedir_base, exp_id)
fname = hju.get_job_fname(savedir)
if os.path.exists(fname):
job_id_list += [hu.load_json(fname)["job_id"]]
jobs_dict = hju.get_jobs_dict(self.api, job_id_list)
# fill summary
summary_dict = {'table':[], 'status':[], 'logs_failed':[], 'logs':[]}
for exp_dict in self.exp_list:
result_dict = copy.deepcopy(exp_dict)
exp_id = hu.hash_dict(exp_dict)
savedir = os.path.join(self.savedir_base, exp_id)
result_dict["exp_id"] = exp_id
fname = hju.get_job_fname(savedir)
# Job results
result_dict["job_id"] = None
result_dict["job_state"] = 'NEVER LAUNCHED'
if os.path.exists(fname):
job_dict = hu.load_json(fname)
job_id = job_dict["job_id"]
if job_id not in jobs_dict:
continue
job = jobs_dict[job_id]
result_dict['started at (Montreal)'] = job_dict["started at (Montreal)"]
result_dict["job_id"] = job_id
result_dict["job_state"] = job.state
summary_dict['table'] += [copy.deepcopy(result_dict)]
result_dict["command"] = job.command[2]
if job.state == "FAILED":
fname = os.path.join(savedir, "err.txt")
if os.path.exists(fname):
result_dict["logs"] = hu.read_text(fname)[-max_lines:]
summary_dict['logs_failed'] += [result_dict]
else:
if self.verbose:
print('%s: err.txt does not exist' % exp_id)
else:
fname = os.path.join(savedir, "logs.txt")
if os.path.exists(fname):
result_dict["logs"] = hu.read_text(fname)[-max_lines:]
summary_dict['logs'] += [result_dict]
else:
if self.verbose:
print('%s: logs.txt does not exist' % exp_id)
else:
result_dict['job_state'] = 'NEVER LAUNCHED'
summary_dict['table'] += [copy.deepcopy(result_dict)]
# get info
df = pd.DataFrame(summary_dict['table'])
df = df.set_index('exp_id')
if columns:
df = df[[c for c in columns if (c in df.columns and c not in ['err'])]]
if "job_state" in df:
stats = np.vstack(np.unique(df['job_state'].fillna("NaN"),return_counts=True)).T
status = ([{a:b} for (a,b) in stats])
else:
df['job_state'] = None
summary_dict['status'] = status
summary_dict['table'] = df
summary_dict['queuing'] = df[df['job_state']=='QUEUING']
summary_dict['running'] = df[df['job_state']=='RUNNING']
summary_dict['succeeded'] = df[df['job_state']=='SUCCEEDED']
summary_dict['failed'] = df[df['job_state']=='FAILED']
return summary_dict
def get_summary_list(self,
failed_only=False,
columns=None,
max_lines=10,
wrap_size=8,
add_prefix=False,
get_logs=True,
exp_list=None,
savedir_base=None):
savedir_base = savedir_base or self.savedir_base
exp_list = exp_list or self.exp_list
# get job key
job_id_list = []
for exp_dict in exp_list:
exp_id = hu.hash_dict(exp_dict)
savedir = os.path.join(savedir_base, exp_id)
fname = get_job_fname(savedir)
if os.path.exists(fname):
job_id_list += [hu.load_json(fname)["job_id"]]
jobs_dict = self.get_jobs_dict(job_id_list)
# get summaries
summary_list = []
for exp_dict in exp_list:
result_dict = {}
exp_id = hu.hash_dict(exp_dict)
savedir = os.path.join(savedir_base, exp_id)
job_fname = get_job_fname(savedir)
# General info
result_dict = {}
result_dict['exp_dict'] = exp_dict
result_dict["exp_id"] = exp_id
result_dict["job_id"] = None
result_dict["job_state"] = 'NEVER LAUNCHED'
if os.path.exists(job_fname):
job_dict = hu.load_json(job_fname)
job_id = job_dict["job_id"]
if job_id not in jobs_dict:
continue
fname_exp_dict = os.path.join(savedir, "exp_dict.json")
job = jobs_dict[job_id]
if hasattr(job_dict, 'command'):
command = job_dict['command']
else:
command = None
# Job info
result_dict['started_at'] = hu.time_to_montreal(fname_exp_dict)
result_dict["job_id"] = job_id
result_dict["job_state"] = job["state"]
result_dict["restarts"] = len(job["runs"])
result_dict["command"] = command
if get_logs:
# Logs info
if job["state"] == "FAILED":
logs_fname = os.path.join(savedir, "err.txt")
else:
logs_fname = os.path.join(savedir, "logs.txt")
if os.path.exists(logs_fname):
result_dict["logs"] = hu.read_text(
logs_fname)[-max_lines:]
summary_list += [result_dict]
return summary_list