def __init__(self, split, datadir, exp_dict): self.split = split self.exp_dict = exp_dict self.n_classes = 3#2 if split == "train": fname = os.path.join(datadir, 'image_sets', 'training.txt') elif split == "val": fname = os.path.join(datadir, 'image_sets', 'validation.txt') elif split == "test": fname = os.path.join(datadir, 'image_sets', 'test.txt') self.img_names = [name.replace(".jpg\n","") for name in hu.read_text(fname)] self.path = os.path.join(datadir, 'images')
def get_summary(self, failed_only=False, columns=None, max_lines=200): """[summary] Returns ------- [type] [description] """ add_job_utils() import haven_jobs_utils as hju # get job ids job_id_list = [] for exp_dict in self.exp_list: exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(self.savedir_base, exp_id) fname = hju.get_job_fname(savedir) if os.path.exists(fname): job_id_list += [hu.load_json(fname)["job_id"]] jobs_dict = hju.get_jobs_dict(self.api, job_id_list) # fill summary summary_dict = {'table':[], 'status':[], 'logs_failed':[], 'logs':[]} for exp_dict in self.exp_list: result_dict = copy.deepcopy(exp_dict) exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(self.savedir_base, exp_id) result_dict["exp_id"] = exp_id fname = hju.get_job_fname(savedir) # Job results result_dict["job_id"] = None result_dict["job_state"] = 'NEVER LAUNCHED' if os.path.exists(fname): job_dict = hu.load_json(fname) job_id = job_dict["job_id"] if job_id not in jobs_dict: continue job = jobs_dict[job_id] result_dict['started at (Montreal)'] = job_dict["started at (Montreal)"] result_dict["job_id"] = job_id result_dict["job_state"] = job.state summary_dict['table'] += [copy.deepcopy(result_dict)] result_dict["command"] = job.command[2] if job.state == "FAILED": fname = os.path.join(savedir, "err.txt") if os.path.exists(fname): result_dict["logs"] = hu.read_text(fname)[-max_lines:] summary_dict['logs_failed'] += [result_dict] else: if self.verbose: print('%s: err.txt does not exist' % exp_id) else: fname = os.path.join(savedir, "logs.txt") if os.path.exists(fname): result_dict["logs"] = hu.read_text(fname)[-max_lines:] summary_dict['logs'] += [result_dict] else: if self.verbose: print('%s: logs.txt does not exist' % exp_id) else: result_dict['job_state'] = 'NEVER LAUNCHED' summary_dict['table'] += [copy.deepcopy(result_dict)] # get info df = pd.DataFrame(summary_dict['table']) df = df.set_index('exp_id') if columns: df = df[[c for c in columns if (c in df.columns and c not in ['err'])]] if "job_state" in df: stats = np.vstack(np.unique(df['job_state'].fillna("NaN"),return_counts=True)).T status = ([{a:b} for (a,b) in stats]) else: df['job_state'] = None summary_dict['status'] = status summary_dict['table'] = df summary_dict['queuing'] = df[df['job_state']=='QUEUING'] summary_dict['running'] = df[df['job_state']=='RUNNING'] summary_dict['succeeded'] = df[df['job_state']=='SUCCEEDED'] summary_dict['failed'] = df[df['job_state']=='FAILED'] return summary_dict
def get_summary_list(self, failed_only=False, columns=None, max_lines=10, wrap_size=8, add_prefix=False, get_logs=True, exp_list=None, savedir_base=None): savedir_base = savedir_base or self.savedir_base exp_list = exp_list or self.exp_list # get job key job_id_list = [] for exp_dict in exp_list: exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(savedir_base, exp_id) fname = get_job_fname(savedir) if os.path.exists(fname): job_id_list += [hu.load_json(fname)["job_id"]] jobs_dict = self.get_jobs_dict(job_id_list) # get summaries summary_list = [] for exp_dict in exp_list: result_dict = {} exp_id = hu.hash_dict(exp_dict) savedir = os.path.join(savedir_base, exp_id) job_fname = get_job_fname(savedir) # General info result_dict = {} result_dict['exp_dict'] = exp_dict result_dict["exp_id"] = exp_id result_dict["job_id"] = None result_dict["job_state"] = 'NEVER LAUNCHED' if os.path.exists(job_fname): job_dict = hu.load_json(job_fname) job_id = job_dict["job_id"] if job_id not in jobs_dict: continue fname_exp_dict = os.path.join(savedir, "exp_dict.json") job = jobs_dict[job_id] if hasattr(job_dict, 'command'): command = job_dict['command'] else: command = None # Job info result_dict['started_at'] = hu.time_to_montreal(fname_exp_dict) result_dict["job_id"] = job_id result_dict["job_state"] = job["state"] result_dict["restarts"] = len(job["runs"]) result_dict["command"] = command if get_logs: # Logs info if job["state"] == "FAILED": logs_fname = os.path.join(savedir, "err.txt") else: logs_fname = os.path.join(savedir, "logs.txt") if os.path.exists(logs_fname): result_dict["logs"] = hu.read_text( logs_fname)[-max_lines:] summary_list += [result_dict] return summary_list