def show_ensemble_results(self, files=None, unc=True, unc_metric=None, metric_name='dice_score'): assert self.df_ens is not None, "Please run `get_ensemble_results` first." df = self.df_ens if files is not None: df = df.reset_index().set_index('file', drop=False).loc[files] for _, r in df.iterrows(): imgs = [] imgs.append(_read_img(r.image_path)[:]) if metric_name in r.index: try: msk = self.ds.labels[r.file][:] except: msk = _read_msk(r.mask_path, n_classes=self.n_classes, instance_labels=self.instance_labels) imgs.append(msk) hastarget = True else: hastarget = False imgs.append( np.argmax(zarr.load(r.softmax_path), axis=-1).astype('uint8')) if unc: imgs.append(zarr.load(r.uncertainty_path)) plot_results(*imgs, df=r, hastarget=hastarget, metric_name=metric_name, unc_metric=unc_metric)
def export_imagej_rois(self, output_folder='ROI_sets', **kwargs): assert self.df_ens is not None, "Please run prediction first." output_folder = Path(output_folder) output_folder.mkdir(exist_ok=True, parents=True) for idx, r in progress_bar(self.df_ens.iterrows(), total=len(self.df_ens)): mask = np.argmax(zarr.load(r.softmax_path), axis=-1).astype('uint8') uncertainty = zarr.load(r.uncertainty_path) export_roi_set(mask, uncertainty, name=r.file, path=output_folder, ascending=False, **kwargs)
def show_cellpose_results(self, files=None, unc=True, unc_metric=None, metric_name='mean_average_precision'): assert self.df_ens is not None, "Please run `get_ensemble_results` first." df = self.df_ens.reset_index() if files is not None: df = df.set_index('file', drop=False).loc[files] for _, r in df.iterrows(): imgs = [] imgs.append(_read_img(r.image_path)[:]) if metric_name in r.index: try: mask = self.ds.labels[idx][:] except: mask = _read_msk(r.mask_path, n_classes=self.n_classes, instance_labels=self.instance_labels) _, comps = cv2.connectedComponents( (mask == self.cellpose_export_class).astype('uint8'), connectivity=4) imgs.append(label2rgb(comps, bg_label=0)) hastarget = True else: hastarget = False imgs.append(label2rgb(self.cellpose_masks[r['index']], bg_label=0)) if unc: imgs.append(zarr.load(r.uncertainty_path)) plot_results(*imgs, df=r, hastarget=hastarget, metric_name=metric_name, unc_metric=unc_metric)
def get_cellpose_results(self, export_dir=None): assert self.df_ens is not None, "Please run `get_ensemble_results` first." cl = self.cellpose_export_class assert cl < self.n_classes, f'{cl} not avaialable from {self.n_classes} classes' smxs, preds = [], [] for _, r in self.df_ens.iterrows(): softmax = zarr.load(r.softmax_path) smxs.append(softmax) preds.append(np.argmax(softmax, axis=-1).astype('uint8')) probs = [x[..., cl] for x in smxs] masks = [x == cl for x in preds] cp_masks = run_cellpose(probs, masks, model_type=self.cellpose_model, diameter=self.cellpose_diameter, min_size=self.min_pixel_export, gpu=torch.cuda.is_available()) if export_dir: export_dir = Path(export_dir) cp_path = export_dir / 'cellpose_masks' cp_path.mkdir(parents=True, exist_ok=True) for idx, r in self.df_ens.iterrows(): tifffile.imwrite(cp_path / f'{r.file}_class{cl}.tif', cp_masks[idx], compress=6) self.cellpose_masks = cp_masks return cp_masks
def to_pandas(self, start=0, end=None, chunk_size=None, max_chunks=None): start = start or self.start end = end or self.end chunk_size = chunk_size or self.chunk_size max_chunks = max_chunks or self.max_chunks if not chunk_size or not max_chunks: # One shot load, suitable for small zarr files df = zarr.load(self.root.joinpath(self.zarr_path).as_posix()).get( self.field) df = df[start:end] df = map(self.parse, df) else: # Chunked load, suitable for large zarr files df = [] with zarr.open(self.root.joinpath(self.zarr_path).as_posix(), "r") as zf: end = start + max_chunks * chunk_size if end is None else min( end, start + max_chunks * chunk_size) for i_start in range(start, end, chunk_size): items = zf[self.field][i_start:min(i_start + chunk_size, end)] items = map(self.parse, items) df.append(items) df = it.chain(*df) df = pd.DataFrame.from_records(df) for col, col_dtype in self.dtypes.items(): df[col] = df[col].astype(col_dtype, copy=False) return df
def show_valid_results(self, model_no=None, files=None, **kwargs): if self.df_val is None: self.get_valid_results(**kwargs) df = self.df_val if files is not None: df = df.set_index('file', drop=False).loc[files] if model_no is not None: df = df[df.model_no == model_no] for _, r in df.iterrows(): img = self.ds.get_data(r.img_path)[0][:] msk = self.ds.get_data(r.img_path, mask=True)[0] pred = zarr.load(r.pred_path) std = zarr.load(r.std_path) _d_model = f'Model {r.model_no}' if self.tta: plot_results(img, msk, pred, std, df=r, model=_d_model) else: plot_results(img, msk, pred, np.zeros_like(pred), df=r, model=_d_model)
def score_ensemble_results(self, mask_dir=None, label_fn=None): if not label_fn: label_fn = get_label_fn(self.df_ens.img_path[0], self.path / mask_dir) for idx, r in self.df_ens.iterrows(): msk_path = self.label_fn(r.img_path) msk = _read_msk(msk_path) self.df_ens.loc[idx, 'msk_path'] = msk_path pred = zarr.load(r.pred_path) self.df_ens.loc[idx, 'iou'] = iou(msk, pred) return self.df_ens
def get_mask(self, index): imgid = self.ids[index] split = 'training' if not is_testing_split(self.split) else 'testing' imginfo = self.get_img_info(index) width = imginfo['width'] height = imginfo['height'] if split == 'training': mask = zarr.load( os.path.join(self.root, 'object', split, self.shape_prior_base, 'mask_2', imgid + '.zarr')) != 0 mask = SegmentationMask(mask, (width, height), mode='mask') else: mask = SegmentationMask(np.zeros((height, width)), (width, height), mode='mask') return mask
def show_ensemble_results(self, files=None, model_no=None, unc=True, unc_metric=None): if self.df_ens is None: assert print("Please run `get_ensemble_results` first.") if model_no is None: df = self.df_ens else: df = self.df_models[df_models.model_no == model_no] if files is not None: df = df.set_index('file', drop=False).loc[files] for _, r in df.iterrows(): imgs = [] imgs.append(_read_img(r.img_path)[:]) if 'iou' in r.index: imgs.append(_read_msk(r.msk_path)) hastarget = True else: hastarget = False imgs.append(zarr.load(r.pred_path)) if unc: imgs.append(zarr.load(r.std_path)) plot_results(*imgs, df=r, hastarget=hastarget, unc_metric=unc_metric)
def export_cellpose_rois(self, output_folder='cellpose_ROI_sets', **kwargs): output_folder = Path(output_folder) output_folder.mkdir(exist_ok=True, parents=True) for idx, r in progress_bar(self.df_ens.iterrows(), total=len(self.df_ens)): mask = self.cellpose_masks[idx] uncertainty = zarr.load(r.uncertainty_path) export_roi_set(mask, uncertainty, instance_labels=True, name=r.file, path=output_folder, ascending=False, **kwargs)
def compress_dataset(self): """ Loads the uncompressed data file, select all sample until the index specified in "startIdx.txt", compresses it and exports it :return: """ data = zarr.load(self.crazyara_binary_dir + "data_" + self.device_name + ".zarr") export_dir, time_stamp = self.create_export_dir() zarr_path = export_dir + time_stamp + ".zip" nan_detected = compress_zarr_dataset(data, zarr_path, start_idx=0) if nan_detected is True: logging.error("NaN value detected in file %s.zip" % time_stamp) new_export_dir = self.crazyara_binary_dir + time_stamp os.rename(export_dir, new_export_dir) export_dir = new_export_dir self._move_game_data_to_export_dir(export_dir)
def score_ensemble_results(self, mask_dir=None, label_fn=None): if mask_dir is not None and label_fn is None: label_fn = get_label_fn(self.df_ens.image_path[0], self.path / mask_dir) for i, r in self.df_ens.iterrows(): if label_fn is not None: msk_path = self.label_fn(r.image_path) msk = _read_msk(msk_path, n_classes=self.n_classes, instance_labels=self.instance_labels) self.df_ens.loc[i, 'mask_path'] = msk_path else: msk = self.ds.labels[r.file][:] pred = np.argmax(zarr.load(r.softmax_path), axis=-1).astype('uint8') self.df_ens.loc[i, 'dice_score'] = dice_score(msk, pred) return self.df_ens
def compress_dataset(self, device_name: str): """ Loads the uncompressed data file, selects all sample until the index specified in "startIdx.txt", compresses it and exports it. :param device_name: The currently active device name (context_device-id) :return: """ data = zarr.load(self.binary_dir + "data_" + device_name + ".zarr") export_dir, time_stamp = self.create_export_dir(device_name) zarr_path = export_dir + time_stamp + ".zip" nan_detected = compress_zarr_dataset(data, zarr_path, start_idx=0) if nan_detected is True: logging.error("NaN value detected in file %s.zip" % time_stamp) new_export_dir = self.binary_dir + time_stamp os.rename(export_dir, new_export_dir) export_dir = new_export_dir self.move_game_data_to_export_dir(export_dir, device_name)
def get_kins_mask(self, index, len=None): imgid = self.ids[index] # split = 'training' if self.split != 'test' else 'testing' split = 'training' if not is_testing_split(self.split) else 'testing' imginfo = self.get_img_info(index) width = imginfo['width'] height = imginfo['height'] # try: p = os.path.join(self.root, 'object', split, 'kins_mask_2', imgid + '.zarr') if split == 'training' and os.path.exists(p): mask = zarr.load(p) != 0 mask = SegmentationMask(torch.tensor(mask).byte(), (width, height), mode='mask') # else: # raise Exception() # except Exception as e: else: mask = torch.ones((len, height, width)) mask = SegmentationMask(mask.byte(), (width, height), mode='mask') return mask
def init(data_dir=None): """Must be called first to specify paths to data Args: data_dir(str): Path to the diretory hosting the training data. """ # initiate global variables global LC_path, LC_zarr, cat_path, train_cat, meta_data, valid_IDs # if on sciserver, nothing is passed, then assign path to default if data_dir == None: data_dir = ( "/home/idies/workspace/Temporary/ywx649999311/LSST_AGN/Class_Training/Data/" ) LC_path = os.path.join(data_dir, "LCs.zarr.zip") LC_zarr = zarr.load(LC_path) cat_path = os.path.join(data_dir, "AllMasters.parquet") meta_data = yaml.safe_load(open(os.path.join(data_dir, "meta.yaml"))) # get train_df and assign to global variable train_cat = _get_train_cat() valid_IDs = _get_valid_ids()
def test_zarr(selenium): import numpy as np import zarr from numcodecs import Blosc # basic test z = zarr.zeros((1000, 1000), chunks=(100, 100), dtype="i4") assert z.shape == (1000, 1000) # test assignment z[0, :] = np.arange(1000) assert z[0, 1] == 1 # test saving and loading a1 = np.arange(10) zarr.save("/tmp/example.zarr", a1) a2 = zarr.load("/tmp/example.zarr") np.testing.assert_equal(a1, a2) # test compressor compressor = Blosc(cname="zstd", clevel=3, shuffle=Blosc.BITSHUFFLE) data = np.arange(10000, dtype="i4").reshape(100, 100) z = zarr.array(data, chunks=(10, 10), compressor=compressor) assert z.compressor == compressor
def get_left_img(self, index): leftimg = zarr.load(osp.join(self.leftimgdir, str(index) + '.zarr')) leftimg = torch.from_numpy(leftimg) return leftimg
def load_zarr(path): """ Utility to load the file from either dir that pytest might be called from. If called from root the path will be different than in the test dir """ return zarr.load(str(dir_.joinpath(path)))
def get_disparity(self, index): disparity = torch.from_numpy(zarr.load(osp.join(self.disparitydir, str(index) + '.zarr'))) return disparity
def get_right_img(self, index): rightimg = zarr.load(osp.join(self.rightimgdir, str(index) + '.zarr')) rightimg = torch.from_numpy(rightimg) return rightimg
def _5(obj : ConstraintFormat) -> np.ndarray : # for C in regress return zarr.load(str(obj))
help="Output a video, video frames or a plot.") return parser.parse_args() if __name__ == "__main__": args = parse_args() config = getConfiguration(args.config) xambgfile = config['range_doppler_map_fname'] if config['range_doppler_map_ftype'] == 'hdf5': f = h5py.File(xambgfile, 'r') xambg = np.abs(f['/xambg']) f.close() else: xambg = np.abs(zarr.load(xambgfile)) print("Loaded range-doppler maps.") Nframes = xambg.shape[2] print("Applying CFAR filter...") # CFAR filter each frame using a 2D kernel CF = np.zeros(xambg.shape) for i in tqdm(range(Nframes)): CF[:, :, i] = CFAR_2D(xambg[:, :, i], 18, 4) print("Applying Kalman Filter...") history = simple_target_tracker(CF, config['max_range_actual'], config['max_doppler_actual']) estimate = history['estimate'] measurement = history['measurement']
def read_zarr(): _ = zarr.load('test.zarr')
def ensemble_results(self, files, path=None, export_dir=None, filetype='.png', use_tta=None, **kwargs): use_tta = use_tta or self.pred_tta if export_dir: export_dir = Path(export_dir) pred_path = export_dir / 'masks' pred_path.mkdir(parents=True, exist_ok=True) if use_tta: unc_path = export_dir / 'uncertainties' unc_path.mkdir(parents=True, exist_ok=True) store = str(path / 'ensemble') if path else zarr.storage.TempStore() root = zarr.group(store=store, overwrite=True) chunk_store = root.chunk_store.path g_smx, g_seg, g_std, g_eng = root.create_groups( 'ens_smx', 'ens_seg', 'ens_std', 'ens_energy') res_list = [] for f in files: df_fil = self.df_models[self.df_models.file == f.name] assert len(df_fil) == len( self.models), "Predictions and models to not match." m_smx, m_std, m_eng = tta.Merger(), tta.Merger(), tta.Merger() for idx, r in df_fil.iterrows(): m_smx.append(zarr.load(r.smx_path)) m_std.append(zarr.load(r.std_path)) m_eng.append(zarr.load(r.eng_path)) smx = m_smx.result().numpy() g_smx[f.name] = smx g_seg[f.name] = np.argmax(smx, axis=-1) g_std[f.name] = m_std.result().numpy() eng = m_eng.result() g_eng[f.name] = eng.numpy() m_eng_max = energy_max(eng, ks=self.energy_ks).numpy() df_tmp = pd.Series({ 'file': f.name, 'model': f'{self.arch}_ensemble', 'energy_max': m_eng_max, 'img_path': f, 'pred_path': f'{chunk_store}/{g_seg.path}/{f.name}', 'smx_path': f'{chunk_store}/{g_smx.path}/{f.name}', 'std_path': f'{chunk_store}/{g_std.path}/{f.name}', 'eng_path': f'{chunk_store}/{g_eng.path}/{f.name}' }) res_list.append(df_tmp) if export_dir: save_mask(g_seg[f.name][:], pred_path / f'{df_tmp.file}_{df_tmp.model}_mask', filetype) if use_tta: save_unc(g_std[f.name][:], unc_path / f'{df_tmp.file}_{df_tmp.model}_unc', filetype) return pd.DataFrame(res_list)