def get_dataset(clazz, ext='', override=False): # ====== all path ====== # name = clazz.get_name(ext) + '.zip' path = base64.decodebytes(DataLoader.ORIGIN).decode() + name zip_path = clazz.get_zip_path(ext) out_path = clazz.get_ds_path(ext) # ====== check out_path ====== # if os.path.isfile(out_path): raise RuntimeError("Found a file at path: %s, we need a folder " "to unzip downloaded files." % out_path) elif os.path.isdir(out_path): if override or len(os.listdir(out_path)) == 0: shutil.rmtree(out_path) else: return Dataset(out_path, read_only=True) # ====== download the file ====== # if os.path.exists(zip_path) and override: os.remove(zip_path) if not os.path.exists(zip_path): get_file(name, path, DataLoader.BASE_DIR) # ====== upzip dataset ====== # unzip_aes(in_path=zip_path, out_path=out_path) ds = Dataset(out_path, read_only=True) if os.path.exists(zip_path): os.remove(zip_path) return ds
def get_dataset(clazz, ext='', override=False): # ====== all path ====== # name = clazz.get_name(ext) + '.zip' path = base64.decodebytes(DataLoader.ORIGIN).decode() + name zip_path = clazz.get_zip_path(ext) out_path = clazz.get_ds_path(ext) # ====== check out_path ====== # if os.path.isfile(out_path): raise RuntimeError("Found a file at path: %s, we need a folder " "to unzip downloaded files." % out_path) elif os.path.isdir(out_path): if override or len(os.listdir(out_path)) == 0: shutil.rmtree(out_path) else: return Dataset(out_path, read_only=True) # ====== download the file ====== # if os.path.exists(zip_path) and override: os.remove(zip_path) if not os.path.exists(zip_path): get_file(name, path, DataLoader.BASE_DIR) # ====== upzip dataset ====== # unzip_aes(in_path=zip_path, out_path=out_path) ds = Dataset(out_path, read_only=True) if os.path.exists(zip_path): os.remove(zip_path) return ds
def read_CITEseq_CBMC(filtered_genes=True, override=False, verbose=True): download_path = os.path.join(DOWNLOAD_DIR, "CBMC_original") if not os.path.exists(download_path): os.mkdir(download_path) preprocessed_path = _CITEseq_CBMC_PREPROCESSED if not os.path.exists(preprocessed_path): os.mkdir(preprocessed_path) elif override: if verbose: print("Overriding path: %s" % _CITEseq_CBMC_PREPROCESSED) shutil.rmtree(_CITEseq_CBMC_PREPROCESSED) os.mkdir(_CITEseq_CBMC_PREPROCESSED) # ******************** preprocessed data NOT found ******************** # if not os.path.exists(os.path.join(preprocessed_path, 'X')): X, X_row, X_col = [], None, None y, y_row, y_col = [], None, None # ====== download the data ====== # url = str(base64.decodebytes(_URL), 'utf-8') base_name = os.path.basename(url) zip_path = os.path.join(download_path, base_name) download_file(filename=zip_path, url=url, override=False, md5=r"beb76d01a67707c61c21bfb188e1b69f") # ====== extract the data ====== # data_dict = {} for name, data in crypto.unzip_aes(zip_path, password=_PASSWORD, verbose=False): base_name = os.path.splitext(name)[0] if '.npz' in name: data = sp.sparse.load_npz(BytesIO(data)).todense() elif '.csv' in name: data = np.loadtxt(StringIO(str(data, 'utf-8')), dtype=str, delimiter=',') else: raise RuntimeError("Unknown format: %s" % name) data_dict[base_name] = data # ====== post-processing ====== # X = np.array(data_dict['X'].astype('float32')) X_row, X_col = data_dict['X_row'], data_dict['X_col'] X, X_col = remove_allzeros_columns(matrix=X, colname=X_col) assert len(X_row) == X.shape[0] and len(X_col) == X.shape[1] y = data_dict['y'].astype('float32') y_row, y_col = data_dict['y_row'], data_dict['y_col'] assert len(y_row) == y.shape[0] and len(y_col) == y.shape[1] assert np.all(X_row == y_row), \ "Cell order mismatch between gene count and protein count" # save data if verbose: print(f"Saving data to {preprocessed_path} ...") save_to_dataset(preprocessed_path, X, X_col, y, y_col, rowname=X_row, print_log=verbose) sco = SingleCellOMIC(X, cell_id=X_row, gene_id=X_col) sc.pp.filter_cells(sco, min_genes=200) sc.pp.filter_genes(sco, min_cells=3) sc.pp.normalize_total(sco, target_sum=1e4) result = sc.pp.filter_genes_dispersion(sco.X, min_mean=0.0125, max_mean=3, min_disp=0.5, log=False, n_top_genes=2000) sco._inplace_subset_var(result.gene_subset) with open(os.path.join(preprocessed_path, 'top_genes'), 'wb') as f: pickle.dump(set(sco.var_names.values), f) del sco # ====== read preprocessed data ====== # ds = Dataset(preprocessed_path, read_only=True) sco = SingleCellOMIC( X=ds['X'], cell_id=ds['X_row'], gene_id=ds['X_col'], omic='transcriptomic', name=f"cbmcCITEseq{'' if filtered_genes else 'all'}", ).add_omic('proteomic', ds['y'], ds['y_col']) if filtered_genes: with open(os.path.join(preprocessed_path, 'top_genes'), 'rb') as f: top_genes = pickle.load(f) sco._inplace_subset_var([i in top_genes for i in sco.var_names]) return sco
def read_FACS(n_protein, override=False, verbose=False): download_path = os.path.join(DOWNLOAD_DIR, "FACS_original") if not os.path.exists(download_path): os.mkdir(download_path) n_protein = int(n_protein) assert n_protein in (2, 5) preprocessed_path = _FACS_PREPROCESSED % n_protein if not os.path.exists(preprocessed_path): os.mkdir(preprocessed_path) elif override: shutil.rmtree(preprocessed_path) os.mkdir(preprocessed_path) # ******************** preprocessed data NOT found ******************** # if not os.path.exists(os.path.join(preprocessed_path, 'X')): X, X_row, X_col = [], None, None y, y_row, y_col = [], None, None # ====== download the data ====== # url = str(base64.decodebytes(_URL), 'utf-8') % n_protein base_name = os.path.basename(url) zip_path = os.path.join(download_path, base_name) urlretrieve(url=url, filename=zip_path) # ====== extract the data ====== # data_dict = {} for name, data in crypto.unzip_aes(zip_path, password=_PASSWORD, verbose=False): base_name = os.path.splitext(name)[0] if '.npz' in name: data = sp.sparse.load_npz(BytesIO(data)).todense() elif '.csv' in name: data = np.loadtxt(StringIO(str(data, 'utf-8')), dtype=str, delimiter=',') else: raise RuntimeError("Unknown format: %s" % name) data_dict[base_name] = data if verbose: print('%-12s' % base_name, ':', data.shape) # ====== post-processing ====== # X = data_dict['X'].astype('float32') X = np.array(X) X_row, X_col = data_dict['X_row'], data_dict['X_col'] assert len(X_row) == X.shape[0] and len(X_col) == X.shape[1] y = data_dict['y'].astype('float32') y_row, y_col = data_dict['y_row'], data_dict['y_col'] assert len(y_row) == y.shape[0] and len(y_col) == y.shape[1] assert np.all(X_row == y_row), \ "Cell order mismatch between gene count and protein count" # ====== filter zero columns ====== # X, X_col = remove_allzeros_columns(matrix=X, colname=X_col, print_log=verbose) save_to_dataset(path=preprocessed_path, X=X, X_col=X_col, y=y, y_col=y_col, rowname=X_row, print_log=verbose) # ******************** read preprocessed data ******************** # ds = Dataset(preprocessed_path, read_only=True) return ds