def _expand_tiepoint_array_5km(self, arr, lines, cols): arr = da.repeat(arr, lines * 2, axis=1) arr = da.repeat(arr.reshape((-1, self.cscan_full_width - 1)), cols, axis=1) if self.cscan_full_width == 271: return da.hstack((arr[:, :2], arr, arr[:, -2:])) else: return da.hstack((arr[:, :2], arr, arr[:, -5:], arr[:, -2:]))
def navigate_dnb(h5f): scans = h5f.get_node("/All_Data/NumberOfScans").read()[0] geo_dset = h5f.get_node("/All_Data/VIIRS-DNB-GEO_All") all_c_align = geo_dset.AlignmentCoefficient.read()[np.newaxis, np.newaxis, :, np.newaxis] all_c_exp = geo_dset.ExpansionCoefficient.read()[np.newaxis, np.newaxis, :, np.newaxis] all_lon = geo_dset.Longitude.read() all_lat = geo_dset.Latitude.read() res = [] # FIXME: this supposes there is only one tiepoint zone in the # track direction scan_size = h5f.get_node_attr("/All_Data/VIIRS-DNB-SDR_All", "TiePointZoneSizeTrack")[0] track_offset = h5f.get_node_attr("/All_Data/VIIRS-DNB-SDR_All", "PixelOffsetTrack")[0] scan_offset = h5f.get_node_attr("/All_Data/VIIRS-DNB-SDR_All", "PixelOffsetScan")[0] try: group_locations = geo_dset.TiePointZoneGroupLocationScanCompact.read() except KeyError: group_locations = [0] param_start = 0 for tpz_size, nb_tpz, start in \ zip(h5f.get_node_attr("/All_Data/VIIRS-DNB-SDR_All", "TiePointZoneSizeScan"), geo_dset.NumberOfTiePointZonesScan.read(), group_locations): lon = all_lon[:, start:start + nb_tpz + 1] lat = all_lat[:, start:start + nb_tpz + 1] c_align = all_c_align[:, :, param_start:param_start + nb_tpz, :] c_exp = all_c_exp[:, :, param_start:param_start + nb_tpz, :] param_start += nb_tpz nties = nb_tpz if (np.max(lon) - np.min(lon) > 90) or (np.max(abs(lat)) > 60): x, y, z = lonlat2xyz(lon, lat) x, y, z = (expand_array(x, scans, c_align, c_exp, scan_size, tpz_size, nties, track_offset, scan_offset), expand_array(y, scans, c_align, c_exp, scan_size, tpz_size, nties, track_offset, scan_offset), expand_array(z, scans, c_align, c_exp, scan_size, tpz_size, nties, track_offset, scan_offset)) res.append(xyz2lonlat(x, y, z)) else: res.append( (expand_array(lon, scans, c_align, c_exp, scan_size, tpz_size, nties, track_offset, scan_offset), expand_array(lat, scans, c_align, c_exp, scan_size, tpz_size, nties, track_offset, scan_offset))) lons, lats = zip(*res) return da.hstack(lons), da.hstack(lats)
def _expand_tiepoint_array_5km(self, arr, lines, cols): arr = da.repeat(arr, lines * 2, axis=1) arr = da.repeat(arr.reshape((-1, self.cscan_full_width - 1)), cols, axis=1) factor = self.fscan_width // self.cscan_width if self.cscan_full_width == 271: return da.hstack((arr[:, :2 * factor], arr, arr[:, -2 * factor:])) else: return da.hstack((arr[:, :2 * factor], arr, arr[:, -self.fscan_width:], arr[:, -2 * factor:]))
def test_hstack(): x = np.arange(5) y = np.ones(5) a = da.arange(5, chunks=2) b = da.ones(5, chunks=2) assert_eq(np.hstack((x[None, :], y[None, :])), da.hstack((a[None, :], b[None, :]))) assert_eq(np.hstack((x, y)), da.hstack((a, b)))
def navigate_dnb(h5f): scans = h5f.get_node("/All_Data/NumberOfScans").read()[0] geo_dset = h5f.get_node("/All_Data/VIIRS-DNB-GEO_All") all_c_align = geo_dset.AlignmentCoefficient.read()[ np.newaxis, np.newaxis, :, np.newaxis] all_c_exp = geo_dset.ExpansionCoefficient.read()[np.newaxis, np.newaxis, :, np.newaxis] all_lon = geo_dset.Longitude.read() all_lat = geo_dset.Latitude.read() res = [] # FIXME: this supposes there is only one tiepoint zone in the # track direction scan_size = h5f.get_node_attr("/All_Data/VIIRS-DNB-SDR_All", "TiePointZoneSizeTrack")[0] track_offset = h5f.get_node_attr("/All_Data/VIIRS-DNB-SDR_All", "PixelOffsetTrack")[0] scan_offset = h5f.get_node_attr("/All_Data/VIIRS-DNB-SDR_All", "PixelOffsetScan")[0] try: group_locations = geo_dset.TiePointZoneGroupLocationScanCompact.read() except KeyError: group_locations = [0] param_start = 0 for tpz_size, nb_tpz, start in \ zip(h5f.get_node_attr("/All_Data/VIIRS-DNB-SDR_All", "TiePointZoneSizeScan"), geo_dset.NumberOfTiePointZonesScan.read(), group_locations): lon = all_lon[:, start:start + nb_tpz + 1] lat = all_lat[:, start:start + nb_tpz + 1] c_align = all_c_align[:, :, param_start:param_start + nb_tpz, :] c_exp = all_c_exp[:, :, param_start:param_start + nb_tpz, :] param_start += nb_tpz nties = nb_tpz if (np.max(lon) - np.min(lon) > 90) or (np.max(abs(lat)) > 60): x, y, z = lonlat2xyz(lon, lat) x, y, z = ( expand_array(x, scans, c_align, c_exp, scan_size, tpz_size, nties, track_offset, scan_offset), expand_array(y, scans, c_align, c_exp, scan_size, tpz_size, nties, track_offset, scan_offset), expand_array( z, scans, c_align, c_exp, scan_size, tpz_size, nties, track_offset, scan_offset)) res.append(xyz2lonlat(x, y, z)) else: res.append( (expand_array(lon, scans, c_align, c_exp, scan_size, tpz_size, nties, track_offset, scan_offset), expand_array(lat, scans, c_align, c_exp, scan_size, tpz_size, nties, track_offset, scan_offset))) lons, lats = zip(*res) return da.hstack(lons), da.hstack(lats)
def _create_dask_slice_from_block_line(self, current_line, chunks): """Create a dask slice from the blocks at the current line.""" pieces = self._get_array_pieces_for_current_line(current_line) dask_pieces = self._get_padded_dask_pieces(pieces, chunks) new_slice = da.hstack(dask_pieces) return new_slice
def _sampling_reconst(self, session, std_scales, random_latent=None): if random_latent is None: random_latent = list() for m, sig, sc in zip(self.config.latent_mean, self.config.latent_std, std_scales): random_latent.append( session.run( tf.random_normal((self.config.batch_size, 1), m, sc * sig, dtype=tf.float32))) random_latent = da.hstack(random_latent) else: try: random_latent = random_latent.compute() except: pass for m, sig, sc, ic in zip(self.config.latent_mean, self.config.latent_std, std_scales, range(random_latent.shape[0])): random_latent[:, ic] = m + (sc * np.sqrt(sig) * random_latent[:, ic]) tensors = [self.x_recons] feed_dict = {self.latent_batch: random_latent} return session.run(tensors, feed_dict=feed_dict)
def test_func(default_val, dataset_flat, shape, dataset): shift_up = array.hstack([ array.zeros((shape[0], 1, shape[2])), dataset[:, :-1, :] ]).transpose([1, 2, 0]).reshape([shape[1] * shape[2], -1]) shift_up_mult = dataset_flat * shift_up del shift_up return array.mean(shift_up_mult, axis=1)
def _expand_tiepoint_array_1km(self, arr, lines, cols): arr = da.repeat(arr, lines, axis=1) arr = da.concatenate( (arr[:, :lines // 2, :], arr, arr[:, -(lines // 2):, :]), axis=1) arr = da.repeat(arr.reshape((-1, self.cscan_full_width - 1)), cols, axis=1) return da.hstack((arr, arr[:, -cols:]))
def angles(self, azi_name, zen_name): all_lat = self.geostuff["Latitude"].value all_zen = self.geostuff[zen_name].value all_azi = self.geostuff[azi_name].value res = [] param_start = 0 for tpz_size, nb_tpz, start in zip(self.tpz_sizes, self.nb_tpzs, self.group_locations): lat = all_lat[:, start:start + nb_tpz + 1] zen = all_zen[:, start:start + nb_tpz + 1] azi = all_azi[:, start:start + nb_tpz + 1] c_align = self.c_align[:, :, param_start:param_start + nb_tpz, :] c_exp = self.c_exp[:, :, param_start:param_start + nb_tpz, :] param_start += nb_tpz if (np.max(azi) - np.min(azi) > 5) or (np.min(zen) < 10) or (np.max(abs(lat)) > 80): expanded = [] for data in angle2xyz(azi, zen): expanded.append( expand_array(data, self.scans, c_align, c_exp, self.scan_size, tpz_size, nb_tpz, self.track_offset, self.scan_offset)) azi, zen = xyz2angle(*expanded) res.append((azi, zen)) else: expanded = [] for data in (azi, zen): expanded.append( expand_array(data, self.scans, c_align, c_exp, self.scan_size, tpz_size, nb_tpz, self.track_offset, self.scan_offset)) res.append(expanded) azi, zen = zip(*res) return da.hstack(azi), da.hstack(zen)
def angles(self, azi_name, zen_name): """Compute the angle datasets.""" all_lat = da.from_array(self.geostuff["Latitude"]) all_lon = da.from_array(self.geostuff["Longitude"]) all_zen = da.from_array(self.geostuff[zen_name]) all_azi = da.from_array(self.geostuff[azi_name]) res = [] param_start = 0 for tpz_size, nb_tpz, start in zip(self.tpz_sizes, self.nb_tpzs, self.group_locations): lat = all_lat[:, start:start + nb_tpz + 1] lon = all_lon[:, start:start + nb_tpz + 1] zen = all_zen[:, start:start + nb_tpz + 1] azi = all_azi[:, start:start + nb_tpz + 1] c_align = self.c_align[:, :, param_start:param_start + nb_tpz, :] c_exp = self.c_exp[:, :, param_start:param_start + nb_tpz, :] param_start += nb_tpz if (np.max(azi) - np.min(azi) > 5) or (np.min(zen) < 10) or ( np.max(abs(lat)) > 80): expanded = [] cart = convert_from_angles(azi, zen, lon, lat) for data in cart: expanded.append(expand_array( data, self.scans, c_align, c_exp, self.scan_size, tpz_size, nb_tpz, self.track_offset, self.scan_offset)) azi, zen = convert_to_angles(*expanded, lon=self.lons, lat=self.lats) res.append((azi, zen)) else: expanded = [] for data in (azi, zen): expanded.append(expand_array( data, self.scans, c_align, c_exp, self.scan_size, tpz_size, nb_tpz, self.track_offset, self.scan_offset)) res.append(expanded) azi, zen = zip(*res) return da.hstack(azi), da.hstack(zen)
def _create_dask_slice_from_block_line(self, current_line, chunks): """Create a dask slice from the blocks at the current line.""" current_blocks = self._find_blocks_covering_line(current_line) current_blocks.sort(key=(lambda x: x.coords['x'][0])) next_line = min((arr.coords['y'][-1] for arr in current_blocks)) current_y = np.arange(current_line, next_line + 1) pieces = [arr.sel(y=current_y) for arr in current_blocks] dask_pieces = self._get_padded_dask_pieces(pieces, chunks) new_slice = da.hstack(dask_pieces) return new_slice
def angles(self, azi_name, zen_name): all_lat = self.geostuff["Latitude"].value all_zen = self.geostuff[zen_name].value all_azi = self.geostuff[azi_name].value res = [] param_start = 0 for tpz_size, nb_tpz, start in zip(self.tpz_sizes, self.nb_tpzs, self.group_locations): lat = all_lat[:, start:start + nb_tpz + 1] zen = all_zen[:, start:start + nb_tpz + 1] azi = all_azi[:, start:start + nb_tpz + 1] c_align = self.c_align[:, :, param_start:param_start + nb_tpz, :] c_exp = self.c_exp[:, :, param_start:param_start + nb_tpz, :] param_start += nb_tpz if (np.max(azi) - np.min(azi) > 5) or (np.min(zen) < 10) or ( np.max(abs(lat)) > 80): expanded = [] for data in angle2xyz(azi, zen): expanded.append(expand_array( data, self.scans, c_align, c_exp, self.scan_size, tpz_size, nb_tpz, self.track_offset, self.scan_offset)) azi, zen = xyz2angle(*expanded) res.append((azi, zen)) else: expanded = [] for data in (azi, zen): expanded.append(expand_array( data, self.scans, c_align, c_exp, self.scan_size, tpz_size, nb_tpz, self.track_offset, self.scan_offset)) res.append(expanded) azi, zen = zip(*res) return da.hstack(azi), da.hstack(zen)
def navigate(self): all_lon = self.geostuff["Longitude"].value all_lat = self.geostuff["Latitude"].value res = [] param_start = 0 for tpz_size, nb_tpz, start in zip(self.tpz_sizes, self.nb_tpzs, self.group_locations): lon = all_lon[:, start:start + nb_tpz + 1] lat = all_lat[:, start:start + nb_tpz + 1] c_align = self.c_align[:, :, param_start:param_start + nb_tpz, :] c_exp = self.c_exp[:, :, param_start:param_start + nb_tpz, :] param_start += nb_tpz expanded = [] switch_to_cart = ((np.max(lon) - np.min(lon) > 90) or (np.max(abs(lat)) > 60)) if switch_to_cart: arrays = lonlat2xyz(lon, lat) else: arrays = (lon, lat) for data in arrays: expanded.append( expand_array(data, self.scans, c_align, c_exp, self.scan_size, tpz_size, nb_tpz, self.track_offset, self.scan_offset)) if switch_to_cart: res.append(xyz2lonlat(*expanded)) else: res.append(expanded) lons, lats = zip(*res) return da.hstack(lons), da.hstack(lats)
def _expand_tiepoint_array_5km(self, arr, lines, cols): if self.level == 2: # Repeat the last column to complete L2 data arr = da.dstack([arr, arr[:, :, -1]]) arr = da.repeat(arr, lines * 2, axis=1) if self.level == 1: arr = da.repeat(arr.reshape((-1, self.cscan_full_width - 1)), cols, axis=1) elif self.level == 2: arr = da.repeat(arr.reshape((-1, self.cscan_full_width)), cols, axis=1) return da.hstack((arr[:, :2], arr, arr[:, -2:]))
def navigate(self): all_lon = self.geostuff["Longitude"].value all_lat = self.geostuff["Latitude"].value res = [] param_start = 0 for tpz_size, nb_tpz, start in zip(self.tpz_sizes, self.nb_tpzs, self.group_locations): lon = all_lon[:, start:start + nb_tpz + 1] lat = all_lat[:, start:start + nb_tpz + 1] c_align = self.c_align[:, :, param_start:param_start + nb_tpz, :] c_exp = self.c_exp[:, :, param_start:param_start + nb_tpz, :] param_start += nb_tpz expanded = [] switch_to_cart = ((np.max(lon) - np.min(lon) > 90) or (np.max(abs(lat)) > 60)) if switch_to_cart: arrays = lonlat2xyz(lon, lat) else: arrays = (lon, lat) for data in arrays: expanded.append(expand_array( data, self.scans, c_align, c_exp, self.scan_size, tpz_size, nb_tpz, self.track_offset, self.scan_offset)) if switch_to_cart: res.append(xyz2lonlat(*expanded)) else: res.append(expanded) lons, lats = zip(*res) return da.hstack(lons), da.hstack(lats)
def project_cone(K, x): s = x[0].compute() v = x[1:] norm_v = da.linalg.norm(v).compute() if norm_v <= -s: projx = 0 * x elif norm_v <= s: projx = 1 * x else: scal = 0.5 * (1 + s / norm_v) s = da.from_array(np.array([norm_v]), chunks=(1, )) projx = scal * da.hstack((s, v)) return projx
def navigate(self): """Generate lon and lat datasets.""" all_lon = da.from_array(self.geostuff["Longitude"]) all_lat = da.from_array(self.geostuff["Latitude"]) res = [] param_start = 0 for tpz_size, nb_tpz, start in zip(self.tpz_sizes, self.nb_tpzs, self.group_locations): lon = all_lon[:, start:start + nb_tpz + 1] lat = all_lat[:, start:start + nb_tpz + 1] c_align = self.c_align[:, :, param_start:param_start + nb_tpz, :] c_exp = self.c_exp[:, :, param_start:param_start + nb_tpz, :] param_start += nb_tpz expanded = [] if self.switch_to_cart: arrays = lonlat2xyz(lon, lat) else: arrays = (lon, lat) for data in arrays: expanded.append( expand_array(data, self.scans, c_align, c_exp, self.scan_size, tpz_size, nb_tpz, self.track_offset, self.scan_offset)) if self.switch_to_cart: res.append(xyz2lonlat(*expanded)) else: res.append(expanded) lons, lats = zip(*res) return da.hstack(lons), da.hstack(lats)
def setup_input(samples, input_pattern, seqid, field): log('Setting up input array ...') input_paths = [input_pattern.format(sample=s) for s in samples] input_stores = [zarr.ZipStore(ip, mode='r') for ip in input_paths] input_roots = [zarr.group(store) for store in input_stores] input_arrays = [ root[s][seqid][field] for root, s in zip(input_roots, samples) ] input_arrays = [da.from_array(a, chunks=a.chunks) for a in input_arrays] # here we add a dim to allow the hstack to work. must share the shape (X, 1, ) input_arrays = [a[:, None] if a.ndim == 1 else a for a in input_arrays] input_array = da.hstack(input_arrays) log('Input array:', input_array) return input_array
def dask_array_resolver(obj, resolver, **kw): def get_partition(obj_id): client = vineyard.connect() np_value = client.get(obj_id) return da.from_array(np_value) meta = obj.meta num = int(meta['partitions_-size']) dask_client = Client(kw['dask_scheduler']) futures = [] indices = [] with_index = True for i in range(num): ts = meta.get_member('partitions_-%d' % i) instance_id = int(ts.meta['instance_id']) partition_index = json.loads(ts.meta['partition_index_']) if partition_index: indices.append((partition_index[0], partition_index[1], i)) else: with_index = False futures.append( # we require the 1-on-1 alignment of vineyard instances and dask workers. # vineyard_sockets maps vineyard instance_ids into ipc_sockets, while # dask_workers maps vineyard instance_ids into names of dask workers. dask_client.submit(get_partition, ts.meta.id, workers={kw['dask_workers'][instance_id]})) arrays = dask_client.gather(futures) if with_index: indices = list(sorted(indices)) nx = indices[-1][0] + 1 ny = indices[-1][1] + 1 assert nx * ny == num rows = [] for i in range(nx): cols = [] for j in range(ny): cols.append(arrays[indices[i * ny + j][2]]) rows.append(da.hstack(cols)) return da.vstack(rows) return da.vstack(arrays)
def _hstack(self, Xs): """ Stacks X horizontally. Supports input types (X): list of numpy arrays, sparse arrays and DataFrames """ types = set(type(X) for X in Xs) if self.sparse_output_: return sparse.hstack(Xs).tocsr() elif dd.Series in types or dd.DataFrame in types: return dd.concat(Xs, axis="columns") elif da.Array in types: return da.hstack(Xs) elif self.preserve_dataframe and (pd.Series in types or pd.DataFrame in types): return pd.concat(Xs, axis="columns") else: return np.hstack(Xs)
def create_fft_freq(dt, nfft, full): """ Creates a list of Fourier Frequencies Inputs: ======= dt: float, Sampling frequency in Hz nfft: int, number of data points to apply FFT to full: bool, if True, then we want the DFT from -fN...0...fN. If false, we do a half-DFT from 0...fN Returns: ======== freqs, ndarray(float): Frequencies of the Fourier Coefficients calculated by FFT. """ freqs = da.fft.fftfreq(nfft, d=dt) if (nfft % 2 == 0): freqs = da.hstack( [freqs[0:nfft // 2], -freqs[nfft // 2], freqs[nfft // 2:nfft]]) if full: freqs = da.fft.fftshift(freqs) else: freqs = freqs[0:nfft // 2 + 1] return (freqs)
def read_bgen( path: PathType, metafile_path: Optional[PathType] = None, sample_path: Optional[PathType] = None, chunks: Union[str, int, Tuple[int, int, int]] = "auto", lock: bool = False, persist: bool = True, contig_dtype: DType = "str", gp_dtype: DType = "float32", ) -> Dataset: """Read BGEN dataset. Loads a single BGEN dataset as dask arrays within a Dataset from a ``.bgen`` file. Parameters ---------- path Path to BGEN file. metafile_path Path to companion index file used to determine BGEN byte offsets. Defaults to ``path`` + ".metafile" if not provided. This file is necessary for reading BGEN genotype probabilities and it will be generated the first time the file is read if it does not already exist. If it needs to be created, it can make the first call to this function much slower than subsequent calls. sample_path Path to ``.sample`` file, by default None. This is used to fetch sample identifiers and when provided it is preferred over sample identifiers embedded in the ``.bgen`` file. chunks Chunk size for genotype probability data (3 dimensions), by default "auto". lock Whether or not to synchronize concurrent reads of file blocks, by default False. This is passed through to [dask.array.from_array](https://docs.dask.org/en/latest/array-api.html#dask.array.from_array). persist Whether or not to persist variant information in memory, by default True. This is an important performance consideration as the metadata file for this data will be read multiple times when False. contig_dtype Data type for contig names, by default "str". This may also be an integer type (e.g. "int"), but will fail if any of the contig names cannot be converted to integers. gp_dtype Data type for genotype probabilities, by default "float32". Warnings -------- Only bi-allelic, diploid BGEN files are currently supported. Returns ------- A dataset containing the following variables: - :data:`sgkit.variables.variant_id_spec` (variants) - :data:`sgkit.variables.variant_contig_spec` (variants) - :data:`sgkit.variables.variant_position_spec` (variants) - :data:`sgkit.variables.variant_allele_spec` (variants) - :data:`sgkit.variables.sample_id_spec` (samples) - :data:`sgkit.variables.call_dosage_spec` (variants, samples) - :data:`sgkit.variables.call_dosage_mask_spec` (variants, samples) - :data:`sgkit.variables.call_genotype_probability_spec` (variants, samples, genotypes) - :data:`sgkit.variables.call_genotype_probability_mask_spec` (variants, samples, genotypes) """ if isinstance(chunks, tuple) and len(chunks) != 3: raise ValueError(f"`chunks` must be tuple with 3 items, not {chunks}") if not np.issubdtype(gp_dtype, np.floating): raise ValueError( f"`gp_dtype` must be a floating point data type, not {gp_dtype}" ) if not np.issubdtype(contig_dtype, np.integer) and np.dtype( contig_dtype ).kind not in {"U", "S"}: raise ValueError( f"`contig_dtype` must be of string or int type, not {contig_dtype}" ) path = Path(path) sample_path = Path(sample_path) if sample_path else path.with_suffix(".sample") if sample_path.exists(): sample_id = read_samples(sample_path).sample_id.values.astype("U") else: sample_id = _default_sample_ids(path) bgen_reader = BgenReader(path, metafile_path=metafile_path, dtype=gp_dtype) df = read_metafile(bgen_reader.metafile_path) if persist: df = df.persist() arrs = dataframe_to_dict(df, METAFILE_DTYPE) variant_id = arrs["id"] variant_contig = arrs["chrom"].astype(contig_dtype) variant_contig, variant_contig_names = encode_contigs(variant_contig) variant_contig_names = list(variant_contig_names) variant_position = arrs["pos"] variant_allele = da.hstack((arrs["a1"][:, np.newaxis], arrs["a2"][:, np.newaxis])) call_genotype_probability = da.from_array( bgen_reader, chunks=chunks, lock=lock, fancy=False, asarray=False, name=f"{bgen_reader.name}:read_bgen:{path}", ) call_dosage = _to_dosage(call_genotype_probability) ds: Dataset = create_genotype_dosage_dataset( variant_contig_names=variant_contig_names, variant_contig=variant_contig, variant_position=variant_position, variant_allele=variant_allele, sample_id=sample_id, call_dosage=call_dosage, call_genotype_probability=call_genotype_probability, variant_id=variant_id, ) return ds
xinfo['xtermcols'] = json.load(jf) h5read = tables.open_file('knockoff-data.h5', mode='r') h5regression = tables.open_file('regression-data.h5', mode='r') X = da.from_array(h5read.root.X) pdim = X.shape[1] Xtilde = da.from_array(h5read.root.Xtilde) Y = da.from_array(h5regression.root.Y) keepcols_svd = list(h5read.root.keepcols) xcolnames_pdim = [] for k in xinfo['xcolnames']: if xinfo['xcolnames'][k] in keepcols_svd: xcolnames_pdim.append(k) with Profiler() as prof, ResourceProfiler() as rprof, CacheProfiler() as cprof: Xaug = da.hstack([X, Xtilde]) betahat_aug = da.linalg.solve(da.matmul(Xaug.T, Xaug), da.matmul(Xaug.T, Y)).compute() Wstat = [ abs(betahat_aug[i]) - abs(betahat_aug[i + pdim]) for i in range(pdim) ] threshold = knockoff_threshold(Wstat, FDR, offset=1) sel = [Wstat[j] >= threshold for j in range(pdim)] Xdrop = X[:, sel] betahat_final = da.linalg.solve(da.matmul(Xdrop.T, Xdrop), da.matmul(Xdrop.T, Y)).compute() colnames_final = [i for i, j in zip(xcolnames_pdim, sel) if j] colnames_dropped = [i for i, j in zip(xcolnames_pdim, sel) if not j] print("desired FDR: ") print(FDR) print("\nKnockoff drops these columns:\n")
def make_blobs( n_samples=100, n_features=2, centers=None, cluster_std=1.0, center_box=(-10.0, 10.0), shuffle=True, random_state=None, chunks=None, ): """ Generate isotropic Gaussian blobs for clustering. This can be used to generate very large Dask arrays on a cluster of machines. When using Dask in distributed mode, the client machine only needs to allocate a single block's worth of data. Parameters ---------- n_samples : int or array-like, optional (default=100) If int, it is the total number of points equally divided among clusters. If array-like, each element of the sequence indicates the number of samples per cluster. n_features : int, optional (default=2) The number of features for each sample. centers : int or array of shape [n_centers, n_features], optional (default=None) The number of centers to generate, or the fixed center locations. If n_samples is an int and centers is None, 3 centers are generated. If n_samples is array-like, centers must be either None or an array of length equal to the length of n_samples. cluster_std : float or sequence of floats, optional (default=1.0) The standard deviation of the clusters. center_box : pair of floats (min, max), optional (default=(-10.0, 10.0)) The bounding box for each cluster center when centers are generated at random. shuffle : boolean, optional (default=True) Shuffle the samples. random_state : int, RandomState instance or None (default) Determines random number generation for dataset creation. Pass an int for reproducible output across multiple function calls. See :term:`Glossary <random_state>`. chunks : int, tuple How to chunk the array. Must be one of the following forms: - A blocksize like 1000. - A blockshape like (1000, 1000). - Explicit sizes of all blocks along all dimensions like ((1000, 1000, 500), (400, 400)). Returns ------- X : array of shape [n_samples, n_features] The generated samples. y : array of shape [n_samples] The integer labels for cluster membership of each sample. Examples -------- >>> from dask_ml.datasets import make_blobs >>> X, y = make_blobs(n_samples=100000, chunks=10000) >>> X dask.array<..., shape=(100000, 2), dtype=float64, chunksize=(10000, 2)> >>> y dask.array<concatenate, shape=(100000,), dtype=int64, chunksize=(10000,)> See Also -------- make_classification: a more intricate variant """ chunks = da.core.normalize_chunks(chunks, (n_samples, n_features)) _check_axis_partitioning(chunks, n_features) if centers is None: # TODO: non-int n_samples? centers = 3 if isinstance(centers, numbers.Integral): # Make a prototype n_centers = centers X, y = sklearn.datasets.make_blobs( n_samples=chunks[0][0], n_features=n_features, centers=centers, shuffle=shuffle, cluster_std=cluster_std, center_box=center_box, random_state=random_state, ) centers = [] centers = np.zeros((n_centers, n_features)) for i in range(n_centers): centers[i] = X[y == i].mean(0) objs = [ dask.delayed(sklearn.datasets.make_blobs, nout=2)( n_samples=n_samples_per_block, n_features=n_features, centers=centers, cluster_std=cluster_std, shuffle=shuffle, center_box=center_box, random_state=i, ) for i, n_samples_per_block in enumerate(chunks[0]) ] Xobjs, yobjs = zip(*objs) Xarrs = [ da.from_delayed(arr, shape=(n, n_features), dtype="f8") for arr, n in zip(Xobjs, chunks[0]) ] X_big = da.vstack(Xarrs) yarrs = [ da.from_delayed(arr, shape=(n,), dtype=np.dtype("int")) for arr, n in zip(yobjs, chunks[0]) ] y_big = da.hstack(yarrs) return X_big, y_big
def read_plink( *, path: Optional[PathType] = None, bed_path: Optional[PathType] = None, bim_path: Optional[PathType] = None, fam_path: Optional[PathType] = None, chunks: Union[str, int, tuple] = "auto", # type: ignore[type-arg] fam_sep: str = " ", bim_sep: str = "\t", bim_int_contig: bool = False, count_a1: bool = True, lock: bool = False, persist: bool = True, ) -> Dataset: """Read PLINK dataset. Loads a single PLINK dataset as dask arrays within a Dataset from bed, bim, and fam files. Parameters ---------- path : Optional[PathType] Path to PLINK file set. This should not include a suffix, i.e. if the files are at `data.{bed,fam,bim}` then only 'data' should be provided (suffixes are added internally). Either this path must be provided or all 3 of `bed_path`, `bim_path` and `fam_path`. bed_path: Optional[PathType] Path to PLINK bed file. This should be a full path including the `.bed` extension and cannot be specified in conjunction with `path`. bim_path: Optional[PathType] Path to PLINK bim file. This should be a full path including the `.bim` extension and cannot be specified in conjunction with `path`. fam_path: Optional[PathType] Path to PLINK fam file. This should be a full path including the `.fam` extension and cannot be specified in conjunction with `path`. chunks : Union[str, int, tuple], optional Chunk size for genotype (i.e. `.bed`) data, by default "auto" fam_sep : str, optional Delimiter for `.fam` file, by default " " bim_sep : str, optional Delimiter for `.bim` file, by default "\t" bim_int_contig : bool, optional Whether or not the contig/chromosome name in the `.bim` file should be interpreted as an integer, by default False. If False, then the `variant/contig` field in the resulting dataset will contain the indexes of corresponding strings encountered in the first `.bim` field. count_a1 : bool, optional Whether or not allele counts should be for A1 or A2, by default True. Typically A1 is the minor allele and should be counted instead of A2. This is not enforced by PLINK though and it is up to the data generating process to ensure that A1 is in fact an alternate/minor/effect allele. See https://www.cog-genomics.org/plink/1.9/formats for more details. lock : bool, optional Whether or not to synchronize concurrent reads of `.bed` file blocks, by default False. This is passed through to [dask.array.from_array](https://docs.dask.org/en/latest/array-api.html#dask.array.from_array). persist : bool, optional Whether or not to persist `.fam` and `.bim` information in memory, by default True. This is an important performance consideration as the plain text files for this data will be read multiple times when False. This can lead to load times that are upwards of 10x slower. Returns ------- Dataset A dataset containing genotypes as 3 dimensional calls along with all accompanying pedigree and variant information. The content of this dataset matches that of `sgkit.create_genotype_call_dataset` with all pedigree-specific fields defined as: - sample_family_id: Family identifier commonly referred to as FID - sample_id: Within-family identifier for sample - sample_paternal_id: Within-family identifier for father of sample - sample_maternal_id: Within-family identifier for mother of sample - sample_sex: Sex code equal to 1 for male, 2 for female, and -1 for missing - sample_phenotype: Phenotype code equal to 1 for control, 2 for case, and -1 for missing See https://www.cog-genomics.org/plink/1.9/formats#fam for more details. Raises ------ ValueError If `path` and one of `bed_path`, `bim_path` or `fam_path` are provided. """ if path and (bed_path or bim_path or fam_path): raise ValueError( "Either `path` or all 3 of `{bed,bim,fam}_path` must be specified but not both" ) if path: bed_path, bim_path, fam_path = [ f"{path}.{ext}" for ext in ["bed", "bim", "fam"] ] # Load axis data first to determine dimension sizes df_fam = read_fam(fam_path, sep=fam_sep) # type: ignore[arg-type] df_bim = read_bim(bim_path, sep=bim_sep) # type: ignore[arg-type] if persist: df_fam = df_fam.persist() df_bim = df_bim.persist() arr_fam = _to_dict(df_fam, dtype=FAM_ARRAY_DTYPE) arr_bim = _to_dict(df_bim, dtype=BIM_ARRAY_DTYPE) # Load genotyping data call_genotype = da.from_array( # Make sure to use asarray=False in order for masked arrays to propagate BedReader(bed_path, (len(df_bim), len(df_fam)), count_A1=count_a1), # type: ignore[arg-type] chunks=chunks, # Lock must be true with multiprocessing dask scheduler # to not get bed-reader errors (it works w/ threading backend though) lock=lock, asarray=False, name=f"bed_reader:read_plink:{bed_path}", ) # If contigs are already integers, use them as-is if bim_int_contig: variant_contig = arr_bim["contig"].astype("int16") variant_contig_names = da.unique(variant_contig).astype(str) variant_contig_names = list(variant_contig_names.compute()) # Otherwise create index for contig names based # on order of appearance in underlying .bim file else: variant_contig, variant_contig_names = encode_array( arr_bim["contig"].compute()) variant_contig = variant_contig.astype("int16") variant_contig_names = list(variant_contig_names) variant_position = arr_bim["pos"] a1 = arr_bim["a1"].astype("str") a2 = arr_bim["a2"].astype("str") # Note: column_stack not implemented in Dask, must use [v|h]stack variant_alleles = da.hstack((a1[:, np.newaxis], a2[:, np.newaxis])) variant_alleles = variant_alleles.astype("S") variant_id = arr_bim["variant_id"] sample_id = arr_fam["member_id"] ds = create_genotype_call_dataset( variant_contig_names=variant_contig_names, variant_contig=variant_contig, variant_position=variant_position, variant_alleles=variant_alleles, sample_id=sample_id, call_genotype=call_genotype, variant_id=variant_id, ) # Assign PLINK-specific pedigree fields ds = ds.assign( **{ f"sample_{f}": (DIM_SAMPLE, arr_fam[f]) for f in arr_fam if f != "member_id" }) return ds # type: ignore[no-any-return]
def run_analysis_wrinkling(self, filter_width, filter_type, c_analytical=False, Parallel=False, every_nth=1): ''' :param filter_width: DNS points to filter :param filter_type: use 'TOPHAT' rather than 'GAUSSIAN :param c_analytical: compute c minus analytically :param Parallel: use False :param every_nth: every nth DNS point to compute the isoArea :return: ''' # run the analysis and compute the wrinkling factor -> real 3D cases # interval is like nth point, skips some nodes self.filter_type = filter_type # joblib parallel computing of c_iso self.Parallel = Parallel self.every_nth = int(every_nth) print('You are using %s filter!' % self.filter_type) self.filter_width = int(filter_width) self.c_analytical = c_analytical if self.c_analytical is True: print('You are using Hypergeometric function for c_minus (Eq.35)!') # filter the c and rho field print('Filtering c field ...') self.rho_filtered = self.apply_filter(self.rho_data_np) self.c_filtered = self.apply_filter(self.c_data_np) # # reduce c for computation of conditioned wrinkling factor # self.reduce_c(c_min=0.75,c_max=0.85) # self.c_filtered_reduced = self.apply_filter(self.c_data_reduced_np) # Compute the scaled Delta (Pfitzner PDF) self.Delta_LES = self.delta_x * self.filter_width * self.Sc * self.Re * np.sqrt( self.p / self.p_0) print('Delta_LES is: %.3f' % self.Delta_LES) flame_thickness = self.compute_flamethickness() print('Flame thickness: ', flame_thickness) #maximum possible wrinkling factor print('Maximum possible wrinkling factor: ', self.Delta_LES / flame_thickness) # Set the Gauss kernel self.set_gaussian_kernel() # compute the wrinkling factor self.get_wrinkling() self.compute_Pfitzner_model() #c_bins = self.compute_c_binning(c_low=0.8,c_high=0.9) # start = time.time() # if self.Parallel is True: # isoArea_coefficient = self.compute_isoArea_parallel(c_iso=0.85) # else: # isoArea_coefficient = self.compute_isoArea(c_iso=0.85) # # end=time.time() print('No c_iso was computed') # write the filtered omega and omega_model * isoArea to file print( 'writing omega DNS filtered and omega_model x isoArea to file ...') filename = join( self.case, 'filtered_data', 'omega_filtered_modeled_' + str(self.filter_width) + '_nth' + str(self.every_nth) + '.csv') # om_iso = self.omega_model_cbar*isoArea_coefficient om_wrinkl = self.omega_model_cbar * self.wrinkling_factor # pd.DataFrame(data=np.hstack([self.omega_DNS.reshape(self.Nx**3,1), # self.omega_DNS_filtered.reshape(self.Nx**3,1), # om_iso.reshape(self.Nx**3,1), # om_wrinkl.reshape(self.Nx**3,1), # self.c_filtered.reshape(self.Nx ** 3, 1)]), # columns=['omega_DNS', # 'omega_filtered', # 'omega_model_by_isoArea', # 'omega_model_by_wrinkling', # 'c_bar']).to_csv(filename) # creat dask array and reshape all data dataArray_da = da.hstack([ self.c_filtered.reshape(self.Nx**3, 1), self.wrinkling_factor.reshape(self.Nx**3, 1), # isoArea_coefficient.reshape(self.Nx**3,1), # self.wrinkling_factor_LES.reshape(self.Nx ** 3, 1), # self.wrinkling_factor_reduced.reshape(self.Nx ** 3, 1), # self.wrinkling_factor_LES_reduced.reshape(self.Nx ** 3, 1), self.omega_model_cbar.reshape(self.Nx**3, 1), self.omega_DNS_filtered.reshape(self.Nx**3, 1), #self.omega_LES_noModel.reshape(self.Nx**3,1), self.c_plus.reshape(self.Nx**3, 1), self.c_minus.reshape(self.Nx**3, 1) ]) if self.c_analytical is True: # write data to csv file filename = join( self.case, 'filter_width_' + self.filter_type + '_' + str(self.filter_width) + '_analytical.csv') else: # write data to csv file filename = join( self.case, 'filter_width_' + self.filter_type + '_' + str(self.filter_width) + '.csv') self.dataArray_dd = dd.io.from_dask_array( dataArray_da, columns=[ 'c_bar', 'wrinkling', # 'isoArea', # 'wrinkling_LES', # 'wrinkling_reduced', # 'wrinkling_LES_reduced', 'omega_model', 'omega_DNS_filtered', # 'omega_cbar', 'c_plus', 'c_minus' ]) # filter the data set and remove unecessary entries self.dataArray_dd = self.dataArray_dd[ self.dataArray_dd['c_bar'] > 0.01] self.dataArray_dd = self.dataArray_dd[ self.dataArray_dd['c_bar'] < 0.99] if self.case is 'planar_flame_test': self.dataArray_dd = self.dataArray_dd[ self.dataArray_dd['wrinkling'] < 1.1] self.dataArray_dd = self.dataArray_dd[ self.dataArray_dd['wrinkling'] > 0.99] #self.dataArray_dd = self.dataArray_dd[self.dataArray_dd['isoArea'] >= 1.0] # this is to reduce the storage size #self.dataArray_dd = self.dataArray_dd.sample(frac=0.3) print('Computing data array ...') self.dataArray_df = self.dataArray_dd.compute() print('Writing output to csv ...') self.dataArray_df.to_csv(filename, index=False) print('Data has been written.\n\n')
def plsa_em_step_dask( block_rows_ndarray, block_cols_ndarray, block_vals_ndarray, p_w_given_z, p_z_given_d, block_row_size, block_col_size, e_step_thresh=1e-32, ): n_d_blocks = block_rows_ndarray.shape[0] n_w_blocks = block_rows_ndarray.shape[1] n = p_z_given_d.shape[0] m = p_w_given_z.shape[1] k = p_z_given_d.shape[1] result_p_w_given_z = [[] for i in range(n_w_blocks)] result_p_z_given_d = [[] for i in range(n_d_blocks)] result_norm_pwz = [] result_norm_pdz = [[] for i in range(n_d_blocks)] for i in range(n_d_blocks): row_start = block_row_size * i row_end = min(row_start + block_row_size, n) for j in range(n_w_blocks): col_start = block_col_size * j col_end = min(col_start + block_col_size, m) row_block = block_rows_ndarray[i, j] col_block = block_cols_ndarray[i, j] val_block = block_vals_ndarray[i, j] kernel_results = plsa_em_step_block_kernel( row_block, col_block, val_block, p_w_given_z[:, col_start:col_end], p_z_given_d[row_start:row_end, :], e_step_thresh=e_step_thresh, ) result_p_w_given_z[j].append( da.from_delayed(kernel_results[0], (k, block_col_size), dtype=np.float32)) result_p_z_given_d[i].append( da.from_delayed(kernel_results[1], (block_row_size, k), dtype=np.float32)) result_norm_pwz.append( da.from_delayed(kernel_results[2], (k, ), dtype=np.float32)) result_norm_pdz[i].append( da.from_delayed(kernel_results[3], (block_row_size, ), dtype=np.float32)) p_w_given_z_blocks = [ da.dstack(result_p_w_given_z[i]).sum(axis=-1) for i in range(n_w_blocks) ] p_z_given_d_blocks = [ da.dstack(result_p_z_given_d[i]).sum(axis=-1) for i in range(n_d_blocks) ] norm_pdz_blocks = [ da.dstack(result_norm_pdz[i]).sum(axis=-1) for i in range(n_d_blocks) ] p_w_given_z = (da.hstack(p_w_given_z_blocks) / da.dstack(result_norm_pwz).sum(axis=-1).T) p_z_given_d = da.vstack(p_z_given_d_blocks) / da.hstack(norm_pdz_blocks).T result = compute(p_w_given_z, p_z_given_d) return result
def hstack(self, *others, **kwargs): others = tuple(ensure_dask_array(d) for d in others) tup = (self, ) + others out = da.hstack(tup) return view_subclass(out, type(self))
def _expand_tiepoint_array_1km(self, arr, lines, cols): arr = da.repeat(arr, lines, axis=1) arr = da.concatenate((arr[:, :lines//2, :], arr, arr[:, -(lines//2):, :]), axis=1) arr = da.repeat(arr.reshape((-1, self.cscan_full_width - 1)), cols, axis=1) return da.hstack((arr, arr[:, -cols:]))
def _expand_tiepoint_array_5km(self, arr, lines, cols): arr = da.repeat(arr, lines * 2, axis=1) arr = da.repeat(arr.reshape((-1, self.cscan_full_width - 1)), cols, axis=1) return da.hstack((arr[:, :2], arr, arr[:, -2:]))
famfile = "/shared/ukbiobank_filtered/filtered_200k.2.fam" G = read_plink1_bin(bedfile, fam=famfile, verbose=False) n = G.shape[0] p_pheno = 11 p = G.shape[1] + 6 start_ind = (p // size) * rank end_ind = (p // size) * (rank + 1) pheno = genfromtxt("/shared/ukbiobank_filtered/ukb_short.filtered.200k.tab", skip_header=1) if rank != size - 1: X_chunk = G[:, start_ind:end_ind].data.compute() else: X_chunk = da.hstack([G[:,start_ind:].data, da.zeros((n, 6))]).compute() X_chunk[:, -11:] = pheno[:, 1:p_pheno + 1] from utils import impute_na X_chunk = impute_na(X_chunk) # normalize if args.normalize: X_chunk -= X_chunk.mean(0) X_chunk /= X_chunk.std(0) X_chunk = torch.tensor(X_chunk) print(X_chunk.shape) X = THDistMat.from_chunks(X_chunk, force_bycol=True) time = torch.tensor(pheno[:, 12]).view(-1, 1).type(TType)
fPitch = lambda Gxyz: -da.arctan2(Gxyz[0, :], da.sqrt(da.sum(da.square(Gxyz[1:, :]), 0))) fRoll = lambda Gxyz: da.arctan2(Gxyz[1, :], Gxyz[2, :]) # da.arctan2(Gxyz[1,:], da.sqrt(da.sum(da.square(Gxyz[(0,2),:]), 0))) #=da.arctan2(Gxyz[1,:], da.sqrt(da.square(Gxyz[0,:])+da.square(Gxyz[2,:])) ) fInclination = lambda Gxyz: da.arctan2(da.sqrt(da.sum(da.square(Gxyz[:-1, :]), 0)), Gxyz[2, :]) fHeading = lambda H, p, r: da.arctan2(H[2, :] * da.sin(r) - H[1, :] * da.cos(r), H[0, :] * da.cos(p) + (H[1, :] * da.sin(r) + H[2, :] * da.cos(r)) * da.sin(p)) fG = lambda Axyz, Ag, Cg: da.dot(Ag.T, (Axyz - Cg[0, :]).T) fGi = lambda Ax, Ay, Az, Ag, Cg, i: da.dot(Ag.T, (da.column_stack((Ax, Ay, Az))[slice(*i)] - Cg[0, :]).T) fbinningClip = lambda x, bin2_iStEn, bin1_nAverage: da.mean(da.reshape(x[slice(*bin2_iStEn)], (-1, bin1_nAverage)), 1) fbinning = lambda x, bin1_nAverage: da.mean(da.reshape(x, (-1, bin1_nAverage)), 1) repeat3shift1 = lambda A2: [A2[t:(len(A2) - 2 + t)] for t in range(3)] median3cols = lambda a, b, c: da.where(a < b, da.where(c < a, a, da.where(b < c, b, c)), da.where(a < c, a, da.where(c < b, b, c))) median3 = lambda x: da.hstack((np.NaN, median3cols(*repeat3shift1(x)), np.NaN)) # not convertable to dask easily: fVabs_old = lambda Gxyz, kVabs: np.polyval(kVabs.flat, np.sqrt(np.tan(fInclination(Gxyz)))) rep2mean = lambda x, bOk: np.interp(np.arange(len(x)), np.flatnonzero(bOk), x[bOk], np.NaN, np.NaN) fForce2Vabs_fitted = lambda x: da.where(x > 2, 2, da.where(x < 1, 0.25 * x, 0.25 * x + 0.3 * (x - 1) ** 4)) fIncl2Force = lambda incl: da.sqrt(da.tan(incl)) fVabs = lambda Gxyz, kVabs: fForce2Vabs_fitted(fIncl2Force(fInclination(Gxyz))) f = lambda fun, *args: fun(*args) positiveInd = lambda i, L: np.int32(da.where(i < 0, L - i, i)) minInterval = lambda iLims1, iLims2, L: f( lambda iL1, iL2: da.transpose([max(iL1[:, 0], iL2[:, 0]), min(iL1[:, -1], iL2[:, -1])]), positiveInd(iLims1, L), positiveInd(iLims2, L)) fStEn2bool = lambda iStEn, length: da.hstack( [(da.ones(iEn2iSt, dtype=np.bool8) if b else da.zeros(iEn2iSt, dtype=np.bool8)) for iEn2iSt, b in da.vstack(( da.diff( da.hstack(
def hstack(self, *others, **kwargs): others = tuple(ensure_dask_array(d) for d in others) tup = (self,) + others out = da.hstack(tup) return view_subclass(out, type(self))