def momenta(self,chanrange=None,p=1): """abs(intensity)-weighted moment Does somewhat better than signed intensity-weighted moment. Parameters ---------- chanrange: range of channels over which to compute moment [startchan, endchan] p: the moment to compute (the power of the frequency in the sum) Returns: The computed moment """ # get the masked array s = self.spec() chupper = len(s)-1 chanrange = self._sanitizechanrange(chanrange,chupper) sum_s = ma.sum(ma.abs(s[chanrange[0]:chanrange[1]+1])) sum_sf = 0 mean = 0 if p > 1: mean = self.moment(chanrange,p-1) for i in range(chanrange[0],chanrange[1]+1): sum_sf += ma.abs(s[i])*math.pow((self._freq[i]-mean),p) return sum_sf/sum_s
def combine_nights(combined_catalog, filterlist, refcat): header = [ 'BEGIN CATALOG HEADER', 'nfields 13', ' ra 1 0 d degrees %10.6f', ' dec 2 0 d degrees %10.6f', ' id 3 0 c INDEF %3d' ] for filt in filterlist: header.append(' {} {:2d} 0 r INDEF %6.3f'.format( filt, len(header) - 1)) header.append(' {}err {:2d} 0 r INDEF %6.3f'.format( filt, len(header) - 1)) header += ['END CATALOG HEADER', ''] catalog = Table([refcat['ra'], refcat['dec'], refcat['id']], meta={'comments': header}, masked=True) for filt in filterlist: mags = combined_catalog['mag'][combined_catalog['filter'] == filt] median = np.median(mags, axis=0) absdev_mag = mags - median mad = np.median(np.abs(absdev_mag), axis=0) * np.sqrt(pi / 2) mags.mask |= np.abs(absdev_mag) > 5 * mad catalog[filt] = np.median(mags, axis=0) catalog[filt + 'err'] = np.median(np.abs(mags - catalog[filt]), axis=0) * np.sqrt(pi / 2) return catalog
def find_lines(peaks, fwhm, y=None, verbose=False): if y is None: y = np.arange(len(peaks)) # Делаем все строки одинаковой длины (по наидленнейшей) peaks = np.array(list(zip_longest(*peaks)), dtype='float') # if verbose: # plt.plot(peaks.T, y, 'o') # plt.show() msk = np.isnan(peaks) peaks = ma.array(peaks, mask=msk) col = ['C' + str(j) for j in range(9)] # print(len(peaks)) # print() for i in range(len(peaks)): f**k = peaks[i:] line = f**k[0] # msk = np.logical_not(np.isnan(line)) # k = ma.polyfit(y, line, 2) # print(k) est = np.ones(len(y)) * ma.median(line) # est = np.polyval(k, y) err = est - line move_right = ma.filled((err > 5 * ma.median(ma.abs(err))), False) move_left = ma.filled((err < -5 * ma.median(ma.abs(err))), False) not_move = np.logical_not(move_right + move_left) # plt.plot(y[not_move], f**k[0][not_move], '.' + col[i % 9]) # plt.plot(y, est, col[i % 9], ls='--') # plt.plot(y[move_right], f**k[0][move_right], 'x' + col[i % 9]) # plt.plot(y[move_left], f**k[0][move_left], '+' + col[i % 9]) # plt.show() # print(i) # print(ma.mean(ma.abs(err))) # print(ma.median(line)) # print() if np.sum(move_right) > 0: # Те, что меньше медианы (слева) nonearray = ma.array([[None] * np.sum(move_right.astype('int'))], mask=[[True] * np.sum(move_right.astype('int'))]) f**k[:, move_right] = ma.append(f**k[:, move_right][1:, :], nonearray, axis=0) if np.sum(move_left) > 0: nonearray = ma.array([[None] * np.sum(move_left.astype('int'))], mask=[[True] * np.sum(move_left.astype('int'))]) f**k[:, move_left] = ma.append(nonearray, f**k[:, move_left][:-1, :], axis=0) # plt.plot(f**k[0], col[i%9]) peaks[i:] = f**k plt.show() peaks = peaks.T msk = np.isnan(peaks) peaks = ma.array(peaks, mask=msk) good_lines = (np.sum(np.logical_not(msk), axis=0) > len(y) / 4.) peaks = peaks[:, good_lines] return peaks
def fitBearingDrift(self, trajectory, windowSize=None, plotFit=False): lags = np.linspace(0, np.round(50. * trajectory.frameRate), 200) tau = lags / trajectory.frameRate if windowSize is None: psi = unwrapma(trajectory.getMaskedPosture(trajectory.psi)) D = drift(psi, lags, trajectory.excluded) sel = ~ma.getmaskarray(psi) #p = np.polyfit(tau, D, 1) p = np.polyfit(trajectory.t[sel], psi[sel], 1) self.k_psi = p[0] else: def result(traj): psi = traj.getMaskedPosture(traj.psi) if float(len(psi.compressed())) / float(len(psi)) > 0.2: psi = unwrapma(psi) D = drift(psi, lags, traj.excluded) sel = ~ma.getmaskarray(psi) p = np.polyfit(traj.t[sel], psi[sel], 1) return D, p[0] else: return ma.zeros((len(lags), )) * ma.masked, ma.masked results = [ result(traj) for traj in trajectory.asWindows(windowSize) ] D = ma.array([resulti[0] for resulti in results]) k = ma.array([resulti[1] for resulti in results]) self.k_psi = ma.abs(k).mean() D = ma.abs(D).T.mean(axis=1) if plotFit: plt.plot(tau, D, 'k.') plt.plot(tau, self.k_psi * tau, 'r-') plt.xlabel(r'$\tau$ (s)') plt.ylabel(r'$\langle |\psi(\tau) - \psi(0)| \rangle$ (rad)') textstr = r'$\langle|k_\psi|\rangle=%.2f$ rad/s' % (self.k_psi) props = dict(boxstyle='round', facecolor='wheat', alpha=0.5) # place a text box in lower left in axes coords ax = plt.gca() ax.text(0.95, 0.05, textstr, transform=ax.transAxes, fontsize=14, horizontalalignment='right', verticalalignment='bottom', bbox=props) plt.show()
def openImage(nameFile, intersection): # this allows GDAL to throw Python Exceptions gdal.UseExceptions() band_num = 1 #Tvis-animated.gif try: src_ds = gdal.Open(nameFile) except (RuntimeError, e): print('Unable to open INPUT.tif') print(e) sys.exit(1) try: srcband = src_ds.GetRasterBand(band_num) except (RuntimeError, e): # for example, try GetRasterBand(10) print('Band ( %i ) not found' % band_num) print(e) sys.exit(1) gt1 = src_ds.GetGeoTransform() Project = src_ds.GetProjection() print(nameFile) print(gt1) xOrigin = gt1[0] yOrigin = gt1[3] pixelWidth = float(gt1[1]) pixelHeight = float(gt1[5]) xmin = intersection[0] ymax = intersection[1] xoff = int((xmin - xOrigin) / pixelWidth) yoff = int((yOrigin - ymax) / pixelWidth) xcount = int( (np.abs(intersection[0]) - np.abs(intersection[2])) / pixelWidth) ycount = int( (np.abs(intersection[1]) - np.abs(intersection[3])) / pixelHeight) #print nameFile if (xoff == 0 and yoff == 0): lc = srcband.ReadAsArray() else: #print nameFile #print xoff #print yoff #print xcount #print ycount #if "MODIS" in nameFile: #lc = srcband.ReadAsArray(xoff, yoff, int(xcount), int(ycount)) #else: lc = srcband.ReadAsArray(xoff, yoff, int(xcount), int(ycount)) return src_ds, lc, gt1, Project
def _calc_correlation(self, values_1, values_2, conf_level=0.95): """ Calculates Pearson's correlation coeffcient. Arguments: values_1 -- first data values_2 -- second data conf_level -- confidence level Returns: (corr_coeff, significance) -- correlation coefficient and significance arrays """ n_samples = values_1.shape[0] # Sample length # Calculate Pearson's correlatiob coefficient values_cov = ma.sum((values_1 - ma.mean(values_1, axis=0)) * (values_2 - ma.mean(values_2, axis=0)), axis=0) corr_coef = values_cov / (ma.std(values_1, axis=0) * ma.std(values_2, axis=0)) / n_samples # Calculate significance using t-distribution with n-2 degrees of freedom. deg_fr = n_samples - 2 # Degrees of freedom. t_distr = ma.abs( corr_coef * ma.sqrt(deg_fr / (1. - corr_coef**2))) # Student's t-distribution. prob = 0.5 + conf_level / 2 # Probability for two tails. cr_value = student_t.ppf(prob, deg_fr) # Student's Critical value. significance = ma.greater(t_distr, cr_value) return corr_coef, significance
def ma_mad(x, axis=None): """Median absolute deviation""" median_x = ma.median(x, axis=axis) if axis is not None: median_x = ma.expand_dims(median_x, axis=axis) return ma.median(ma.abs(x - median_x), axis=axis)
def ray_intersect(self, orig, direction) -> ma.array: normal = np.cross(self.a, self.b) ndotray = ma.array(v3_dots(normal, direction)) ndotray.mask = ma.abs(ndotray) < 1e-3 d = v3_dots(normal, orig) t = (v3_dots(normal, self.corner) + d) / ndotray t.mask |= t < 0 pt = orig + t.reshape(-1, 1) * direction # TODO: try an alternative to find points in a parallelogram (or plane) # if that's easier. Maybe project onto a plane then use a parallelogram # as a basis in 2d? # https://stackoverflow.com/questions/59128744/raytracing-ray-vs-parallelogram # edge 0 t.mask |= v3_dots(normal, np.cross(self.a, pt - self.corner)) < 0 # edge 1 t.mask |= v3_dots(normal, np.cross(pt - self.corner, self.b)) < 0 # edge 2 t.mask |= v3_dots(normal, np.cross(pt - self.corner - self.b, self.a)) < 0 # edge 3 t.mask |= v3_dots(normal, np.cross(self.b, pt - self.corner - self.a)) < 0 return t
def psi_trend(traj, maxMissing): D = ma.array([tsstats.drift(tsstats.unwrapma(trajw.getMaskedPosture(trajw.psi)), lags, exclude=trajw.excluded) for trajw in traj.asWindows(100.) if fractionMissing(trajw)>maxMissing]) D[np.isnan(D)] = ma.masked D = ma.abs(D).mean(axis=0) if len(D) == 0: D = ma.zeros((len(lags),)) D[:] = ma.masked return D
def tdpep(t,fm,PG0): """ Transit-duration - Period - Epoch Parameters ---------- fm : Flux with bad data points masked out. It is assumed that elements of f are evenly spaced in time. PG0 : Initial period grid. Returns ------- epoch2d : Grid (twd,P) of best epoch df2d : Grid (twd,P) of depth epoch count2d : number of filled data for particular (twd,P) noise : Grid (twd) typical scatter PG : The Period grid twd : Grid of trial transit widths. """ assert fm.fill_value ==0 # Determine the grid of periods that corresponds to integer # multiples of cadence values PcadG,PG = P2Pcad(PG0) # Initialize tdur grid. twdMi = a2tdur( P2a( PG[0 ] ) ) /keptoy.lc twdMa = a2tdur( P2a( PG[-1] ) ) /keptoy.lc twdG = np.round(np.linspace(twdMi,twdMa,4)).astype(int) rec2d = [] noise = [] for twd in twdG: dM = mtd(t,fm.filled(),twd) dM.mask = fm.mask | ~isfilled(t,fm,twd) rec2d.append( pep(t[0],dM,PcadG) ) # Noise per transit mad = ma.abs(dM) mad = ma.median(mad) noise.append(mad) rec2d = np.vstack(rec2d) make2d = lambda x : np.tile( np.vstack(x), (1,rec2d.shape[1] )) rec2d = mlab.rec_append_fields(rec2d,'noise',make2d(noise)) rec2d = mlab.rec_append_fields(rec2d,'twd', make2d(twdG)) PG = np.tile( PG, (rec2d.shape[0],1 )) rec2d = mlab.rec_append_fields(rec2d,'PG',PG) s2n = rec2d['fom']/rec2d['noise']*rec2d['count'] rec2d = mlab.rec_append_fields(rec2d,'s2n', s2n ) return rec2d
def combine_nights(combined_catalog, filterlist, refcat): header = ['BEGIN CATALOG HEADER', 'nfields 13', ' ra 1 0 d degrees %10.6f', ' dec 2 0 d degrees %10.6f', ' id 3 0 c INDEF %3d'] for filt in filterlist: header.append(' {} {:2d} 0 r INDEF %6.3f'.format(filt, len(header) - 1)) header.append(' {}err {:2d} 0 r INDEF %6.3f'.format(filt, len(header) - 1)) header += ['END CATALOG HEADER', ''] catalog = Table([refcat['ra'], refcat['dec'], refcat['id']], meta={'comments': header}, masked=True) for filt in filterlist: mags = combined_catalog['mag'][combined_catalog['filter'] == filt] median = np.median(mags, axis=0) absdev_mag = mags - median mad = np.median(np.abs(absdev_mag), axis=0) * np.sqrt(pi / 2) mags.mask |= np.abs(absdev_mag) > 5 * mad catalog[filt] = np.median(mags, axis=0) catalog[filt+'err'] = np.median(np.abs(mags - catalog[filt]), axis=0) * np.sqrt(pi / 2) return catalog
def fitBearingDrift(self, trajectory, windowSize=None, plotFit=False): lags = np.linspace(0, np.round(50.*trajectory.frameRate), 200) tau = lags / trajectory.frameRate if windowSize is None: psi = unwrapma(trajectory.getMaskedPosture(trajectory.psi)) D = drift(psi, lags, trajectory.excluded) sel = ~ma.getmaskarray(psi) #p = np.polyfit(tau, D, 1) p = np.polyfit(trajectory.t[sel], psi[sel], 1) self.k_psi = p[0] else: def result(traj): psi = traj.getMaskedPosture(traj.psi) if float(len(psi.compressed()))/float(len(psi)) > 0.2: psi = unwrapma(psi) D = drift(psi, lags, traj.excluded) sel = ~ma.getmaskarray(psi) p = np.polyfit(traj.t[sel], psi[sel], 1) return D, p[0] else: return ma.zeros((len(lags),))*ma.masked, ma.masked results = [result(traj) for traj in trajectory.asWindows(windowSize)] D = ma.array([resulti[0] for resulti in results]) k = ma.array([resulti[1] for resulti in results]) self.k_psi = ma.abs(k).mean() D = ma.abs(D).T.mean(axis=1) if plotFit: plt.plot(tau, D, 'k.') plt.plot(tau, self.k_psi*tau, 'r-') plt.xlabel(r'$\tau$ (s)') plt.ylabel(r'$\langle |\psi(\tau) - \psi(0)| \rangle$ (rad)') textstr = r'$\langle|k_\psi|\rangle=%.2f$ rad/s'%(self.k_psi) props = dict(boxstyle='round', facecolor='wheat', alpha=0.5) # place a text box in lower left in axes coords ax = plt.gca() ax.text(0.95, 0.05, textstr, transform=ax.transAxes, fontsize=14, horizontalalignment='right', verticalalignment='bottom', bbox=props) plt.show()
def s2n_known(d, t, fm): """ Compute the FFA S/N assuming we knew where the signal was before hand Parameters ---------- d : Simulation Parameters t : time fm : lightcurve (maksed) """ # Compute the twd from the twdG with the closest to the injected tdur tdur = keptoy.tdurMA(d) itwd_close = np.argmin(np.abs(np.array(config.twdG) - tdur / config.lc)) twd_close = config.twdG[itwd_close] dM = tfind.mtd(t, fm, twd_close) # Fold the LC on the closest period we're computing all the FFA # periods around the P closest. This is a little complicated, but # it allows us to use the same functions as we do in the complete # period scan. Pcad0 = int(np.floor(d['P'] / config.lc)) t0cad, Pcad, meanF, countF = tfind.fold(dM, Pcad0) iPcad_close = np.argmin(np.abs(Pcad - d['P'] / config.lc)) Pcad_close = Pcad[iPcad_close] P_close = Pcad_close * config.lc # Find the epoch that is closest to the injected phase. t0 = t[0] + t0cad * config.lc # Epochs in days. phase = np.mod(t0, P_close) / P_close # Phases [0,1) # The following 3 lines compute the difference between the FFA # phases and the injected phase. Phases can be measured going # clockwise or counter clockwise, so we must choose the minmum # value. No phases are more distant than 0.5. dP = np.abs(phase - d['phase']) dP = np.vstack([dP, 1 - dP]) dP = dP.min(axis=0) iphase_close = np.argmin(dP) phase_close = phase[iphase_close] # s2n for the closest twd,P,phas noise = ma.median(ma.abs(dM)) s2nF = meanF / noise * np.sqrt(countF) s2nP = s2nF[iPcad_close] # length Pcad0 array with s2n for all the # P at P_closest s2n_close = s2nP[iphase_close] return phase, s2nP, twd_close, P_close, phase_close, s2n_close
def my_polyfit(x, y, deg, degatt=0): k = ma.polyfit(x, y, degatt) res = my_poly(k, x) resid = ma.abs(res - y) medresid = ma.median(resid, axis=0) if y.ndim != 1: mask = ma.logical_not(ma.sum((resid > 3 * medresid), axis=1).astype('bool')) else: mask = (resid < 3 * medresid) y = y[mask] x = x[mask] k = ma.polyfit(x, y, deg) return(k, mask)
def transform_non_affine(self, a): # NOTE: Critical to truncate valid range inside transform *and* # in limit_range_for_scale or get weird duplicate tick labels. This # is not necessary for positive-only scales because it is harder to # run up right against the scale boundaries. with np.errstate(divide='ignore', invalid='ignore'): m = ma.masked_where((a <= -90) | (a >= 90), a) if m.mask.any(): m = np.deg2rad(m) return ma.log(ma.abs(ma.tan(m) + 1 / ma.cos(m))) else: a = np.deg2rad(a) return np.log(np.abs(np.tan(a) + 1 / np.cos(a)))
def psi_trend(traj, maxMissing): D = ma.array([ tsstats.drift(tsstats.unwrapma(trajw.getMaskedPosture(trajw.psi)), lags, exclude=trajw.excluded) for trajw in traj.asWindows(100.) if fractionMissing(trajw) > maxMissing ]) D[np.isnan(D)] = ma.masked D = ma.abs(D).mean(axis=0) if len(D) == 0: D = ma.zeros((len(lags), )) D[:] = ma.masked return D
def tdpep(t, fm, par): """ Transit-duration - Period - Epoch Parameters ---------- fm : Flux with bad data points masked out. It is assumed that elements of f are evenly spaced in time. P1 : First period (cadences) P2 : Last period (cadences) twdG : Grid of transit durations (cadences) Returns ------- rtd : 2-D record array with the following fields at every trial (twd,Pcad): - noise - s2n - twd - fields in rep """ PcadG = np.arange(par['Pcad1'], par['Pcad2']) twdG = par['twdG'] assert fm.fill_value == 0 # Determine the grid of periods that corresponds to integer # multiples of cadence values try: ntwd = len(twdG) except: ntwd = 1 rtd = [] for i in range(ntwd): # Loop over twd twd = np.int(twdG[i]) dM = mtd(fm, twd) func = lambda Pcad: ep(dM, Pcad) rep = list(map(func, PcadG)) rep = np.hstack(rep) r = np.empty(rep.size, dtype=tddtype) for k in epdtype.names: r[k] = rep[k] r['noise'] = ma.median(ma.abs(dM)) r['twd'] = twd r['t0'] = r['t0cad'] * config.lc + t[0] rtd.append(r) rtd = np.vstack(rtd) rtd['s2n'] = rtd['mean'] / rtd['noise'] * np.sqrt(rtd['count']) return rtd
def calc_directionality(dat, dist_range): nbin = len(dat) min_d,max_d = dist_range if min_d < 0 or max_d < min_d: raise ValueError('calc_insulation() requires 0 <= min_d <= max_d') directionality = ma.zeros(nbin) for i in xrange(nbin): if i < max_d or i >= nbin-max_d: directionality[i] = 0.0 else: up = dat[i,(i-max_d):(i-min_d)].sum() down = dat[i,(i+min_d):(i+max_d)].sum() avg = (up+down)/2.0 directionality[i] = (up-down)/ma.abs(up-down)*((up-avg)**2/avg + (down-avg)**2/avg) return directionality
def _find_fog_profiles( self, n_gates_for_signal_sum: int = 20, signal_sum_threshold: float = 1e-3, variance_threshold: float = 1e-15, ) -> np.ndarray: """Finds saturated (usually fog) profiles from beta_raw.""" signal_sum = ma.sum(ma.abs( self.data["beta_raw"][:, :n_gates_for_signal_sum]), axis=1) variance = _calc_var_from_top_gates(self.data["beta_raw"]) is_fog = (signal_sum > signal_sum_threshold) | (variance < variance_threshold) logging.info(f"Cleaned {sum(is_fog)} profiles with fog filter") return is_fog
def iterative_correction(self, max_iter=100, tolerance=1e-5): totalBias = ma.ones(self.nbin, float) mat = self.dat.copy() for r in xrange(max_iter): print('.', end='', file=sys.stderr) binSum = mat.sum(axis=1) mask = binSum==0 bias = binSum/binSum[~mask].mean() bias[mask] = 1 bias -= 1 bias *= 0.8 bias += 1 totalBias *= bias biasMat = bias.reshape(1,len(bias)) * bias.reshape(len(bias),1) mat = mat / biasMat if ma.abs(bias-1).max() < tolerance: break self.dat = mat corr = totalBias[~mask].mean() self.bias = totalBias/corr self.corrected |= 0x1
def identifyReversals(self, transitionWindow=2.): dpsi = self.getMaskedPosture(self.dpsi) rev = ma.abs(dpsi) > np.pi / 2. inRev = False state = np.zeros(self.t.shape, int) state[~rev] = 1 state[rev] = 2 state[ma.getmaskarray(rev)] = 0 self.state = state revBoundaries = [] currentRev = ma.zeros((2, )) for j in xrange(1, self.t.shape[0] - 2): if not inRev: if ((state[j] == 2 & state[j + 1] == 2) | (state[j] == 2 & state[j + 1] == 0 & state[j + 2] == 2)): if state[j - 1] == 0: currentRev[0] = ma.masked else: currentRev[0] = j inRev = True else: if ((state[j] == 1 & state[j + 1] == 1) | (state[j] == 1 & state[j + 1] == 0 & state[j + 2] == 1)): currentRev[1] = j inRev = False revBoundaries.append(currentRev.copy()) elif (state[j] == 0 & state[j + 1] == 0): currentRev[1] = ma.masked inRev = False revBoundaries.append(currentRev.copy()) self.revBoundaries = ma.array(revBoundaries, dtype='int') revEdges = self.revBoundaries[:].compressed() for boundary in revEdges: self.nearRev[ (self.t > boundary / self.frameRate - transitionWindow / 2.) & (self.t < boundary / self.frameRate + transitionWindow / 2.)] = True
def plot_glider(cube, cmap=plt.cm.viridis, figsize=(9, 3.75), track_inset=False): data = apply_range(cube) x = apply_range(cube.coord(axis="X")) y = apply_range(cube.coord(axis="Y")) z = apply_range(cube.coord(axis="Z")) t = cube.coord(axis="T") t = t.units.num2date(t.points.squeeze()) fig, ax = plt.subplots(figsize=figsize) dist = distance(x, y) z = ma.abs(z) dist, _ = np.broadcast_arrays(dist[..., np.newaxis], z.filled(fill_value=np.NaN)) dist, z = map(ma.masked_invalid, (dist, z)) cs = ax.pcolor(dist, z, data, cmap=cmap, snap=True) kw = dict(orientation="horizontal", extend="both", shrink=0.65) cbar = fig.colorbar(cs, **kw) if track_inset: axin = inset_axes( ax, width=2, height=2, loc=4, bbox_to_anchor=(1.15, 0.35), bbox_transform=ax.figure.transFigure, ) axin.plot(x, y, "k.") start, end = (x[0], y[0]), (x[-1], y[-1]) kw = dict(marker="o", linestyle="none") axin.plot(*start, color="g", **kw) axin.plot(*end, color="r", **kw) axin.axis("off") ax.invert_yaxis() ax.invert_xaxis() ax.set_xlabel("Distance (km)") ax.set_ylabel("Depth (m)") return fig, ax, cbar
def identifyReversals(self, transitionWindow=2.): dpsi = self.getMaskedPosture(self.dpsi) rev = ma.abs(dpsi)>np.pi/2. inRev = False state = np.zeros(self.t.shape, int) state[~rev] = 1 state[rev] = 2 state[ma.getmaskarray(rev)] = 0 self.state = state revBoundaries = [] currentRev = ma.zeros((2,)) for j in xrange(1,self.t.shape[0]-2): if not inRev: if ((state[j]==2 & state[j+1]==2) | (state[j]==2 & state[j+1]==0 & state[j+2]==2)): if state[j-1] == 0: currentRev[0] = ma.masked else: currentRev[0] = j inRev = True else: if ((state[j]==1 & state[j+1]==1) | (state[j]==1 & state[j+1]==0 & state[j+2]==1)): currentRev[1] = j inRev = False revBoundaries.append(currentRev.copy()) elif (state[j]==0 & state[j+1]==0): currentRev[1] = ma.masked inRev = False revBoundaries.append(currentRev.copy()) self.revBoundaries = ma.array(revBoundaries, dtype='int') revEdges = self.revBoundaries[:].compressed() for boundary in revEdges: self.nearRev[(self.t>boundary/self.frameRate-transitionWindow/2.) & (self.t<boundary/self.frameRate+transitionWindow/2.)] = True
def noiseyReg(t, dM, thresh=2): """ Noisey Region If certain regions are much noisier than others, we should remove them. A typical value of the noise is computed using a median absolute deviation (MAD) on individual regions. If certain regions are noiser by thresh, we mask them out. Parameters ---------- dM : Single event statistic (masked array) thresh : If region has MAD > thresh * typical MAD throw it out. """ tm = ma.masked_array(t, mask=dM.mask) label = detrend.sepseg(tm) sL = ma.notmasked_contiguous(label) madseg = np.array([ma.median(ma.abs(dM[s])) for s in sL]) madLC = np.median(madseg) # Typical MAD of the light curve. isNoisey = np.zeros(tm.size).astype(bool) sNL = [] # List to keep track of the noisey segments for s, mads in zip(sL, madseg): if mads > thresh * madLC: isNoisey[s] = True sNL.append(s) if len(sNL) != 0: print "Removed following time ranges because data is noisey" print "----------------------------------------------------" for s in sNL: print "%.2f %.2f " % (t[s.start], t[s.stop - 1]) return isNoisey
def peak(self,chanrange=None): """Return the peak intensity in the given channel range If a mask exists, this function operates on the masked spectrum. Parameters ---------- chanrange: range of channels over which to compute dispersion [startchan, endchan] Returns ---------- Maximum of the absolute value of the spectrum in the channel range max(abs(spectrum[startchan:endchan])) """ s = self.spec() chupper = len(s)-1 chanrange = self._sanitizechanrange(chanrange,chupper) # Handle one-channel ranges. if (chanrange[0] == chanrange[1]): return s[chanrange[0]] return ma.max(ma.abs(s[chanrange[0]:chanrange[1]]))
if np.all(group['dc1'][f0]): dc0 = np.sum(np.power(group['dc1'][f0], -2))**-0.5 c0 = np.sum( group['c1'][f0] * np.power(group['dc1'][f0], -2)) * dc0**2 else: dc0 = 0. c0 = np.mean(group['c1'][f0]) if np.all(group['dc2'][f1]): dc1 = np.sum(np.power(group['dc2'][f1], -2))**-0.5 c1 = np.sum( group['c2'][f1] * np.power(group['dc2'][f1], -2)) * dc1**2 else: dc1 = 0. c1 = np.mean(group['c2'][f1]) color = np.divide(m0 - m1 + z0 - z1, 1 - c0 + c1) dcolor = np.abs(color) * np.sqrt( np.divide(dm0**2 + dm1**2 + dz0**2 + dz1**2, (m0 - m1 + z0 - z1)**2) + np.divide(dc0**2 + dc1**2, (1 - c0 + c1)**2)) for row in group: colors.append(color) dcolors.append(dcolor) targets[filters] = np.array(colors) targets['d' + filters] = np.array(dcolors) # calibrate all the instrumental magnitudes zcol = [ color_to_use[row['filter']][0] if color_to_use[row['filter']] else row['filter'] * 2 for row in targets ]
def manhattan(a, b): """ Manhattan distance between a and b """ return ma.abs(a.real - b.real) + ma.abs(a.imag - b.imag)
def euclid(a,b): """ Euclidian distance between a and b """ return ma.abs(a - b)
def __call__(self, value, clip=None): # read in parameters method = self.stretch exponent = self.exponent midpoint = self.midpoint # ORIGINAL MATPLOTLIB CODE if clip is None: clip = self.clip if np.iterable(value): vtype = 'array' val = ma.asarray(value).astype(np.float) else: vtype = 'scalar' val = ma.array([value]).astype(np.float) self.autoscale_None(val) vmin, vmax = self.vmin, self.vmax if vmin > vmax: raise ValueError("minvalue must be less than or equal to maxvalue") elif vmin == vmax: return 0.0 * val else: if clip: mask = ma.getmask(val) val = ma.array(np.clip(val.filled(vmax), vmin, vmax), mask=mask) result = (val - vmin) * (1.0 / (vmax - vmin)) # CUSTOM APLPY CODE # Keep track of negative values negative = result < 0. if self.stretch == 'Linear': pass elif self.stretch == 'Log': # result = np.log(result * (self.midpoint - 1.) + 1.) \ # / np.log(self.midpoint) result = ma.log10(result * (self.midpoint - 1.) + 1.) \ / ma.log10(self.midpoint) elif self.stretch == 'Sqrt': result = ma.sqrt(ma.abs(result)) elif self.stretch == 'Arcsinh': result = ma.arcsinh(result / self.midpoint) \ / ma.arcsinh(1. / self.midpoint) elif self.stretch == 'Arccosh': result = ma.arccosh(result / self.midpoint) \ / ma.arccosh(1. / self.midpoint) elif self.stretch == 'Power': result = ma.power(result, exponent) elif self.stretch == 'Exp': result = np.exp(result) else: raise Exception("Unknown stretch in APLpyNormalize: %s" % self.stretch) # Now set previously negative values to 0, as these are # different from true NaN values in the FITS image result[negative] = -np.inf if vtype == 'Scalar': result = result[0] return result
def measure(mode, x, y, x0, x1, thresh=0): """ return the a measure of y in the window x0 to x1 """ xm = ma.masked_outside(x, x0, x1) # .compressed() ym = ma.array(y, mask=ma.getmask(xm)) # .compressed() if mode == 'mean': r1 = np.mean(ym) r2 = np.std(ym) if mode == 'max' or mode == 'maximum': r1 = ma.max(ym) r2 = xm[ma.argmax(ym)] if mode == 'min' or mode == 'minimum': r1 = ma.min(ym) r2 = xm[ma.argmin(ym)] if mode == 'minormax': r1p = ma.max(ym) r1n = ma.min(ym) if ma.abs(r1p) > ma.abs(r1n): r1 = r1p r2 = xm[ma.argmax(ym)] else: r1 = r1n r2 = xm[ma.argmin(ym)] if mode == 'median': r1 = ma.median(ym) r2 = 0 if mode == 'p2p': # peak to peak r1 = ma.ptp(ym) r2 = 0 if mode == 'std': # standard deviation r1 = ma.std(ym) r2 = 0 if mode == 'var': # variance r1 = ma.var(ym) r2 = 0 if mode == 'cumsum': # cumulative sum r1 = ma.cumsum(ym) # Note: returns an array r2 = 0 if mode == 'anom': # anomalies = difference from averge r1 = ma.anom(ym) # returns an array r2 = 0 if mode == 'sum': r1 = ma.sum(ym) r2 = 0 if mode == 'area' or mode == 'charge': r1 = ma.sum(ym) / (ma.max(xm) - ma.min(xm)) r2 = 0 if mode == 'latency': # return first point that is > threshold sm = ma.nonzero(ym > thresh) r1 = -1 # use this to indicate no event detected r2 = 0 if ma.count(sm) > 0: r1 = sm[0][0] r2 = len(sm[0]) if mode == '1090': #measure 10-90% time, also returns max r1 = ma.max(ym) r2 = xm[ma.argmax(ym)] y10 = 0.1 * r1 y90 = 0.9 * r1 sm1 = ma.nonzero(ym >= y10) sm9 = ma.nonzero(ym >= y90) r1 = xm[sm9] - xm[sm1] if mode == 'count': r1 = ma.count(ym) r2 = 0 if mode == 'maxslope': return (0, 0) slope = np.array([]) win = ma.flatnotmasked_contiguous(ym) st = int(len(win) / 20) # look over small ranges for k in win: # move through the slope measurementwindow tb = range(k - st, k + st) # get tb array newa = np.array(self.dat[i][j, thisaxis, tb]) ppars = np.polyfit( x[tb], ym[tb], 1) # do a linear fit - smooths the slope measures slope = np.append(slope, ppars[0]) # keep track of max slope r1 = np.amax(slope) r2 = np.argmax(slope) return (r1, r2)
z0, dz0 = average_in_flux(group['z1'][f0], group['dz1'][f0]) z1, dz1 = average_in_flux(group['z2'][f1], group['dz2'][f1]) if np.all(group['dc1'][f0]): dc0 = np.sum(np.power(group['dc1'][f0], -2))**-0.5 c0 = np.sum(group['c1'][f0] * np.power(group['dc1'][f0], -2)) * dc0**2 else: dc0 = 0. c0 = np.mean(group['c1'][f0]) if np.all(group['dc2'][f1]): dc1 = np.sum(np.power(group['dc2'][f1], -2))**-0.5 c1 = np.sum(group['c2'][f1] * np.power(group['dc2'][f1], -2)) * dc1**2 else: dc1 = 0. c1 = np.mean(group['c2'][f1]) color = np.divide(m0 - m1 + z0 - z1, 1 - c0 + c1) dcolor = np.abs(color) * np.sqrt( np.divide(dm0**2 + dm1**2 + dz0**2 + dz1**2, (m0 - m1 + z0 - z1)**2) + np.divide(dc0**2 + dc1**2, (1 - c0 + c1)**2) ) for row in group: colors.append(color) dcolors.append(dcolor) targets[filters] = np.array(colors) targets['d'+filters] = np.array(dcolors) # calibrate all the instrumental magnitudes zcol = [color_to_use[row['filter']][0] if color_to_use[row['filter']] else row['filter']*2 for row in targets] zeropoint = np.choose(zcol == targets['zcol1'], [targets['z2'], targets['z1']]) dzeropoint = np.choose(zcol == targets['zcol1'], [targets['dz2'], targets['dz1']]) colorterm = np.choose(zcol == targets['zcol1'], [targets['c2'], targets['c1']]) dcolorterm = np.choose(zcol == targets['zcol1'], [targets['dc2'], targets['dc1']])
def pgram_max(t,fm,par): """ Periodogram: Check max values Computes s2n for range of P, t0, and twd. However, for every putative transit, we evaluate the transit depth having cut the deepest transit and the second deepest transit. We require that the mean depth after having cut the test max two values not be too much smaller. Good at removing locations with 2 outliers. Parameters ---------- t : t[0] provides starting time fm : masked array with fluxes par : dict with following keys - Pcad1 (lower period limit) - Pcad2 (upper period limit) - twdG (grid of trial durations to compute) Returns ------- pgram : Record array with following fields """ ncad = fm.size PcadG = np.arange(par['Pcad1'],par['Pcad2']) get_frac_Pcad = lambda P : np.arange(P,P+1,1.0*P / ncad) PcadG = np.hstack(map(get_frac_Pcad,PcadG)) twdG = par['twdG'] icad = np.arange(ncad) dtype_pgram = [ ('Pcad',float), ('twd',float), ('s2n',float), ('c',float), ('mean',float), ('t0',float), ('noise',float), ] pgram = np.zeros( (len(twdG),len(PcadG)),dtype=dtype_pgram) # dtype of the record array returned from tdpep() dtype_temp = [ ('c',int), ('mean',float), ('s2n',float), ('col',int), ('t0',float) ] # Loop over different transit durations for itwd,twd in enumerate(twdG): res = foreman_mackey_1d(fm,twd) dM = ma.masked_array( res['depth_1d'], ~res['good_trans'].astype(bool), fill_value=0 ) # Compute noise (robust, based on MAD) on twd timescales noise = ma.median( ma.abs(dM) ) * 1.5 pgram[itwd,:]['noise'] = noise pgram[itwd,:]['twd'] = twd for iPcad,Pcad in enumerate(PcadG): # Compute row and columns for folded data row,col = wrap_icad(icad,Pcad) ncol = np.max(col) + 1 nrow = np.max(row) + 1 icol = np.arange(ncol) # Shove data and mask into appropriate positions data = np.zeros((nrow,ncol)) mask = np.ones((nrow,ncol)).astype(bool) data[row,col] = dM.data mask[row,col] = dM.mask # Sum along columns (clipping top 0, 1, 2 values) datasum,datacnt = cumsum_top(data,mask,2) # datasum[-1] are the is the summed columns having not # clipped any values. Update results array. For t0, add # half the transit with because column index corresponds # in ingress s = datasum[-1,:] c = datacnt[-1,:] r = np.zeros(ncol,dtype=dtype_temp) r['s2n'] = -1 r['mean'] = s/c r['s2n'] = s / np.sqrt(c) / noise r['c'][:] = c r['col'] = icol r['t0'] = ( r['col'] + twd / 2.0) * config.lc + t[0] # Compute mean transit depth after removing the deepest # transit, datacnt[-2], and the second deepest transit, # datacnt[-3]. The mean transit depth must be > 0.5 it's # former value. Also, require 3 transits. mean_clip1 = datasum[-2] / datacnt[-2] mean_clip2 = datasum[-3] / datacnt[-3] b = ( (mean_clip1 > 0.5 * r['mean'] ) & (mean_clip2 > 0.5 * r['mean']) & (r['c'] >= 3) ) if ~np.any(b): continue rcut = r[b] rmax = rcut[np.argmax(rcut['s2n'])] names = ['mean','s2n','c','t0'] for n in names: pgram[itwd,iPcad][n] = rmax[n] pgram[itwd,iPcad]['Pcad'] = Pcad # Compute the maximum return twd with the maximum s2n pgram = pgram[np.argmax(pgram['s2n'],axis=0),np.arange(pgram.shape[1])] return pgram
def measure(mode, x, y, x0, x1, thresh=0): """ return the a measure of y in the window x0 to x1 """ xm = ma.masked_outside(x, x0, x1) # .compressed() ym = ma.array(y, mask=ma.getmask(xm)) # .compressed() if mode == "mean": r1 = np.mean(ym) r2 = np.std(ym) if mode == "max" or mode == "maximum": r1 = ma.max(ym) r2 = xm[ma.argmax(ym)] if mode == "min" or mode == "minimum": r1 = ma.min(ym) r2 = xm[ma.argmin(ym)] if mode == "minormax": r1p = ma.max(ym) r1n = ma.min(ym) if ma.abs(r1p) > ma.abs(r1n): r1 = r1p r2 = xm[ma.argmax(ym)] else: r1 = r1n r2 = xm[ma.argmin(ym)] if mode == "median": r1 = ma.median(ym) r2 = 0 if mode == "p2p": # peak to peak r1 = ma.ptp(ym) r2 = 0 if mode == "std": # standard deviation r1 = ma.std(ym) r2 = 0 if mode == "var": # variance r1 = ma.var(ym) r2 = 0 if mode == "cumsum": # cumulative sum r1 = ma.cumsum(ym) # Note: returns an array r2 = 0 if mode == "anom": # anomalies = difference from averge r1 = ma.anom(ym) # returns an array r2 = 0 if mode == "sum": r1 = ma.sum(ym) r2 = 0 if mode == "area" or mode == "charge": r1 = ma.sum(ym) / (ma.max(xm) - ma.min(xm)) r2 = 0 if mode == "latency": # return first point that is > threshold sm = ma.nonzero(ym > thresh) r1 = -1 # use this to indicate no event detected r2 = 0 if ma.count(sm) > 0: r1 = sm[0][0] r2 = len(sm[0]) if mode == "1090": # measure 10-90% time, also returns max r1 = ma.max(ym) r2 = xm[ma.argmax(ym)] y10 = 0.1 * r1 y90 = 0.9 * r1 sm1 = ma.nonzero(ym >= y10) sm9 = ma.nonzero(ym >= y90) r1 = xm[sm9] - xm[sm1] if mode == "count": r1 = ma.count(ym) r2 = 0 if mode == "maxslope": return (0, 0) slope = np.array([]) win = ma.flatnotmasked_contiguous(ym) st = int(len(win) / 20) # look over small ranges for k in win: # move through the slope measurementwindow tb = range(k - st, k + st) # get tb array newa = np.array(self.dat[i][j, thisaxis, tb]) ppars = np.polyfit(x[tb], ym[tb], 1) # do a linear fit - smooths the slope measures slope = np.append(slope, ppars[0]) # keep track of max slope r1 = np.amax(slope) r2 = np.argmax(slope) return (r1, r2)
def measure(mode, x, y, x0, x1, thresh=0, slopewin=1.0): """ return the a measure of y in the window x0 to x1 """ xm = ma.masked_outside(x, x0, x1)# .compressed() ym = ma.array(y, mask = ma.getmask(xm))# .compressed() if mode == 'mean': r1 = np.mean(ym) r2 = np.std(ym) if mode == 'max' or mode == 'maximum': r1 = ma.max(ym) r2 = xm[ma.argmax(ym)] if mode == 'min' or mode == 'minimum': r1 = ma.min(ym) r2 = xm[ma.argmin(ym)] if mode == 'minormax': r1p = ma.max(ym) r1n = ma.min(ym) if ma.abs(r1p) > ma.abs(r1n): r1 = r1p r2 = xm[ma.argmax(ym)] else: r1 = r1n r2 = xm[ma.argmin(ym)] if mode == 'median': r1 = ma.median(ym) r2 = 0 if mode == 'p2p': # peak to peak r1 = ma.ptp(ym) r2 = 0 if mode == 'std': # standard deviation r1 = ma.std(ym) r2 = 0 if mode == 'var': # variance r1 = ma.var(ym) r2 = 0 if mode == 'cumsum': # cumulative sum r1 = ma.cumsum(ym) # Note: returns an array r2 = 0 if mode == 'anom': # anomalies = difference from averge r1 = ma.anom(ym) # returns an array r2 = 0 if mode == 'sum': r1 = ma.sum(ym) r2 = 0 if mode == 'area' or mode == 'charge': r1 = ma.sum(ym)/(ma.max(xm)-ma.min(xm)) r2 = 0 if mode == 'latency': # return first point that is > threshold sm = ma.nonzero(ym > thresh) r1 = -1 # use this to indicate no event detected r2 = 0 if ma.count(sm) > 0: r1 = sm[0][0] r2 = len(sm[0]) if mode == '1090': #measure 10-90% time, also returns max r1 = ma.max(ym) r2 = xm[ma.argmax(ym)] y10 = 0.1*r1 y90 = 0.9*r1 sm1 = ma.nonzero(ym >= y10) sm9 = ma.nonzero(ym >= y90) r1 = xm[sm9] - xm[sm1] if mode == 'count': r1 = ma.count(ym) r2 = 0 if mode == 'maxslope': slope = [] win = ma.flatnotmasked_contiguous(ym) dt = x[1]-x[0] st = int(slopewin/dt) # use slopewin duration window for fit. print('st: ', st) for k, w in enumerate(win): # move through the slope measurementwindow tb = range(k-st, k+st) # get tb array ppars = np.polyfit(x[tb], ym[tb], 1) # do a linear fit - smooths the slope measures slope.append(ppars[0]) # keep track of max slope r1 = np.max(slope) r2 = np.argmax(slope) return(r1, r2)
if v in var2: arr1 = var1[v][:] arr2 = var2[v][:] mask1 = ma.getmask(arr1) mask2 = ma.getmask(arr2) diffm1 = ma.where((mask1 == True) & (mask2 == False)) diffm2 = ma.where((mask2 == 1) & (mask1 == 0)) if len(diffm1[0]) > 0: print( "WARNING: SOME " + v + " PIXEL ARE MASKED FOR THE FIRST FILE AND NOT FOR THE SECOND ONE" ) if len(diffm2[0]) > 0: print( "WARNING: SOME " + v + " PIXEL ARE MASKED FOR THE SECOND FILE AND NOT FOR THE FIRST ONE" ) arr1 = ma.array(arr1, mask=mask2) print(arr1.shape) arr2 = ma.array(arr2, mask=mask1) diff = ma.abs((arr1[:, 2] - arr2[:, 2])) * 100 / ma.abs(arr2[:, 2]) diffm = ma.where(diff > .5) if len(diffm[0]) > 0: print(arr1[diffm]) print(arr2[diffm]) for t in time[diffm]: print(t) print("WARNING NOT ALL VALUES ARE EQUAL FOR " + v + " VARIABLES")
def manhattan(a,b): """ Manhattan distance between a and b """ return ma.abs(a.real - b.real) + ma.abs(a.imag - b.imag)
def __fit__(self, rating, row=True): if isinstance(rating, ma.MaskedArray): self._rating = rating else: self._rating = ma.masked_equal(rating, 0) self._mean = ma.mean(self._rating, axis=1, keepdims=True) self._mean_center_rating = self._rating - self._mean self._rating_filled = self._rating.filled(0) if row: self._sim = person(mean_center_rating=self._mean_center_rating) else: self._rating = self._rating.T self._mean_center_rating = self._mean_center_rating.T self._sim = person(mean_center_rating=self._mean_center_rating) self._sim[np.diag_indices(self._sim.shape[0])] = -999 self._skip_columns = np.where(self._rating.count(axis=0) == 0)[0] # params self._neighborhood = np.argsort(self._sim, axis=1)[:, -self.config.topk:] self._neighborhood_idx = ([ int(i / self._neighborhood.shape[1]) for i in range(self._neighborhood.size) ], self._neighborhood.flatten()) if row: self._m, self._n = rating.shape else: self._n, self._m = rating.shape if "_weight" not in self.__dict__: self._weight = np.random.randn(self._m, self.config.topk) if "_m_bias" not in self.__dict__: self._m_bias = np.random.randn(self._m) if "_n_bias" not in self.__dict__: self._n_bias = np.random.randn(self._n) assert self._weight.shape[1] == self.config.topk assert self._m_bias.shape[0] == self._m assert self._n_bias.shape[0] == self._n start = time.perf_counter() step = self._epoch * self._n for epoch in range(self._epoch, self.config.epochs): self._epoch = epoch for j in range(self._n): if j in self._skip_columns: continue # forward step += 1 _hat_rating, mid_data = self.__forward__(j) _loss = self.__loss__(self._rating[:, j], _hat_rating) logger.debug( "[{:4d} step in {:4d} epoch\ttime:{:.2f}s] {}'s loss:{:.2f}" .format(step, self._epoch, time.perf_counter() - start, j, _loss)) # backward _g_m_bias, _g_ngb_m_bias, _g_ngb_n_bias, _g_weight = self.__backward__( _hat_rating, self._rating[:, j], mid_data[0], mid_data[1]) if not ma.any(_g_m_bias): continue for i, g in zip(self._neighborhood.flat, _g_ngb_m_bias.flat): if g is not ma.masked: _g_m_bias[i] += g # check gradient if self.config.check_gradient: logger.debug("check gradient") self.__check_gradient__(j, _g_weight, _g_m_bias, _g_ngb_n_bias) logger.debug( "[gradient] max(m_bias): {}\tmax(n_bias): {}\tmax(weight):{}" .format(ma.max(ma.abs(_g_m_bias)), ma.abs(_g_ngb_n_bias), ma.max(ma.abs(_g_weight)))) # update gradient self._m_bias -= self.config.lr / self._m * _g_m_bias + self.config.wdecay * self._m_bias self._n_bias[ j] -= self.config.lr / self._m * _g_ngb_n_bias + self.config.wdecay * self._n_bias[ j] self._weight -= self.config.lr / self._m * _g_weight + self.config.wdecay * self._weight logger.debug( "[{:4d} epoch\ttime:{:.2f}s] epoch loss:{:.2f}".format( epoch, time.perf_counter() - start, self.__loss__(self._rating, self.__predict__()))) if epoch % self.config.save_per_epochs == 0: self.save() if self._epoch % self.config.save_per_epochs != 0: self.save()
#select variable Salt = RomsNC.variables['salt'][:] Prime2 = (Salt - ma.mean(Salt))**2 varname = Prime2 #horizontal gradients ds_dx = gr.x_grad_rho(RomsFile, RomsGrd, varname) ds_dy = gr.y_grad_rho(RomsFile, RomsGrd, varname) #grid corrections xCor = gr.x_grad_GridCor_Rho(RomsFile, RomsGrd, varname) yCor = gr.y_grad_GridCor_Rho(RomsFile, RomsGrd, varname) #ratio x_rat = ma.array(ma.abs(xCor)/ma.abs(ds_dx)) y_rat = ma.array(ma.abs(yCor)/ma.abs(ds_dy)) #depth median ratio DM_ratx = ma.median(x_rat[0,:,:,:], axis = 0) DM_raty = ma.median(y_rat[0,:,:,:], axis = 0) #mask land values Land = ma.getmask(varname) Landdt = rt.AddDepthTime(RomsFile, Land) Xratio = ma.array(x_rat, mask = Land).flatten() Yratio = ma.array(y_rat, mask = Land).flatten() #remove mask
def euclid(a, b): """ Euclidian distance between a and b """ return ma.abs(a - b)
def mean_sigma_predictor(mean, sigma, z, sim, **not_used_kwargs): return mean + sigma * ma.sum(sim * z, axis=1) / ma.sum(ma.abs(sim), axis=1)
def correct_dualPRF_cmean(radar, field='velocity', Nprf=3, corr_method='median', Nmin=2): v_ma = radar.fields[field]['data'] vcorr_ma = v_ma.copy() out_mask = np.zeros(v_ma.shape) # Dual-PRF parameters Vny = radar.instrument_parameters['nyquist_velocity']['data'][0] prf_flag = radar.instrument_parameters['prf_flag']['data'] # Array with the primary Nyquist velocity corresponding to each bin Nprf_arr = primary_vel(v_ma.shape, Nprf, prf_flag=prf_flag) Vny_arr = Vny / Nprf_arr for nsweep, sweep_slice in enumerate(radar.iter_slice()): v0 = v_ma[sweep_slice] # velocity field vp = Vny_arr[sweep_slice] # primary velocities nprfp = Nprf_arr[sweep_slice] # dual-PRF factors ref, out_mask[sweep_slice] = outlier_detector_cmean(v0, Vny, nprfp, Nmin=Nmin) # Convert non-outliers to zero for correction procedure v0_out = v0 * out_mask[sweep_slice] vp_out = vp * out_mask[sweep_slice] ref_out = ref * out_mask[sweep_slice] vp_out_L = vp_out.copy() # Only low PRF outliers vp_out_L[nprfp == Nprf] = 0 dev = ma.abs(v0_out - ref_out) nuw = np.zeros(v0.shape) # Number of unwraps (initialisation) for ni in range(-Nprf, (Nprf + 1)): # New velocity values for identified outliers if abs(ni) == Nprf: vcorr_out = v0_out + 2 * ni * vp_out_L else: vcorr_out = v0_out + 2 * ni * vp_out # New deviation for new velocity values dev_tmp = ma.abs(vcorr_out - ref_out) # Compare with previous delta = dev - dev_tmp # Update unwrap number nuw[delta > 0] = ni # Update corrected velocity and deviation vcorr_out_tmp = v0_out + 2 * nuw * vp_out dev = ma.abs(vcorr_out_tmp - ref_out) # Corrected velocity field vcorr_ma[sweep_slice] = v0 + 2 * nuw * vp return vcorr_ma, out_mask
def run(self): """ Main method of the class. Reads data arrays, process them and returns results. """ self.logger.info('Started!') # Get inputs input_uids = self._data_helper.input_uids() assert input_uids, 'Error! No input arguments!' # Get parameters parameters = None if len(input_uids ) == MAX_N_INPUT_ARGUMENTS: # If parameters are given. parameters = self._data_helper.get( input_uids[INPUT_PARAMETERS_INDEX]) feature = self._get_parameter('Feature', parameters, DEFAULT_VALUES) exceedance = self._get_parameter('Exceedance', parameters, DEFAULT_VALUES) condition = self._get_parameter('Condition', parameters, DEFAULT_VALUES) calc_mode = self._get_parameter('Mode', parameters, DEFAULT_VALUES) self.logger.info('Calculation feature: %s', feature) self.logger.info('Exceedance: %s', exceedance) if condition is not None: self.logger.info('Condition: %s', condition) self.logger.info('Calculation mode: %s', calc_mode) # Get outputs output_uids = self._data_helper.output_uids() assert output_uids, '(CalcExceedance::run) No output arguments!' # Get time segments and levels study_time_segments = self._data_helper.get_segments( input_uids[STUDY_UID]) study_vertical_levels = self._data_helper.get_levels( input_uids[STUDY_UID]) # Normals time segments should be set for year 1 (as set in a pdftails file) normals_time_segments = deepcopy(study_time_segments) percentile = self._data_helper.get_levels(input_uids[NORMALS_UID])[ 0] # There should be only one level - percentile. for segment in normals_time_segments: segment['@beginning'] = '0001' + segment['@beginning'][4:] segment['@ending'] = '0001' + segment['@ending'][4:] # Read normals data normals_data = self._data_helper.get(input_uids[NORMALS_UID], segments=normals_time_segments) study_data = self._data_helper.get(input_uids[STUDY_UID], segments=study_time_segments) if exceedance == 'low': comparison_func = operator.lt elif exceedance == 'high': comparison_func = operator.gt else: self.logger.error('Error! Unknown exceedance value: \'%s\'', exceedance) raise ValueError data_func = ma.max # For calc_mode == 'data' we calculate max over all segments. input_description = self._data_helper.get_data_info( output_uids[0])['description'] output_description = {'@title': input_description['@title']} if feature != 'total': output_description['@title'] = feature.capitalize( ) + ' of ' + output_description['@title'] for level in study_vertical_levels: all_segments_data = [] for segment in study_time_segments: normals_values = normals_data['data'][percentile][ segment['@name']]['@values'] study_values = study_data['data'][level][ segment['@name']]['@values'] study_time_grid = study_data['data'][level][ segment['@name']]['@time_grid'] # Remove Feb 29 from the study array (we do not take this day into consideration). study_values = self._remove_feb29(study_values, study_time_grid) # Apply conditions if there are any. if condition is not None: self._apply_condition(study_values, condition) # Compare values according chosen exceedance. comparison_mask = comparison_func(study_values, normals_values) # Perform calculation for the current time segment. if feature == 'frequency': # We can just average 'True's to get a fraction and multiply by 100%. one_segment_data = ma.mean(comparison_mask, axis=0) * 100 output_description['@units'] = '%' if feature == 'intensity': diff = ma.abs(study_values - normals_values) # Calculate difference diff.mask = ma.mask_or( diff.mask, ~comparison_mask, shrink=False) # and mask out unnecessary values. one_segment_data = ma.mean(diff, axis=0) output_description['@units'] = 'days' if feature == 'duration': one_segment_data = self._calc_duration(comparison_mask) output_description['@units'] = 'days' if feature == 'total': study_values.mask = ma.mask_or(study_values.mask, ~comparison_mask, shrink=False) one_segment_data = ma.sum(study_values, axis=0) output_description['@units'] = 'days' # For segment-wise averaging send to the output current time segment results # or store them otherwise. if calc_mode == 'segment': self._data_helper.put( output_uids[0], values=one_segment_data, level=level, segment=segment, longitudes=study_data['@longitude_grid'], latitudes=study_data['@latitude_grid'], fill_value=study_data['@fill_value'], meta=study_data['meta'], description=output_description) elif calc_mode == 'data': all_segments_data.append(one_segment_data) else: self.logger.error( 'Error! Unknown calculation mode: \'%s\'', calc_mode) raise ValueError # For data-wise analysis analyse segments analyses :) if calc_mode == 'data': data_out = data_func(ma.stack(all_segments_data), axis=0) # Make a global segment covering all input time segments full_range_segment = deepcopy( study_time_segments[0] ) # Take the beginning of the first segment... full_range_segment['@ending'] = study_time_segments[-1][ '@ending'] # and the end of the last one. full_range_segment[ '@name'] = 'GlobalSeg' # Give it a new name. output_description[ '@title'] = 'Maximum ' + output_description['@title'] self._data_helper.put(output_uids[0], values=data_out, level=level, segment=full_range_segment, longitudes=study_data['@longitude_grid'], latitudes=study_data['@latitude_grid'], fill_value=study_data['@fill_value'], meta=study_data['meta'], description=output_description) self.logger.info('Finished!')