def calc_norm_summary_tables(accuracy_tbl, time_tbl): """ Calculate normalized performance/ranking summary, as numpy matrices as usual for convenience, and matrices of additional statistics (min, max, percentiles, etc.) Here normalized means relative to the best which gets a 1, all others get the ratio resulting from dividing by the performance of the best. """ # Min across all minimizers, i.e. for each fit problem what is the lowest chi-squared and the lowest time min_sum_err_sq = np.nanmin(accuracy_tbl, 1) min_runtime = np.nanmin(time_tbl, 1) # create normalised tables norm_acc_rankings = accuracy_tbl / min_sum_err_sq[:, None] norm_runtimes = time_tbl / min_runtime[:, None] summary_cells_acc = np.array([np.nanmin(norm_acc_rankings, 0), np.nanmax(norm_acc_rankings, 0), nanmean(norm_acc_rankings, 0), nanmedian(norm_acc_rankings, 0) ]) summary_cells_runtime = np.array([np.nanmin(norm_runtimes, 0), np.nanmax(norm_runtimes, 0), nanmean(norm_runtimes, 0), nanmedian(norm_runtimes, 0) ]) return norm_acc_rankings, norm_runtimes, summary_cells_acc, summary_cells_runtime
def triangleMAPs(savefilename,basename): with open(savefilename,'rb') as savefile: bf= numpy.array(pickle.load(savefile)) samples= numpy.array(pickle.load(savefile)) bf_g15= numpy.array(pickle.load(savefile)) samples_g15= numpy.array(pickle.load(savefile)) bf_zero= numpy.array(pickle.load(savefile)) samples_zero= numpy.array(pickle.load(savefile)) labels= [] for jj in range(samples.shape[2]): labels.append(r"$\mathrm{param}\ %i$" % jj) maps= define_rcsample.MAPs() for ii, map in enumerate(maps.map()): if ii >= len(bf): break tfeh= numpy.nanmedian(map['FE_H']) tafe= numpy.nanmedian(map[define_rcsample._AFETAG]) for tbf,tsamples,ext in zip([bf,bf_g15,bf_zero], [samples,samples_g15,samples_zero], ['fid','g15','zero']): try: triangle.corner(tsamples[ii,].T,quantiles=[0.16, 0.5, 0.84], labels=labels, show_titles=True,title_args={"fontsize": 12}, bins=21) except ValueError: pass else: bovy_plot.bovy_text(r'$[\mathrm{{Fe/H}}] = {feh:.1f},$'\ .format(feh=tfeh)+'\n' +r'$[\alpha/\mathrm{{Fe}}] = {afe:.2f}$'\ .format(afe=tafe), top_left=True,size=16.) bovy_plot.bovy_end_print(basename+"_%i_%s.png" % (ii,ext)) return None
def find_bounds(model): """ Return the median upper and lower bound of the metabolic model. Bounds can vary from model to model. Cobrapy defaults to (-1000, 1000) but this may not be the case for merged or autogenerated models. In these cases, this function is used to iterate over all the bounds of all the reactions and find the median bound values in the model, which are then used as the 'most common' bounds. Parameters ---------- model : cobra.Model The metabolic model under investigation. """ lower_bounds = np.asarray([rxn.lower_bound for rxn in model.reactions], dtype=float) upper_bounds = np.asarray([rxn.upper_bound for rxn in model.reactions], dtype=float) lower_bound = np.nanmedian(lower_bounds[lower_bounds != 0.0]) upper_bound = np.nanmedian(upper_bounds[upper_bounds != 0.0]) if np.isnan(lower_bound): LOGGER.warning("Could not identify a median lower bound.") lower_bound = -1000.0 if np.isnan(upper_bound): LOGGER.warning("Could not identify a median upper bound.") upper_bound = 1000.0 return lower_bound, upper_bound
def collapse_cube(w1, w2): """ Collapse a MUSE data cube. Arguments cube : MUSE data cube name containing both data and stat extensions. iext : Initial extension to be used. Default is one for combined cubes. """ fits = "slice_w{0}_{1}.fits".format(w1, w2) outfits = "collapsed_w{0}_{1}.fits".format(w1, w2) data = pf.getdata(fits, 0) error = pf.getdata(fits, 1) h = pf.getheader(fits, 0) h2 = pf.getheader(fits, 1) h["NAXIS"] = 2 del h["NAXIS3"] h2["NAXIS"] = 2 del h2["NAXIS3"] print "Starting collapsing process..." start = time.time() w = wavelength_array(fits) # newdata = np.trapz(data, dx=np.diff(w)[0], axis=0) # newdata = np.nansum(data, axis=0) * np.diff(w)[0] newdata = np.nanmedian(data, axis=0) noise = 1.482602 / np.sqrt(6.) * np.nanmedian(np.abs(2.* data - \ np.roll(data, 2, axis=0) - np.roll(data, -2, axis=0)), \ axis=0) end = time.time() print "Collapsing lasted {0} minutes.".format((end - start)/60.) hdu = pf.PrimaryHDU(newdata, h) hdu2 = pf.ImageHDU(noise, h2) hdulist = pf.HDUList([hdu, hdu2]) hdulist.writeto(outfits, clobber=True) return
def make_NRCS_image( nobj, bandname, fn='', dir='.', max=np.nan, min=np.nan, **kwargs): if not fn: if 'reduced' in bandname: fn = bandname[:9]+'.png' else: fn = bandname+'.png' resize(nobj) try: s0 = 10.0*np.log10(nobj[bandname]) except: n_obj.undo() raise s0[np.where(np.isinf(s0))]=np.nan #if nobj.fileName[-2:]=='nc': # s0 = flipdim(nobj,s0) caption='dB' if np.isnan(min): min = np.nanmedian(s0,axis=None)-2.0*np.nanstd(s0,axis=None) if np.isnan(max): max = np.nanmedian(s0,axis=None)+2.0*np.nanstd(s0,axis=None) nansatFigure(s0, min, max, dir, fn) nobj.undo() return fn
def plot_divergences(Ns, Ks, ivars): divs = np.array([divergence(ivar, Truth) for ivar in ivars]) small = (Ns * Ks) < 300 med = ((Ns * Ks) > 3000) * ((Ns * Ks) < 5000) big = (Ns * Ks) > 60000 Ksteps = 2. ** np.arange(0, 9) mediansmalldivs = np.array([_hoggmedian((divs[small])[np.isclose(Ks[small], Kstep)]) for Kstep in Ksteps]) medianmeddivs = np.array([_hoggmedian((divs[med])[np.isclose(Ks[med], Kstep)]) for Kstep in Ksteps]) medianbigdivs = np.array([_hoggmedian((divs[big])[np.isclose(Ks[big], Kstep)]) for Kstep in Ksteps]) plt.clf() plt.axhline(np.median(divs[small]), color="k", alpha=0.25) plt.axhline(np.median(divs[med] ), color="k", alpha=0.25) plt.axhline(np.median(divs[big] ), color="k", alpha=0.25) plt.plot(Ks[small], divs[small], "k_", ms= 6, alpha=0.5) plt.plot(Ks[med], divs[med], "k_", ms=12, alpha=0.5) plt.plot(Ks[big], divs[big], "k_", ms=18, alpha=0.5) good = np.isfinite(mediansmalldivs) plt.plot(Ksteps[good], mediansmalldivs[good], "k_", ms= 6, mew=4) plt.plot(Ksteps, medianmeddivs, "k_", ms=12, mew=4) plt.plot(Ksteps, medianbigdivs, "k_", ms=18, mew=4) plt.loglog() plt.xlim(np.min(Ks) / 1.5, np.max(Ks) * 1.5) plt.ylim(np.nanmedian(divs[big]) / 30., np.nanmedian(divs[small]) * 30.) plt.xlabel("number of photons per image $K$") plt.ylabel("divergence from the Truth") hogg_savefig("divergences.png") return None
def _make_tuples(self, key): # Get behavior filename behavior_path = (experiment.Session() & key).fetch1('behavior_path') local_path = lab.Paths().get_local_path(behavior_path) filename = (experiment.Scan.BehaviorFile() & key).fetch1('filename') full_filename = os.path.join(local_path, filename) # Read file data = h5.read_behavior_file(full_filename) # Get counter timestamps and convert to seconds ts = h5.ts2sec(data['ts'], is_packeted=True) # Read temperature (if available) and invalidate points with unreliable timestamps temp_raw = data.get('temperature', None) if temp_raw is None: raise PipelineException('Scan {animal_id}-{session}-{scan_idx} does not have ' 'temperature data'.format(**key)) temp_raw[np.isnan(ts)] = float('nan') # Read temperature and smooth it temp_celsius = (temp_raw * 100 - 32) / 1.8 # F to C sampling_rate = int(round(1 / np.nanmedian(np.diff(ts)))) # samples per second smooth_temp = signal.low_pass_filter(temp_celsius, sampling_rate, cutoff_freq=1, filter_size=2 * sampling_rate) # Resample at 1 Hz downsampled_ts = ts[::sampling_rate] downsampled_temp = smooth_temp[::sampling_rate] # Insert self.insert1({**key, 'temp_time': downsampled_ts, 'temperatures': downsampled_temp, 'median_temperature': np.nanmedian(downsampled_temp)}) self.notify(key)
def flaremeter(data): ''' Obtain median of data across baselines, polarizations, and frequencies to create a time series indicated whether a flare has occurred. Values returned will be close to unity if no flare. Returns: tlevel: Array of levels at each time, nominally near unity bflag: Array of flags indicating nominal background (where True) or elevated background (where False) indicating possible flare ''' nbl,npol,nf,nt = data.shape tlevel = np.zeros(nt,'float') background = np.sqrt(np.abs(data[:,0,:,:])**2 + np.abs(data[:,1,:,:])**2) init_bg = np.nanmedian(background,2) # Initially take background as median over entire time range bflag = np.ones(nt,'bool') # flags indicating "good" background times (not in flare) for i in range(nt): good, = np.where(bflag[:i] == True) # List of indexes of good background times up to current time ngood = len(good) # Truncate list of indexes to last 100 elements (or fewer) if ngood > 100: good = good[ngood-100:] # Calculate median over good background times bg = np.nanmedian(background[:,:,good],2) else: # If there haven't been 100 times with good backgrounds yet, just use the initial one. # This is supposed to avoid startup transients. bg = init_bg # Generate levels for each baseline and frequency for this time level = np.sqrt(abs(data[:,0,:,i])**2 + abs(data[:,1,:,i])**2)/bg # Take median over baseline and frequency to give a single number for this time tlevel[i] = np.nanmedian(level) if tlevel[i] > 1.05: # If the level of the current time is higher than 1.05, do not include this time in future backgrounds bflag[i] = False return tlevel, bflag
def subtract(args): for im_name in args.input1: if args.overwrite: new_name = im_name elif args.suffix: new_name = utilities.add_suffix_prefix(im_name, suffix=args.suffix) # Read image, separate data and header im = fits.open(im_name) data = im[0].data hdr = im[0].header # Extract the overscan region. Notice that what we call x,y are the second and first axes y0, y1, x0, x1 = args.region overscan = data.copy()[x0:x1, y0:y1] # Average over the short axis if overscan.shape[0] < overscan.shape[1]: average = numpy.nanmedian(overscan, axis=0) # Fit a polynomial and return the fitted values fitted_overscan = fit_pol(average, args.deg) data[:, y0:y1] -= fitted_overscan else: average = numpy.nanmedian(overscan, axis=1) # Fit a polynomial and return the fitted values fitted_overscan = fit_pol(average, args.deg) data[x0:x1, :] = (data[x0:x1, :].T - fitted_overscan).T # Write to the output file hdr.add_comment("Overscan region subtracted. Region: [{0}:{1},{2}:{3}]".format(x0, x1, y0, y1)) fits.writeto(new_name, data, hdr, clobber=True) return None
def cutoff(self, df, z_score=3.0): """ Cut off extreme values using Median Absolute Deviation Parameters ---------- df : pd.DataFrame Returns ------- pd.DataFrame """ df = self._align_univariate(df) df = self._mask_non_index_member(df) axis = 1 x = df.values median = np.nanmedian(x, axis=axis).reshape(-1, 1) diff = x - median diff_abs = np.abs(diff) mad = np.nanmedian(diff_abs, axis=axis).reshape(-1, 1) mask = diff_abs > z_score * mad x[mask] = 0 x = x + z_score * mad * np.sign(diff * mask) + mask * median return pd.DataFrame(index=df.index, columns=df.columns, data=x)
def test_multiFringes(self): """Test that multi-fringe results are handled correctly by the task. """ self.config.filters.append("_unknown_") self.config.large = 16 task = FringeTask(name="multiFringeMock", config=self.config) config = isrMock.IsrMockConfig() config.fringeScale = [750.0, 240.0, 220.0] config.fringeX0 = [100.0, 150.0, 200.0] config.fringeY0 = [0.0, 200.0, 0.0] dataRef = isrMock.FringeDataRefMock(config=config) exp = dataRef.get("raw") medianBefore = np.nanmedian(exp.getImage().getArray()) fringes = task.readFringes(dataRef, assembler=None) solution, rms = task.run(exp, **fringes.getDict()) medianAfter = np.nanmedian(exp.getImage().getArray()) stdAfter = np.nanstd(exp.getImage().getArray()) self.assertLess(medianAfter, medianBefore) self.assertFloatsAlmostEqual(medianAfter, 3002.233, atol=1e-4) self.assertFloatsAlmostEqual(stdAfter, 3549.9375, atol=1e-4) deviation = np.abs(solution - config.fringeScale) self.assertTrue(np.all(deviation / rms < 1.0))
def sigmaclip(data, factor, replacement=None, median=False, maxiter = 100): std = np.std(data) iteration=0 if median: center = np.nanmedian(data) else: center = np.nanmean(data) if not replacement: replacement = np.nan elif replacement == 'mean': replacement = center indx = (data>(center+std*factor))+(data<(center-std*factor)) while np.sum(indx) > 0 and iteration < maxiter: #print indx, np.sum(indx) #pl.plot(data) #pl.plot([0,len(data)],[center+std*factor,center+std*factor]) #pl.plot([0,len(data)],[center-std*factor,center-std*factor]) data[indx] = replacement std = np.std(data) if median: center = np.nanmedian(data) else: center = np.nanmean(data) if not replacement: replacement = np.nan elif replacement == 'mean': replacement = center indx = (data>(center+std*factor))+(data<(center-std*factor)) #print indx, np.sum(indx) #pl.plot(data,'ko') #pl.show() iteration+=1 return data
def Scatter(y, win=13, remove_outliers=False): ''' Return the scatter in ppm based on the median running standard deviation for a window size of :py:obj:`win` = 13 cadences (for K2, this is ~6.5 hours, as in VJ14). :param ndarray y: The array whose CDPP is to be computed :param int win: The window size in cadences. Default `13` :param bool remove_outliers: Clip outliers at 5 sigma before computing \ the CDPP? Default `False` ''' if remove_outliers: # Remove 5-sigma outliers from data # smoothed on a 1 day timescale if len(y) >= 50: ys = y - Smooth(y, 50) else: ys = y M = np.nanmedian(ys) MAD = 1.4826 * np.nanmedian(np.abs(ys - M)) out = [] for i, _ in enumerate(y): if (ys[i] > M + 5 * MAD) or (ys[i] < M - 5 * MAD): out.append(i) out = np.array(out, dtype=int) y = np.delete(y, out) if len(y): return 1.e6 * np.nanmedian([np.std(yi) / np.sqrt(win) for yi in Chunks(y, win, all=True)]) else: return np.nan
def savePredictors(indata): ''' compute velocity quantiles for each inter-saccade event ''' import numpy as np data,coderid,gazeLag=indata try: preds=[] coords=[] data.extractBasicEvents() data.driftCorrection() if coderid!=-1: importFailed=data.importComplexEvents(coderid=coderid) if importFailed: return [[data.vp,-1]] g=data.getGaze();totdur=float(g.shape[0]) hz=100#hz of the coordinate output padstart=20# insert nans at start to allow lag if gazeLag>=0: gg=data.getGaze(hz=hz)[:,np.newaxis,[7,8]] tr=data.getTraj(hz=hz) mnn=min(gg.shape[0],tr.shape[0]) tt=data.getGaze(hz=hz)[:mnn,0] gg=gg[:mnn,:,:];tr=tr[:mnn,:,:] vel=data.getVelocity() for si in range(len(data.sev)): if si+2<len(data.sev): e=data.sev[si+1][0] else: e=-1 s=data.sev[si][1];d=e-s if g[s,0]-gazeLag<0 and g[e,0]-gazeLag<=0: continue preds.append([data.vp,data.block,data.trial, data.sev[si][-1],si,s,d,s/totdur,d/totdur]) if gazeLag>=0: sel1=np.logical_and(tt>(g[s,0]),tt<(g[e,0])) sel2=np.logical_and(tt>(g[s,0]-gazeLag),tt<(g[e,0]-gazeLag)) #print '1',sel1.sum(),sel2.sum() trt=tr[sel2,:,:]; #print '2',trt.shape trt=trt[:min(sel2.sum(),sel1.sum()),:,:] #print '3',trt.shape temp=np.ones((sel1.sum(),14,2))*np.nan #print '4',temp.shape temp[-sel2.sum():,:,:]=trt coords.append(np.concatenate([gg[sel1,:,:],temp],axis=1)) tps=[s,s+d/4.,s+d/2.,s+3*d/4.,e] tps=np.int32(np.round(tps)) for ti in range(len(tps)-1): preds[-1].append(np.nanmedian(vel[tps[ti]:tps[ti+1]])) dist=np.nanmedian(data.dist[tps[ti]:tps[ti+1],:],0) di=np.argsort(dist)[:4]#take four nearest agents preds[-1].extend(dist[di]) dev=np.abs(data.dev[tps[ti]:tps[ti+1],:]) dev=np.nanmedian(dev[:,di],0) preds[-1].extend(dev) if len(preds): if gazeLag>=0: return [preds,coords] else: return [preds,[]] else: return [[[data.vp,-1]],[]] except: print 'Error at vp %d b %d t %d'%(data.vp,data.block,data.trial) raise
def values_to_sizes(values, size, size_range): maxval, minval = np.nanmax(values), np.nanmin(values) try: medval = np.nanmedian(values) except KeyError: medval = np.nanmedian(values.values) sizes = (values-medval)/(maxval-medval)*size_range + size return sizes
def plot_lc_white(self, ax=None): # ax.plot(self.time, self.flux_r, lw=1) # ax.plot(self.time, self.trtime, lw=1) # ax.plot(self.time, self.trposi+4*(np.percentile(self.flux_r, [99])[0]-1), lw=1) ax.plot(self.time, self.flux_r-self.trposi+np.nanmedian(self.trposi) -self.trtime+np.nanmedian(self.trtime), '.') [ax.axvline(self.bls.tc+i*self._rbls['bls_period'], alpha=0.25, ls='--', lw=1) for i in range(35)] setp(ax,xlim=self.time[[0,-1]], xlabel='Time', ylabel='Normalised flux')
def get_bb_ratio(bb_height, bb_width, quality, zp_r): """Returns the Bright Band ratio of each PR bin With *SR*, we refer to precipitation radars based on space-born platforms such as TRMM or GPM. This function basically applies the Bright Band (BB) information as provided by the corresponding SR datasets per beam, namely BB height and width, as well as quality flags of the SR beams. A BB ratio of <= 0 indicates that a bin is located below the melting layer (ML), >=1 above the ML, and in between 0 and 1 inside the ML. Parameters ---------- bb_height : :class:`numpy:numpy.ndarray` Array of shape (nscans, nbeams) containing the SR beams' BB heights in meters. bb_width : :class:`numpy:numpy.ndarray` Array of shape (nscans, nbeams) containing the SR beams' BB widths in meters. quality : :class:`numpy:numpy.ndarray` Array of shape (nscans, nbeams) containing the SR beams' BB quality index. zp_r : :class:`numpy:numpy.ndarray` Array of SR bin altitudes of shape (nscans, nbeams, nbins). Returns ------- ratio : :class:`numpy:numpy.ndarray` Array of shape (nscans, nbeams, nbins) containing the BB ratio of every SR bin. - ratio <= 0: below ml - 0 < ratio < 1: between ml - 1 <= ratio: above ml ibb : :class:`numpy:numpy.ndarray` Boolean array containing the indices of SR bins connected to the BB. """ # parameters for bb detection ibb = (bb_height > 0) & (bb_width > 0) & (quality == 1) # set non-bb-pixels to np.nan bb_height = bb_height.copy() bb_height[~ibb] = np.nan bb_width = bb_width.copy() bb_width[~ibb] = np.nan # get median of bb-pixels bb_height_m = np.nanmedian(bb_height) bb_width_m = np.nanmedian(bb_width) # approximation of melting layer top and bottom zmlt = bb_height_m + bb_width_m / 2. zmlb = bb_height_m - bb_width_m / 2. # get ratio connected to brightband height ratio = (zp_r - zmlb) / (zmlt - zmlb) return ratio, ibb
def SysRem(time, flux, err, ncbv=5, niter=50, sv_win=999, sv_order=3, **kwargs): ''' Applies :py:obj:`SysRem` to a given set of light curves. :param array_like time: The time array for all of the light curves :param array_like flux: A 2D array of the fluxes for each of the light \ curves, shape `(nfluxes, ntime)` :param array_like err: A 2D array of the flux errors for each of the \ light curves, shape `(nfluxes, ntime)` :param int ncbv: The number of signals to recover. Default 5 :param int niter: The number of :py:obj:`SysRem` iterations to perform. \ Default 50 :param int sv_win: The Savitsky-Golay filter window size. Default 999 :param int sv_order: The Savitsky-Golay filter order. Default 3 ''' nflx, tlen = flux.shape # Get normalized fluxes med = np.nanmedian(flux, axis=1).reshape(-1, 1) y = flux - med # Compute the inverse of the variances invvar = 1. / err ** 2 # The CBVs for this set of fluxes cbvs = np.zeros((ncbv, tlen)) # Recover `ncbv` components for n in range(ncbv): # Initialize the weights and regressors c = np.zeros(nflx) a = np.ones(tlen) f = y * invvar # Perform `niter` iterations for i in range(niter): # Compute the `c` vector (the weights) c = np.dot(f, a) / np.dot(invvar, a ** 2) # Compute the `a` vector (the regressors) a = np.dot(c, f) / np.dot(c ** 2, invvar) # Remove this component from all light curves y -= np.outer(c, a) # Save this regressor after smoothing it a bit if sv_win >= len(a): sv_win = len(a) - 1 if sv_win % 2 == 0: sv_win -= 1 cbvs[n] = savgol_filter(a - np.nanmedian(a), sv_win, sv_order) return cbvs
def draw(self): #return np.fmax(self.ads.draw(), self.cds.draw()) adens = self.ads.draw() cdens = self.cds.draw() dens = cdens.copy() for ii in range(dens.shape[0]): if np.nanmedian(adens[ii]) > np.nanmedian(cdens[ii]): dens[ii] = adens[ii] return dens
def snr(flux, axis=0): """ Calculates the S/N ratio of a spectra. Translated from the IDL routine der_snr.pro """ signal = np.nanmedian(flux, axis=axis) noise = 1.482602 / np.sqrt(6.) * np.nanmedian(np.abs(2.*flux - \ np.roll(flux, 2, axis=axis) - np.roll(flux, -2, axis=axis)), \ axis=axis) return signal, noise, signal / noise
def _getMagnitudes(g): if any([not np.isnan(x) for x in g.Mag]): mag = np.nanmedian(g.Mag) else: mag = np.NaN if any([not np.isnan(x) for x in g.ProEnMag]): PEmag = np.nanmedian(g.ProEnMag) else: PEmag = np.NaN return mag, PEmag
def populate_temp_elev_dict(dic, grid_temp, grid_elev, single_value=True): r"""Populate a grid with average values from the dictionary. Takes values with temperature and elevation information and assigns them to a grid position. For each grid averages of the values within the grid are calculated. Parameters ---------- dic : dict Dictionary containing lists of the values and their temperature and elevation. grid_temp, grid_elev : int Dimension of the grid. single_value : bool Used to determine, whether there is a single value, as is the case with tsi data or an average of data, as is the case for psf data. Default is `True`. Returns ------- dict A dictionary of lists as defined by `generate_temp_elev_dict`. """ ext = _get_temp_elev_dict_ext(dic) g = generate_temp_elev_dict() for __temp in range(grid_temp): for __elev in range(grid_elev): bound = _get_temp_elev_bound(ext, grid_temp, grid_elev, __temp, __elev) std, mean, med, var, val = [[],[],[],[],[]] for k, v in enumerate(dic['temp']): if dic['temp'][k] >= bound[0] and dic['temp'][k] <= bound[1]: if dic['elev'][k] >= bound[2] and dic['temp'][k] <= bound[3]: if single_value: val.append(dic['val'][k]) else: std.append(dic['std'][k]) mean.append(dic['mean'][k]) med.append(dic['med'][k]) var.append(dic['var'][k]) g['temp'].append((bound[0] + bound[1])/2) g['elev'].append((bound[2] + bound[3])/2) if single_value: g['std'].append(np.nanstd(val)) g['mean'].append(np.nanmean(val)) g['med'].append(np.nanmedian(val)) g['var'].append(np.nanvar(val)) g['err'].append(np.nanstd(val)/math.sqrt(len(val))) else: g['std'].append(np.nanmedian(std)) g['mean'].append(np.nanmedian(mean)) g['med'].append(np.nanmedian(med)) g['var'].append(np.nanmedian(var)) g['err'].append(np.nanstd(mean)/math.sqrt(len(std))) return g
def mad(arr): """ Median average deviation. The parameter `arr` is any list-like object (list, tuple, numpy.ndarray) """ if not IsListlike(arr): raise ValueError("The input to mad must be a list-like object!") median = np.nanmedian(arr) arr = np.array(arr) return np.nanmedian(np.abs(arr - median))
def mad(arr): """ Median Absolute Deviation: a "Robust" version of standard deviation. Indices variabililty of the sample. https://en.wikipedia.org/wiki/Median_absolute_deviation :param arr: numpy array value :return: return mad """ # arr = np.ma.array(arr).compressed() # should be faster to not use masked arrays. med = np.nanmedian(arr) return 1.4826 * np.nanmedian(np.absolute(arr - med))
def nanmad(a, axis=None): """ Calculates the Median Absolute Deviation from an array. """ a = np.array(a, copy=False) a_median = np.nanmedian(a, axis=axis) # re-broadcast the output median array to subtract it if axis is not None: a_median = np.expand_dims(a_median, axis=axis) # calculated the median average deviation return np.nanmedian(np.abs(a - a_median), axis=axis)
def mad(arr): """ Median average deviation :param arr: A list-like object :return: """ if not IsListlike(arr): raise ValueError("The input to mad must be a list-like object!") median = np.nanmedian(arr) arr = np.array(arr) return np.nanmedian(np.abs(arr - median))
def mad_based_outlier(data, thresh=3.5): data = numpy.array(data) if len(data.shape) == 1: data = data[:,None] median = numpy.nanmedian(data) diff = numpy.nansum((data - median)**2, dtype=float, axis=-1) diff = numpy.sqrt(diff) med_abs_deviation = numpy.nanmedian(diff) modified_z_score = 0.6745 * diff / med_abs_deviation return modified_z_score > thresh
def response_curve(x1, _ticker='SP500', f=None, md=True, sigd=False): univ_ib_gd = cr_cret.retrieve(univ_ib_eqidx_ext + 'ExchOpen')[_ticker].values univ_ib_vl = cr_vol_all_adj.retrieve(univ_ib_eqidx_ext + 'vol_pb_120')[_ticker].values univ_ib_s1 = cr_sig_mr_sg.retrieve(univ_ib_eqidx_ext + x1)[_ticker].values univ_ib_cl = cr_cret.retrieve(univ_ib_eqidx_ext + 'Close')[_ticker].values z = list(np.where(univ_ib_gd.astype('int') == 1)[0]) univ_ib_s1 = univ_ib_s1[z] univ_ib_cl = univ_ib_cl[z] univ_ib_vl = univ_ib_vl[z] univ_ib_cl = filt.ret(univ_ib_cl)/filt.lag(univ_ib_vl, 1) univ_ib_s1 = filt.lag(univ_ib_s1, 1)/filt.lag(univ_ib_vl, 1) univ_ib_cl, univ_ib_s1 = reduce_nonnan(univ_ib_cl, univ_ib_s1) _bins = 20 _range = np.maximum(np.percentile(univ_ib_s1, 99), -np.percentile(univ_ib_s1, 1)) _delta = _range/_bins if f is not None: pyl.figure(f) else: pyl.figure(1) for i in range(0, 16): if i == 0: uis1 = univ_ib_s1 uic1 = univ_ib_cl else: uis1 = filt.lag(univ_ib_s1, i) uic1 = filt.sma(univ_ib_cl, i+1) uis1, uic1 = reduce_nonnan(uis1, uic1) uis1_b = np.linspace(-_range, _range, num=_bins+1) uic1_b = np.zeros(_bins+1)*np.nan for j in range(0, _bins+1): # j = 1 if j==0: tmp__ = np.where(uis1 <= uis1_b[j]+_delta)[0] elif j == _bins+1: tmp__ = np.where(uis1 > uis1_b[j]-_delta)[0] else: tmp__ = np.where((uis1 <= uis1_b[j]+_delta) & (uis1 > uis1_b[j]-_delta))[0] if tmp__.shape[0] > 0: if md: if not sigd: uic1_b[j] = np.nanmedian(uic1[tmp__]) #/np.nanstd(uic1[tmp__]) else: uic1_b[j] = np.nanmedian(uic1[tmp__])/np.nanstd(uic1[tmp__]) else: if not sigd: uic1_b[j] = np.nanmean(uic1[tmp__]) #/np.nanstd(uic1[tmp__]) else: uic1_b[j] = np.nanmean(uic1[tmp__])/np.nanstd(uic1[tmp__]) pyl.subplot(4, 4, i+1) pyl.plot(uis1_b, uic1_b)
def summarize(fitted, rchi2, measurement, rchi2_limit=1.5): """ Create summaries of the input measurement :param fitted: :param rchi2: :param measurement: :param summary: :param rchi2_limit: :return: """ nlon = measurement.shape[1] mean = np.zeros(shape=nlon) std = np.zeros_like(mean) median = np.zeros_like(mean) mad = np.zeros_like(mean) n_found = np.zeros_like(mean) for i in range(0, nlon): # Find where the successful fits were successful_fit = fitted[:, i] # Reduced chi-squared rc2 = rchi2[:, i] # Successful fit f = np.logical_and(successful_fit, rc2 < rchi2_limit) # Indices of the successful fits trial_index = np.nonzero(f) # Number of successful trials n_found[i] = np.sum(f) m = measurement[trial_index, i] mean[i] = np.nansum(m) / (1.0 * n_found[i]) std[i] = np.std(m) median[i] = np.nanmedian(m) mad[i] = np.nanmedian(np.abs(m - median[i])) mean_mean = np.nanmean(mean) mean_std = np.nanmean(std) median_median = np.nanmedian(median) median_mad = np.nanmedian(mad) return ("mean value (standard deviation)", mean, std, mean_mean, mean_std),\ ("median value (median absolute deviation)", median, mad, median_median, median_mad),\ ("n_found", n_found)
def test_empty(self): mat = np.zeros((0, 3)) for axis in [0, None]: with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') assert_(np.isnan(np.nanmedian(mat, axis=axis)).all()) assert_(len(w) == 1) assert_(issubclass(w[0].category, RuntimeWarning)) for axis in [1]: with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') assert_equal(np.nanmedian(mat, axis=axis), np.zeros([])) assert_(len(w) == 0)
elif product == "LS8_OLI_LASRC": validValues = [322, 386, 834, 898, 1346, 324, 388, 836, 900, 1348] cloud_mask = isin(nbar["pixel_qa"].values, validValues) for band in bands: datos = np.where( np.logical_and(nbar.data_vars[band] != nodata, cloud_mask), nbar.data_vars[band], np.nan) allNan = ~np.isnan(datos) if normalized: m = np.nanmean(datos.reshape((datos.shape[0], -1)), axis=1) st = np.nanstd(datos.reshape((datos.shape[0], -1)), axis=1) datos = np.true_divide( (datos - m[:, np.newaxis, np.newaxis]), st[:, np.newaxis, np.newaxis]) * np.nanmean(st) + np.nanmean(m) medians1[band] = np.nanmedian(datos, 0) medians1[band][np.sum(allNan, 0) < minValid] = np.nan del datos # In[7]: from sklearn.externals import joblib # In[21]: #Preprocesar: nmed = None nan_mask = None for band in bands: b = medians1[band].ravel() if nan_mask is None:
def plotsupp2(supp_value_frame, analysis_path): ''' 用于输出csv文件和图 :param supp_value_frame: pd.DataFrame 供应商方面数据 :param analysis_path: string 文件路径 :return: None 仅用于作图 ''' # 输出第一张表 grouped = supp_value_frame.groupby('supp_name') summary = [] for supp_name, group in grouped: caigousum = group["pur_bill_id"].drop_duplicates().count() skusum = group["item_sku_id"].drop_duplicates().count() vltmean = np.mean(group["vlt"]) vltstd = np.std(group["vlt"]) vltcv = vltstd / vltmean manzu = np.sum(group["actual_pur_qtty"]) / np.sum(group["originalnum"]) manzu2 = np.nansum(group["actual_pur_qtty"]) / np.nansum(group["plan_pur_qtty"]) summary.append({'supp_name':supp_name, 'caigousum': caigousum, 'skusum': skusum, 'vltmean': vltmean, 'vltstd': vltstd, 'vltcv': vltcv, 'manzu':manzu, 'manzu2': manzu2}) table1 = pd.DataFrame.from_dict(summary) calname = ['caigousum','skusum','vltmean','vltstd','vltcv','manzu'] for i, colname in enumerate(calname): min_num = np.min(table1[colname]) max_num = np.max(table1[colname]) cal_val = np.array(map(lambda x: (x - min_num)/(max_num - min_num)*9 + 1,table1[colname].values)) if i == 0: sum_val = cal_val else: sum_val += cal_val sum_val /= 6 table2 = pd.concat([table1, pd.DataFrame(sum_val.T, columns=["rank"])], axis=1) table2 = table2.loc[:,['supp_name','caigousum','skusum','vltmean','vltstd','vltcv','manzu', 'rank']] table2 = table2.sort_values(['rank'], ascending=False) table2.to_csv(analysis_path + os.sep + 'supp_info.csv') # 画vlt图 binsn = np.append(np.arange(0, 0.6, 0.05), 1) plothistper(table2['vltcv'], binsn, u'vltcv', u'个数', u'vltcv分布', analysis_path + '//report//vltcv', intshu=False) # 画满足率 binsn2 = np.arange(0.3, 1.1, 0.1) plothistper(table1['manzu'], binsn2, u'满足率', u'个数', u'实际满足率分布', analysis_path + '//report//manzu', intshu=False) plothistper(table1[table1['manzu2'].notnull()]["manzu2"], binsn2, u'满足率', u'个数', u'实际相对回告满足率分布', analysis_path + '//report//manzu2', intshu=False) print "实际满足率的基本统计信息,最小值:{0:.2f}%,中位数:{1:.2f}%,均值:{2:.2f}%,最大值:{3:.2f}%。".format( np.nanmin(table1[table1['manzu']>0.01]['manzu'].values),np.nanmedian(table1['manzu'].values), np.nanmean(table1['manzu'].values),np.nanmax(table1['manzu'].values)) print "实际相对回告满足率分布的基本统计信息,最小值:{0:.2f}%,中位数:{1:.2f}%,均值:{2:.2f}%,最大值:{3:.2f}%。".format( np.nanmin(table1[table1['manzu2']>0.01]['manzu2'].values),np.nanmedian(table1['manzu2'].values), np.nanmean(table1['manzu2'].values),np.nanmax(table1['manzu2'].values)) # 画 vltcv * 满足率 图 fig, ax = plt.subplots(figsize=(12, 8)) # ax.grid() ax.set_xlabel(u"vlt波动性") ax.set_ylabel(u"平均满足率") ax.set_xlim(0.05, 0.5) ax.set_ylim(0.2, 1.1) plt.scatter(table1['vltcv'], table1['manzu'], color='#0070C0') plt.plot([0.3, 0.3], [0.2, 1.1], '--', color='red') plt.plot([0.05, 0.5], [0.8, 0.8], '--', color='red') plt.annotate('(1)', xy=(0.16, 0.9), fontsize=20, color='red') plt.annotate('(2)', xy=(0.39, 0.9), fontsize=20, color='red') plt.annotate('(3)', xy=(0.16, 0.5), fontsize=20, color='red') plt.annotate('(4)', xy=(0.39, 0.5), fontsize=20, color='red') plt.title(u'vlt稳定性与订单满足率散点图') plt.savefig(analysis_path + '//report//vltcv_manzu')
def nanmedian(values, axis=None, skipna=True, mask=None): """ Parameters ---------- values : ndarray axis: int, optional skipna : bool, default True mask : ndarray[bool], optional nan-mask if known Returns ------- result : float Unless input is a float array, in which case use the same precision as the input array. Examples -------- >>> import pandas.core.nanops as nanops >>> s = pd.Series([1, np.nan, 2, 2]) >>> nanops.nanmedian(s) 2.0 """ def get_median(x): mask = notna(x) if not skipna and not mask.all(): return np.nan return np.nanmedian(x[mask]) values, mask, dtype, dtype_max, _ = _get_values(values, skipna, mask=mask) if not is_float_dtype(values): values = values.astype("f8") if mask is not None: values[mask] = np.nan if axis is None: values = values.ravel() notempty = values.size # an array from a frame if values.ndim > 1: # there's a non-empty array to apply over otherwise numpy raises if notempty: if not skipna: return _wrap_results( np.apply_along_axis(get_median, axis, values), dtype) # fastpath for the skipna case return _wrap_results(np.nanmedian(values, axis), dtype) # must return the correct shape, but median is not defined for the # empty set so return nans of shape "everything but the passed axis" # since "axis" is where the reduction would occur if we had a nonempty # array shp = np.array(values.shape) dims = np.arange(values.ndim) ret = np.empty(shp[dims != axis]) ret.fill(np.nan) return _wrap_results(ret, dtype) # otherwise return a scalar value return _wrap_results(get_median(values) if notempty else np.nan, dtype)
def multilook_data(data, lks_y=1, lks_x=1, method='mean'): """Modified from Praveen on StackOverflow: link: https://stackoverflow.com/questions/34689519/how-to-coarser-the-2-d-array-data-resolution Parameters: data : 2D / 3D np.array in real or complex lks_y : int, number of multilook in y/azimuth direction lks_x : int, number of multilook in x/range direction method : str, multilook method, mean, median or nearest Returns: coarse_data : 2D / 3D np.array after multilooking in last two dimension """ method_list = ['mean', 'median', 'nearest'] if method not in method_list: msg = 'un-supported multilook method: {}. '.format(method) msg += 'Available methods: {}'.format(method_list) raise ValueError(msg) # do nothing if no multilook is applied lks_y = int(lks_y) lks_x = int(lks_x) if lks_y * lks_x == 1: return data dtype = data.dtype shape = np.array(data.shape, dtype=float) ysize = int(shape[0] / lks_y) xsize = int(shape[1] / lks_x) if len(shape) == 2: if method in ['mean', 'median']: # crop data to the exact multiple of the multilook number new_shape = np.floor(shape / (lks_y, lks_x)).astype(int) * (lks_y, lks_x) crop_data = data[:new_shape[0], :new_shape[1]] # reshape to more dimensions and collapse the extra dimensions with mean temp = crop_data.reshape( (new_shape[0] // lks_y, lks_y, new_shape[1] // lks_x, lks_x)) with warnings.catch_warnings(): warnings.simplefilter("ignore", category=RuntimeWarning) if method == 'mean': coarse_data = np.nanmean(temp, axis=(1, 3)) elif method == 'median': coarse_data = np.nanmedian(temp, axis=(1, 3)) elif method == 'nearest': coarse_data = data[int(lks_y / 2)::lks_y, int(lks_x / 2)::lks_x] # fix size discrepency from average method if coarse_data.shape != (ysize, xsize): coarse_data = coarse_data[:ysize, :xsize] elif len(shape) == 3: if method in ['mean', 'median']: # crop data to the exact multiple of the multilook number new_shape = np.floor( shape / (1, lks_y, lks_x)).astype(int) * (1, lks_y, lks_x) crop_data = data[:new_shape[0], :new_shape[1], :new_shape[2]] # reshape to more dimensions and collapse the extra dimensions with mean temp = crop_data.reshape((new_shape[0], new_shape[1] // lks_y, lks_y, new_shape[2] // lks_x, lks_x)) with warnings.catch_warnings(): warnings.simplefilter("ignore", category=RuntimeWarning) if method == 'mean': coarse_data = np.nanmean(temp, axis=(2, 4)) elif method == 'median': coarse_data = np.nanmedian(temp, axis=(2, 4)) elif method == 'nearest': coarse_data = data[:, int(lks_y / 2)::lks_y, int(lks_x / 2)::lks_x] # fix size discrepency from average method if coarse_data.shape[-2:] != (ysize, xsize): coarse_data = coarse_data[:, :ysize, :xsize] # ensure output data type coarse_data = np.array(coarse_data, dtype=dtype) return coarse_data
def get_median(x): mask = notna(x) if not skipna and not mask.all(): return np.nan return np.nanmedian(x[mask])
# In[73]: #check why "local" is an object data type as it is supposed to be binary and does not contain any "?" in the expanded view above print(ad_df.loc[:, "local"].unique()) # In[74]: # Coerce to numeric and impute medians for height continuous column ad_df.loc[:, "height continuous"] = pd.to_numeric(ad_df.loc[:, "height continuous"], errors='coerce') HasNan = np.isnan(ad_df.loc[:, "height continuous"]) ad_df.loc[HasNan, "height continuous"] = np.nanmedian(ad_df.loc[:, "height continuous"]) # In[75]: plt.hist(ad_df.loc[:, "height continuous"]) # In[76]: ## The high limit for acceptable values is the mean plus 2 standard deviations LimitHi = ad_df.loc[:, "height continuous"].mean() + 2 * ( ad_df.loc[:, "height continuous"].std()) print(LimitHi) # In[77]: #Replace outliers
def draw_mask_map_strip(image, seg_comb, mask_comb, stars, ma_example=None, r_core=None, vmin=None, vmax=None, pad=0, save=False, save_dir='./'): """ Visualize mask map w/ strips """ from matplotlib import patches star_pos_A = stars.star_pos_verybright + pad star_pos_B = stars.star_pos_medbright + pad if ma_example is not None: mask_strip, mask_cross = ma_example mu = np.nanmedian(image) std = mad_std(image) if vmin is None: vmin = mu - std if vmax is None: vmax = mu + 10 * std fig, (ax1, ax2, ax3) = plt.subplots(ncols=3, nrows=1, figsize=(20, 6)) mask_strip[mask_cross.astype(bool)] = 0.5 ax1.imshow(mask_strip, cmap="gray_r") ax1.plot(star_pos_A[0][0], star_pos_A[0][1], "r*", ms=18) ax1.set_title("Strip/Cross") n_label = seg_comb.max() ax2.imshow(seg_comb, vmin=1, vmax=n_label - 3, cmap=make_rand_cmap(n_label)) ax2.plot(star_pos_A[:, 0], star_pos_A[:, 1], "r*", ms=18) ax2.set_title("Mask Comb.") image3 = image.copy() image3[mask_comb] = 0 im3 = ax3.imshow(image3, norm=LogNorm(), aspect='auto', vmin=vmin, vmax=vmax) ax3.plot(star_pos_A[:, 0], star_pos_A[:, 1], "r*", ms=18) ax3.set_title("'Sky'") colorbar(im3) if r_core is not None: if np.ndim(r_core) == 0: r_core = [r_core, r_core] aper = CircularAperture(star_pos_A, r=r_core[0]) aper.plot(color='lime', lw=2, label="", alpha=0.9, axes=ax3) aper = CircularAperture(star_pos_B, r=r_core[1]) aper.plot(color='c', lw=2, label="", alpha=0.7, axes=ax3) size = image.shape[0] - pad * 2 rec = patches.Rectangle((pad, pad), size, size, facecolor='none', edgecolor='w', linewidth=2, linestyle='--', alpha=0.8) ax3.add_patch(rec) plt.tight_layout() if save: plt.savefig(os.path.join(save_dir, "Mask_strip.png"), dpi=120) plt.show() plt.close() else: plt.show()
def map_to_gamma_template(in_file, out_file=None, no_neg=False, smooth_arcmin=0.0, rm_median=False, flag_dist_arcmin=0.0, normalize=True): """ PURPOSE: This function extracts a map from a fits fil and modifies it applying smoothing, baseline removal, flag of negative values, and masking. The new image is then writtin in a new file. INPUT: - in_file (string): input file full name - out_file (string): (output file full name) - no_deg (bool): set negative values to zero (default is no) - smooth_arcmin (float): Gaussian smoothing FWHM, in arcmin, to apply - rm_median (bool): remove the median of the map (default is no) - flag_dist_arcmin (float): set to zero pixels beyond this limit, from the center - normalize (bool): set this keyword to normalize the map to 1 (i.e. setting the integral sum(map)*reso^2 to 1) OUTPUT: - The new map is written in a new file - image (2d numpy array): the new map - header : the corresponding map header """ #---------- Data extraction data = fits.open(in_file)[0] image = data.data header = data.header wcs = WCS(header) reso_x = abs(wcs.wcs.cdelt[0]) reso_y = abs(wcs.wcs.cdelt[1]) Npixx = image.shape[0] Npixy = image.shape[1] #---------- Data modification wnan = np.isnan(image) image[wnan] = 0.0 winf = np.isinf(image) image[winf] = 0.0 if smooth_arcmin >= 0: sigma_sm = smooth_arcmin / 60.0 / np.array( [reso_x, reso_x]) / (2 * np.sqrt(2 * np.log(2))) image = ndimage.gaussian_filter(image, sigma=sigma_sm) if rm_median: image = image - np.nanmedian(image) if no_neg: image[image < 0] = 0.0 if flag_dist_arcmin > 0: ra_map, dec_map = get_radec_map(data.header) ra_ctr = np.mean(ra_map) dec_ctr = np.mean(dec_map) distmap = greatcircle(ra_map, dec_map, ra_ctr, dec_ctr) image[distmap > flag_dist_arcmin / 60.0] = 0.0 if normalize == True: norm = np.sum(image) * reso_x * reso_y * (np.pi / 180.0)**2 image = image / norm #---------- Write FITS if out_file is not None: hdu = fits.PrimaryHDU(header=header) hdu.data = image hdu.writeto(out_file, overwrite=True) return image, header
def check_pointing_using_2mass_catalog(filename, out_dir='./'): """Check that stars from an external 2MASS catalog are present at the expected locations within filename. Parameters ---------- filename : str Name of fits file containing the observation to check. This file must contain a valid GWCS (i.e. it must have gone through the assign_wcs pipeline step.) out_dir : str Name of directory into which source catalogs are written Returns ------- med_dist : float Median offset between expected and measured source locations, in arcsec dev_dist : float Standard deviation of the offset between expected and measured source locations, in arcsec mean_unc : float Mean uncertainty in the source locoations in the 2MASS catalog, in arcsec d2d_arcsec : list Offsets between expected and measured source locations for all matched sources, in arcsec """ print("Working on: {}".format(os.path.basename(filename))) model = ImageModel(filename) pix_scale = model.meta.wcsinfo.cdelt1 * 3600. # Read in catalog from 2MASS. We'll be using the positions in this # catalog as "truth" catalog_2mass = os.path.join(os.path.dirname(__file__), 'car19_2MASS_source_catalog.txt') cat_2mass = ascii.read(catalog_2mass) ra_unc = cat_2mass['err_maj'] * u.arcsec dec_unc = cat_2mass['err_min'] * u.arcsec total_unc = np.sqrt(ra_unc**2 + dec_unc**2) # Calculate the pixel corresponding to the RA, Dec value of the stars # in the 2MASS catalog world2det = model.meta.wcs.get_transform('world', 'detector') star_x, star_y = world2det(cat_2mass['ra'], cat_2mass['dec']) cat_2mass['x'] = star_x cat_2mass['y'] = star_y # Remove stars that are not on the detector ydim, xdim = model.data.shape good = ((star_x > 0) & (star_x < xdim) & (star_y > 0) & (star_y < ydim)) print('{} 2MASS sources should be present on the detector.'.format( np.sum(good))) if np.sum(good) == 0: print('Skipping.\n') return None, None, None, None cat_2mass = cat_2mass[good] # Find sources in the data sub_fwhm = get_fwhm(model.meta.instrument.filter) plot_file = '{}_source_map.png'.format(os.path.basename(filename)) plot_file = os.path.join(out_dir, plot_file) sources = find_sources(model.data, threshold=50, fwhm=sub_fwhm, plot_name=plot_file) # Calculate RA, Dec of the sources in the new catalog det2world = model.meta.wcs.get_transform('detector', 'world') found_ra, found_dec = det2world(sources['xcentroid'].data, sources['ycentroid'].data) sources['calc_RA'] = found_ra sources['calc_Dec'] = found_dec # Match catalogs found_sources = SkyCoord(ra=found_ra * u.degree, dec=found_dec * u.degree) from_2mass = SkyCoord(ra=cat_2mass['ra'] * u.degree, dec=cat_2mass['dec'] * u.degree) idx, d2d, d3d = from_2mass.match_to_catalog_3d(found_sources) # Print table of sources d2d_arcsec = d2d.to(u.arcsec).value cat_2mass['image_x'] = sources[idx]['xcentroid'] cat_2mass['image_y'] = sources[idx]['ycentroid'] cat_2mass['image_ra'] = sources[idx]['calc_RA'] cat_2mass['image_dec'] = sources[idx]['calc_Dec'] cat_2mass['delta_sky'] = d2d_arcsec cat_2mass['delta_pix'] = d2d_arcsec / pix_scale for colname in ['x', 'y', 'image_x', 'image_y', 'delta_pix']: cat_2mass[colname].info.format = '7.3f' print(cat_2mass['x', 'y', 'image_x', 'image_y', 'delta_pix']) # Save the table of sources table_file = 'comp_found_sources_to_2MASS_{}.txt'.format( os.path.basename(filename)) table_file = os.path.join(out_dir, table_file) print( 'Table of source location comparison saved to: {}'.format(table_file)) ascii.write(cat_2mass, table_file, overwrite=True) # Get info on median offsets med_dist = np.nanmedian(d2d_arcsec) dev_dist = np.nanstd(d2d_arcsec) print(( "Median distance between sources in 2MASS catalog and those found in the " "data: {0:1.2f} arcsec = {1:1.2f} pixels".format( med_dist, med_dist / pix_scale))) mean_unc = np.mean(total_unc) mean_unc_pix = np.mean(total_unc.value) / pix_scale print(( "Mean uncertainty in the source locations within the 2MASS catalog: {0:1.2f} = {1:1.2f} " "pixels\n\n".format(mean_unc, mean_unc_pix))) return med_dist, dev_dist, mean_unc, d2d_arcsec
observations = data['Classification'].loc[data['Obj ID'] == obj] if any(observations == 'EB'): print('At least one EB found!') objTable = data.loc[data['Obj ID'] == obj] classification = objTable['Classification'].tolist() files = objTable['Filename'].copy() i = 0 # Plot all observations on the same light curve plt.figure(figsize=(20, 6)) for file in files: fitsTable = fits.open(file, memmap=True) curveTable = Table(fitsTable[1].data).to_pandas() curveData = curveTable.loc[curveTable['QUALITY'] == 0].dropna( subset=['TIME']).dropna(subset=['PDCSAP_FLUX']).copy() fluxMed = np.nanmedian(curveData['PDCSAP_FLUX']) curveData['REL_FLUX'] = curveData['PDCSAP_FLUX'].div(fluxMed) curveData['REL_FLUX_ERR'] = curveData['PDCSAP_FLUX_ERR'].div( fluxMed) # Determine EB/non-EB and colorize accordingly if classification[i] == 'EB': color = 'tab:purple' else: color = 'tab:gray' figName = obj + '.png' plt.scatter(curveData['TIME'], curveData['REL_FLUX'], color=color, s=.2)
def pixel_value(self): """The median pixel value of the ROI.""" masked_img = self.circle_mask() return np.nanmedian(masked_img)
train["Dependents"] = train["Dependents"].replace("3+", 3) train["Dependents"] = train["Dependents"].replace("0", 0) new_dependents = np.where(train["Dependents"].isnull(), 0, train["Dependents"]) train["Dependents"] = new_dependents train["Dependents"] = pd.to_numeric(train["Dependents"]) train["Dependents"].dtypes train["Dependents"].isnull().sum() train["Self_Employed"].value_counts() new_Self_Employed = np.where(train["Self_Employed"].isnull(), "No", train["Self_Employed"]) train["Self_Employed"] = new_Self_Employed train["Self_Employed"].isnull().sum() train["LoanAmount"] = np.where(train["LoanAmount"].isnull(), np.nanmedian(train["LoanAmount"]), train["LoanAmount"]) train["LoanAmount"].isnull().sum() train["Loan_Amount_Term"].value_counts() train["Loan_Amount_Term"] = np.where(train["Loan_Amount_Term"].isnull(), 360, train["Loan_Amount_Term"]) train["Loan_Amount_Term"].isnull().sum() train["Credit_History"].value_counts() train["Credit_History"] = np.where(train["Credit_History"].isnull(), 1.0, train["Credit_History"]) train["Credit_History"].isnull().sum() check_missing = train.isnull().sum()
try: bbig = np.concatenate(bbig, axis=0) ffin = np.concatenate(ffin, axis=0) aarr = np.concatenate(aarr, axis=0) taarr = np.concatenate(taarr, axis=0) nnz = np.concatenate(nnz, axis=0) except ValueError: print('return') bla = np.nansum(aarr, 0) / np.nansum(nnz, 0) bla1 = np.nansum(ffin, 0) blab = np.nansum(bbig, 0) / np.nansum(ffin,0) * 100 tbla = np.nanmedian(taarr, 0) out.append((bla,bla1,blab, tbla)) ######### 2d plots f = plt.figure(figsize=(12, 5), dpi=400) ll = [30, 60, 90, 120, 180] # keys lll = [15, 30, 60, 90, 120,120] for ind, k in enumerate(ll):
def readLogFile(fnameOrFolder, subtractDark=False, skip_first=0, asDataStorage=True, last=None, srcur_min=30): """ read id9 style logfile; last before data will be used as keys ... only srcur>srcur_min will be kept subtractDark is not needed for data collected with waxscollect """ if os.path.isdir(fnameOrFolder): fname = findLogFile(fnameOrFolder) else: fname = fnameOrFolder log.info("Reading id9 logfile: %s" % fname) data = utils.files.readLogFile(fname,skip_first=skip_first,last=last,\ output = "array",converters=dict(delay=_delayToNum)) # work on darks if needed if subtractDark: ## find darks with open(fname, "r") as f: lines = f.readlines() lines = [line.strip() for line in lines] # look only for comment lines lines = [line for line in lines if line[0] == "#"] for line in lines: if line.find("pd1 dark/sec") >= 0: darks['pd1ic'] = _findDark(line) if line.find("pd2 dark/sec") >= 0: darks['pd2ic'] = _findDark(line) if line.find("pd3 dark/sec") >= 0: darks['pd3ic'] = _findDark(line) ## subtract darks for diode in ['pd1ic', 'pd2ic', 'pd3ic', 'pd4ic']: if diode in darks: data[diode] = data[diode] - darks[diode] * data['timeic'] # srcur filter if "currentmA" in data.dtype.names: idx_cur = data['currentmA'] > srcur_min if (idx_cur.sum() < idx_cur.shape[0] * 0.5): log.warn("Minimum srcur filter has kept only %.1f%%" % (idx_cur.sum() / idx_cur.shape[0] * 100)) log.warn("Minimum srcur: %.2f, median(srcur): %.2f" % (srcur_min, np.nanmedian(data["currentmA"]))) data = data[idx_cur] else: log.warn("Could not find currentmA in logfile, skipping filtering") info = DataStorage() # usually folders are named sample/run if os.path.isdir(fnameOrFolder): folder = fnameOrFolder else: folder = os.path.dirname(fnameOrFolder) dirs = folder.split(os.path.sep) ylabel = ".".join(dirs[-2:]) info.name = ".".join(dirs[-2:]) try: reprate = readReprate(fname) info.reprate = reprate ylabel += " %.2f Hz" % reprate except: print("Could not read rep rate info") try: time_info = timesToInfo(data['time']) info.duration = time_info ylabel += "\n" + time_info except: print("Could not read time duration info") info.ylabel = ylabel if asDataStorage: data = DataStorage( dict((name, data[name]) for name in data.dtype.names)) return data, info
def median_performance_reached_in_last_ntrials(mid, df, n_trials=500): mouse_df = df[df['animal_id'] == mid] max_performance_reached = np.nanmedian(mouse_df['cumulative_performance'][-n_trials:]) return max_performance_reached
# run orthologs loop print("MODE: re-cluster orthogroups with ETE(SO)+MCL") logging.info("Analyse orthogroups in %s" % (phy_fo)) orthogroup_loop() print("DONE") elif ani == "opti": # run optimisation loop print("MODE: inflation optimisation with %i random phylogenies" % nopt) logging.info("Find optimal inflation") inf_lis, mod_lis = optimisation_loop(nopt=nopt) # plot optimisation histograms print("# median inflation is %f" % np.nanmedian(inf_lis)) with PdfPages('%s.optimise_inflation.pdf' % out_fn) as pdf: # inflation plt.figure(figsize=(4,3)) plt.title("Hist inflation I\nmedian = %f" % np.nanmedian(inf_lis)) plt.hist(inf_lis[~np.isnan(inf_lis)]) pdf.savefig(bbox_inches='tight') # modularity plt.figure(figsize=(4,3)) plt.title("Hist modularity Q\nmedian = %f" % np.nanmedian(mod_lis)) plt.hist(mod_lis[~np.isnan(mod_lis)]) pdf.savefig(bbox_inches='tight') # close pdf plt.close() print("DONE")
nicest=hdulist2[0].data[:,:] meanxcousedata = (~np.isnan(mom012))&(nicest>0)&(mom012>3*mom0rms) tex=hdulist3[0].data[:,:] av = nicest/9.4e20/2. print 'average NH',np.nanmean(nicest[meanxcousedata]),'average mom0',np.nanmean(mom012[meanxcousedata]),'mean Xco',np.nanmean(nicest[meanxcousedata])/np.nanmean(mom012[meanxcousedata]) hdulist1.close() hdulist2.close() hdulist3.close() rawxco = nicest/mom012 hdulist4[0].data = nicest/mom012 hdulist4[0].data[~meanxcousedata] = np.nan #hdulist4.writeto('XCO.fits', output_verify='exception', clobber=True, checksum=False) print 'nanmin XCO',np.nanmin(hdulist4[0].data),'nanmax XCO',np.nanmax(hdulist4[0].data),'nanmedian XCO',np.nanmedian(hdulist4[0].data) def plothist(data,pdfname): xx = data[~np.isnan(data)] #xx = xxx[xxx<1.e2] x = xx[xx>0] p=plt.figure(figsize=(7,6)) fig, ax = plt.subplots(1,1) # the histogram of the data n, bins, patches = plt.hist(x, 100, histtype='step', color='k') #print n,bins #plt.xscale('log') #plt.yscale('log') plt.xlabel(r'$X_{\rm CO}~({\rm cm}^{-2}~{\rm (K~km~s^{-1})}^{-1})$') plt.ylabel('counts') plt.xlim(0,4.e20)
data = pd.concat([Train_data['SalePrice'], Train_data['GrLivArea']], axis=1) data.plot.scatter(x='GrLivArea', y='SalePrice', ylim=(0,800000)); #Seperating the character and numeric columns for data visualization Train_data_char = Train_data[Character_columns_Train] Train_data_num = Train_data[Numeric_columns_Train] #correlation matrix corrmat = Train_data.corr() f, ax = plt.subplots(figsize=(15, 12)) sns.heatmap(corrmat, vmax=.8, square=True); #Preprocessing the data by combining both Test and Train data Train_data['isTrain'] = pd.Series(1, index = Train_data.index) Test_data['isTrain'] = pd.Series(0, index = Test_data.index) house = pd.concat([Train_data, Test_data]) house['MasVnrArea'] = house.MasVnrArea.replace(np.NaN, np.mean(house['MasVnrArea'])) #Replacing null values with mean values house['LotFrontage'] = house.LotFrontage.replace(np.NaN, np.nanmedian(house['LotFrontage'])) #Replacing null values with median values house['Street'] = house['Street'].astype('category') #Label Encoder implement here***** house[Character_columns_Train] = house[Character_columns_Train].astype('category') house[Character_columns_Train] = house[Character_columns_Train].apply(lambda x: x.cat.codes)
pcp_ens = np.zeros( (np.shape(pcp)[0], np.shape(pcp)[1], np.shape(pcp)[2], ens_num)) # create ens array for one member tmean_ens = np.zeros( (np.shape(pcp)[0], np.shape(pcp)[1], np.shape(pcp)[2], ens_num)) trange_ens = np.zeros( (np.shape(pcp)[0], np.shape(pcp)[1], np.shape(pcp)[2], ens_num)) pcp_ens[:, :, :, i] = pcp tmean_ens[:, :, :, i] = tmean trange_ens[:, :, :, i] = trange #================================================================================= # calculate time-series mean values of mean and std. (time,y,x) pcp_ens_mean = np.nanmean(pcp_ens, axis=3) pcp_ens_median = np.nanmedian(pcp_ens, axis=3) pcp_ens_std = np.std(pcp_ens, axis=3) pcp_ens_lb = np.percentile(pcp_ens, lb_perct, axis=3) pcp_ens_ub = np.percentile(pcp_ens, ub_perct, axis=3) del pcp_ens tmean_ens_mean = np.nanmean(tmean_ens, axis=3) tmean_ens_median = np.nanmedian(tmean_ens, axis=3) tmean_ens_std = np.std(tmean_ens, axis=3) tmean_ens_lb = np.percentile(tmean_ens, lb_perct, axis=3) tmean_ens_ub = np.percentile(tmean_ens, ub_perct, axis=3) del tmean_ens trange_ens_mean = np.nanmean(trange_ens, axis=3) trange_ens_median = np.nanmedian(trange_ens, axis=3) trange_ens_std = np.std(trange_ens, axis=3)
def measure_azimuthal_profile(data, star, add_noise=False, model=False): radii_data = make_radii(data, star) # Mask region outside data's PSF subtraction zone and inside IWA. r_mask = 9 data[radii_data >= 135] = np.nan data[radii_data < r_mask] = np.nan # # Scale data brightness by sb_scale. # data *= sb_scale # OFF! Model flux calibration. # data *= conv_WtoJy*sb_scale*data # [mJy arcsec^-2] # Add Gaussian noise to image if desired. if add_noise: data += np.random.normal(scale=0.1, size=data.shape) # Convolve model with Gaussian simulating instrument PSF. if model: data = filters.gaussian_filter(data.copy(), 1.2) #1.31) fontSize = 16 # Clean up the edges of the data, which often have artifacts. # Do this by smoothing the data to expand the NaN's already present outside # the edges, ignoring the inner mask, and creating a new mask from this. mask_edges = 2. if mask_edges: mask = np.ma.masked_invalid(filters.gaussian_filter(data, mask_edges)).mask mask[star[0] - r_mask * 4:star[0] + r_mask * 4, star[1] - r_mask * 4:star[1] + r_mask * 4] = False data = np.ma.masked_array(data, mask=mask).filled(np.nan) # Brightness cuts. cut_colors = ['C4', 'C9', 'C2', 'C8', 'C4', 'C9', 'C2', 'C8'] bw = 3 # width of slice [pix] fs = 6 # filter size [pix] data_rot45 = rot_array(data, star, -45 * np.pi / 180) # data rotated -45 deg sl_pa0 = np.sum(data[:, star[1] - (bw // 2):star[1] + (bw // 2) + 1], axis=1)[::-1] # data PA=0 slice sl_pa45 = np.sum(data_rot45[:, star[1] - (bw // 2):star[1] + (bw // 2) + 1], axis=1)[::-1] # data PA=45 slice sl_pa90 = np.sum(data[star[0] - (bw // 2):star[1] + (bw // 2) + 1], axis=0) # data PA=90 slice sl_pa135 = np.sum(data_rot45[star[0] - (bw // 2):star[1] + (bw // 2) + 1], axis=0) # data PA=135 slice sl_list = np.array([sl_pa0, sl_pa45, sl_pa90, sl_pa135]) pa_sls = [0, 45, 90, 135] # Estimate 'noise' as median of all slices. sl_noise = np.nanmedian(sl_list, axis=0) sl_rms = np.sqrt(np.nanmedian(sl_list**2, axis=0)) # 'root median square' sl_med = np.sqrt(np.nanmedian(sl_list**2)) # Find S/N ratio. # sl_snr = sl_list/filters.median_filter(sl_rms, fs) sl_snr = np.array( [filters.median_filter(sl_list[ii], fs) for ii in range(len(sl_list))]) / sl_med # Plot the image with lines marking PA angles measured. fig0 = plt.figure(3) plt.clf() plt.subplot(111) plt.subplots_adjust(bottom=0.1, right=0.99, top=0.91, left=0.05) im = plt.imshow(data, extent=[-140, 140, -140, 140]) plt.clim(np.percentile(np.nan_to_num(data), 1.5), np.percentile(np.nan_to_num(data), 99.99)) # Plot lines tracing PA=0, 45, 90, 135. ext = im.get_extent() # [-x, +x, -y, +y] plt.plot([0, 0], [ext[2], ext[3]], 'r--', alpha=0.5, linewidth=1) plt.plot([ext[3], ext[2]], [ext[0], ext[1]], 'r--', alpha=0.5, linewidth=1) plt.plot([ext[2], ext[3]], [0, 0], 'r--', alpha=0.5, linewidth=1) plt.plot([ext[1], ext[0]], [ext[3], ext[2]], 'r--', alpha=0.5, linewidth=1) plt.text(5, ext[3] - 15, '-0', fontsize=12) plt.text(ext[0] + 15, ext[3] - 15, '-45', fontsize=12) plt.text(ext[0] + 5, 5, '-90', fontsize=12) plt.text(ext[0] + 15, ext[2] + 5, '-135', fontsize=12) plt.title(fn + "\nPA = 0 is north up", fontsize=10) plt.draw() # Plot surface brightness v. projected separation. fig1 = plt.figure(5) plt.clf() plt.subplot(111) plt.subplots_adjust(bottom=0.13, right=0.98, top=0.91, left=0.15) plt.axvspan(xmin=-r_mask, xmax=r_mask, color='k', alpha=0.1) plt.axvline(x=0, c='0.4', linestyle='--', linewidth=1.) plt.axhline(y=0, c='0.4', linestyle='--', linewidth=1.) label_sls = 4 * ['data'] #4*['mod'] + 4*['data'] for ii, sl in enumerate(sl_list): plt.plot(range(0, data.shape[0]) - star[0], sl, label='%s %d' % (label_sls[ii], pa_sls[ii]), c=cut_colors[ii], marker='.', linestyle='None', alpha=0.5) plt.plot(range(0, data.shape[0]) - star[0], filters.median_filter(sl_noise, fs), label='med', c='k', linestyle=':', linewidth=2.) plt.plot(range(0, data.shape[0]) - star[0], sl_rms, label='rms', c='m', linestyle='-', linewidth=1.5) plt.ylabel("Surface Brightness") plt.xlabel("Projected Separation (px)") plt.title('%s\nsl %d px, filt %d px' % (fn, bw, fs), fontsize=10) plt.legend(ncol=2, fontsize=fontSize - 4, labelspacing=0.2, columnspacing=1, borderpad=0.4, handletextpad=0.3, handlelength=1, framealpha=0.5) plt.draw() # Plot S/N-like things. fig2 = plt.figure(6) plt.clf() plt.subplot(111) plt.subplots_adjust(bottom=0.13, right=0.98, top=0.91, left=0.15) plt.axvspan(xmin=-r_mask, xmax=r_mask, color='k', alpha=0.1) plt.axvline(x=0, c='0.4', linestyle='--', linewidth=1.) plt.axhline(y=0, c='0.4', linestyle='--', linewidth=1.) # for ii, sl in enumerate(sl_list[4:]): # plt.plot(range(0,data_Qr.shape[0])-star[0], sl_dQr_snr[ii], label='%d' % pa_sls[ii], c=cut_colors[ii], linestyle='-', linewidth=1.5) for ii, sl in enumerate(sl_list): plt.plot(range(0, data.shape[0]) - star[0], sl_snr[ii], label='%d' % pa_sls[ii], c=cut_colors[ii], linestyle='-', linewidth=1.5) plt.axhline(y=3, c='0.2', linestyle='--', linewidth=1.5, label='"3$\sigma$"', zorder=0) plt.ylim(-5, 20) plt.ylabel("Pseudo S/N") plt.xlabel("Projected Separation (px)") plt.title('%s\nsl %d px, filt %d px' % (fn, bw, fs), fontsize=10) plt.legend(ncol=2, fontsize=fontSize - 4, labelspacing=0.2, columnspacing=1, borderpad=0.4, handletextpad=0.3, handlelength=1, framealpha=0.5) plt.draw() # Plot folded S/N-like things. fig3 = plt.figure(7) plt.clf() plt.subplot(111) plt.subplots_adjust(bottom=0.13, right=0.98, top=0.91, left=0.15) plt.axvspan(xmin=0, xmax=r_mask, color='k', alpha=0.1) plt.axvline(x=0, c='0.4', linestyle='--', linewidth=1.) plt.axhline(y=0, c='0.4', linestyle='--', linewidth=1.) # for ii, sl in enumerate(sl_list[4:]): # plt.plot(range(0,data_Qr.shape[0])-star[0], sl_dQr_snr[ii], label='%d' % pa_sls[ii], c=cut_colors[ii], linestyle='-', linewidth=1.5) folded_snr = np.array([ sl_snr[ii][0:sl_snr[ii].shape[0] // 2][::-1] + sl_snr[ii][1 + sl_snr[ii].shape[0] // 2:] for ii in range(len(sl_list)) ]) folded_mean = np.nanmean(folded_snr, axis=0) for ii, sl in enumerate(sl_list[:4]): plt.plot(range(0, data.shape[0] // 2), folded_snr[ii], label='%d' % pa_sls[ii], c=cut_colors[ii], linestyle='-', linewidth=1.5) plt.plot(range(0, data.shape[0] // 2), folded_mean, label='Mean', c='k', linestyle='-', linewidth=2.5) plt.axhline(y=3, c='0.2', linestyle='--', linewidth=1.5, label='"3$\sigma$"', zorder=0) plt.ylim(-5, 20) plt.ylabel("Folded Pseudo S/N") plt.xlabel("Projected Separation (px)") plt.title('%s\nsl %d px, filt %d px' % (fn, bw, fs), fontsize=10) plt.legend(ncol=2, fontsize=fontSize - 4, labelspacing=0.2, columnspacing=1, borderpad=0.4, handletextpad=0.3, handlelength=1, framealpha=0.5) plt.draw() # Compile list of hits. snr_hits = np.where(sl_snr >= 3) folded_snr_hits = np.where(folded_snr >= 3) folded_mean_hits = np.where(folded_mean >= 3) N_snr_hits = snr_hits[0].shape[0] N_folded_snr_hits = folded_snr_hits[0].shape[0] N_folded_mean_hits = folded_mean_hits[0].shape[0] print("\nS/N > 3: %d (%d%%)" % (N_snr_hits, 100 * N_snr_hits / sl_snr.size)) print("Folded S/N > 3: %d (%d%%)" % (N_folded_snr_hits, 100 * N_folded_snr_hits / folded_snr.size)) print("Folded Mean S/N > 3: %d (%d%%)" % (N_folded_mean_hits, 100 * N_folded_mean_hits / folded_mean.size)) print("Max slice S/N: %.1f" % np.nanmax(sl_snr)) # Plot flux contours in 3D. # data3d = data.copy() # Smooth slightly and trim negative values for better contouring. data3d = filters.gaussian_filter(np.nan_to_num(data.copy()), 2.) data3d[data3d < 0.] = np.nan fig3d = plt.figure(8) plt.clf() ax = fig3d.add_subplot(111, projection='3d') yy, xx = np.mgrid[:data3d.shape[0], :data3d.shape[1]] yy -= star[0] xx -= star[1] ax.plot([0, 0], [0, 0], [0, np.nanmax(data3d)], 'k--') nlevels = 20 c3d = ax.contour(xx, yy, data3d, levels=nlevels) ax.set_xlabel('X') ax.set_ylabel('Y') ax.set_zlabel('Flux') ax.set_title('%s\n%d contours' % (fn, nlevels), fontsize=10, pad=40) plt.draw() # fig3 = plt.figure(33) # ax1 = plt.subplot(111) # # ax1.imshow((radii**2)*conv_WtoJy*I_scale*mod_hdu[0].data[0,0,0]/1000., vmin=vmin, vmax=vmax, cmap=colmap_devs) #viridis) # # ax1.imshow(np.log10(conv_WtoJy*I_scale*mod_hdu[0].data[0,0,0]), vmin=vmin*2., vmax=vmax*2., cmap=colmap_devs) #viridis) # from matplotlib.colors import SymLogNorm # ax1.imshow(conv_WtoJy*I_scale*mod_hdu[0].data[0,0,i_inc], vmin=0., vmax=vmax*10., cmap=colmap, # norm=SymLogNorm(linthresh=0.05, linscale=1.0, vmin=0., vmax=vmax*10.)) # ax1.set_xlim(star[1]-wdw[1], star[1]+wdw[1]) # ax1.set_ylim(star[0]-wdw[0], star[0]+wdw[0]) # fpm_circle = plt.Circle(star[::-1], 8.68, color='k', fill=False) # ax1.add_artist(fpm_circle) # # plt.title('%.2f x Raw Model * r^2' % I_scale) # plt.title('%.2f x Raw Model I' % I_scale) # ax1.tick_params(which='both', direction='in', color=ctick) # ax1.yaxis.set_ticks(np.arange(90, 191, 50)) # ax1.xaxis.set_ticks(np.arange(90, 191, 50)) # # ax1.yaxis.set_minor_locator(minor_locator) # # ax1.xaxis.set_minor_locator(minor_locator) # ax1.grid(axis='both', which='both', c='0.6', alpha=0.5) # # plt.draw() # if save: # fig0.savefig(os.path.expanduser('~/Research/data/gpi/auto_disk/%s_autodisk_imagecuts' % fn) + '.png', dpi=200, transparent=False, format='png') # fig2.savefig(os.path.expanduser('~/Research/data/gpi/auto_disk/%s_autodisk_slsnr' % fn) + '.png', dpi=200, transparent=False, format='png') # fig3.savefig(os.path.expanduser('~/Research/data/gpi/auto_disk/%s_autodisk_foldedsnr' % fn) + '.png', dpi=200, transparent=False, format='png') # # # Compute the disk brightness; feed in units of ADU/s. # # Assume data are already in ADU/coadd (default GPI pipeline output). # itime = data_hdu[hdu_ind].header['ITIME'] # [s] integration time per coadd # auto_disk_brightness(data/itime, data_Ur/itime, star) # pdb.set_trace() if output: return [ np.nanmax(sl_snr), N_snr_hits, 100 * N_snr_hits / sl_snr.size, N_folded_snr_hits, 100 * N_folded_snr_hits / folded_snr.size, N_folded_mean_hits, 100 * N_folded_mean_hits / folded_mean.size ] else: return
def _perform(self): # image sections for each amp bsec, dsec, tsec, direc = self.action.args.map_ccd namps = len(bsec) # polynomial fit order if namps == 4: porder = 2 else: porder = 7 # header keyword to update key = 'OSCANSUB' keycom = 'Overscan subtracted?' # is it performed? performed = False # loop over amps # plts = [] # plots for each amp for ia in range(namps): # get gain gain = self.action.args.ccddata.header['GAIN%d' % (ia + 1)] # check if we have enough data to fit if (bsec[ia][3] - bsec[ia][2]) > self.config.instrument.minoscanpix: # pull out an overscan vector x0 = bsec[ia][2] + self.config.instrument.oscanbuf x1 = bsec[ia][3] - self.config.instrument.oscanbuf y0 = bsec[ia][0] y1 = bsec[ia][1] + 1 osvec = np.nanmedian(self.action.args.ccddata.data[y0:y1, x0:x1], axis=1) nsam = x1 - x0 xx = np.arange(len(osvec), dtype=np.float) # fit it, avoiding first 50 px if direc[ia]: # forward read skips first 50 px oscoef = np.polyfit(xx[50:], osvec[50:], porder) else: # reverse read skips last 50 px oscoef = np.polyfit(xx[:-50], osvec[:-50], porder) # generate fitted overscan vector for full range osfit = np.polyval(oscoef, xx) # calculate residuals resid = (osvec - osfit) * math.sqrt(nsam) * gain / 1.414 sdrs = float("%.3f" % np.std(resid)) self.logger.info("Amp%d Read noise from oscan in e-: %.3f" % ((ia + 1), sdrs)) self.action.args.ccddata.header['OSCNRN%d' % (ia + 1)] = \ (sdrs, "amp%d RN in e- from oscan" % (ia + 1)) if self.config.instrument.plot_level >= 1: x = np.arange(len(osvec)) p = figure(title=self.action.args.plotlabel + 'OSCAN amp %d' % (ia + 1), x_axis_label='overscan px', y_axis_label='counts', plot_width=self.config.instrument.plot_width, plot_height=self.config.instrument.plot_height) p.circle(x, osvec, legend_label="Data") p.line(x, osfit, line_color='red', line_width=3, legend_label="Fit") bokeh_plot(p, self.context.bokeh_session) # plts.append(p) if self.config.instrument.plot_level >= 2: input("Next? <cr>: ") else: time.sleep(self.config.instrument.plot_pause) # subtract it for ix in range(dsec[ia][2], dsec[ia][3] + 1): self.action.args.ccddata.data[y0:y1, ix] = \ self.action.args.ccddata.data[y0:y1, ix] - osfit performed = True else: self.logger.info("not enough overscan px to fit amp %d") # if self.config.instrument.plot_level >= 1 and len(plts) > 0: # bokeh_plot(gridplot(plts, ncols=(2 if namps > 2 else 1), # plot_width=500, plot_height=300, # toolbar_location=None), # self.context.bokeh_session) # if self.config.instrument.plot_level >= 2: # input("Next? <cr>: ") # else: # time.sleep(self.config.instrument.plot_pause) self.action.args.ccddata.header[key] = (performed, keycom) log_string = SubtractOverscan.__module__ self.action.args.ccddata.header['HISTORY'] = log_string self.logger.info(log_string) return self.action.args
O_E__JK.append(O_E__K), OplusE__JK.append( OplusE__K), O_E_grad__JK.append(O_E_grad__K) print("DEBUG O_Eshapes (-,+,-_-grad):\t{}, {}, {}".format( np.array(O_E__K).shape, np.array(OplusE__K).shape, np.array(O_E_grad__K).shape)) slitdiffnorm__K = np.array(O_E_grad__K) / np.array(OplusE__K) # Determine the Stokes parameter and polarization degrees if filtername == "b_HIGH": offsetangle = 2. * 1.54 elif filtername == "v_HIGH": offsetangle = 2. * 1.8 UQPphi, sigma_UQPphi = polfun.detpol(slitdiffnorm__K, np.nanmedian(np.sqrt(OplusE__K), axis=0), offsxy0__45=[Qoffsx, Qoffsy], offsxy22_5__67_5=[Uoffsx, Uoffsy], offsxy0__22_5=[QUoffsx, QUoffsy], corran=offsetangle) # Append results to list UQPphi_J.append(UQPphi), sigma_UQPphi_J.append(sigma_UQPphi) # Save results to fits files polfun.savefits(UQPphi[1], imsavedir + "/tpl{}".format(j + 1), "Q_i{}j{}".format(i + 1, j + 1)) polfun.savefits(UQPphi[0], imsavedir + "/tpl{}".format(j + 1), "U_i{}j{}".format(i + 1, j + 1)) polfun.savefits(UQPphi[2], imsavedir + "/tpl{}".format(j + 1), "P_i{}j{}".format(i + 1, j + 1)) polfun.savefits(sigma_UQPphi[1], imsavedir + "/tpl{}".format(j + 1), "sigmaQ_i{}j{}".format(i + 1, j + 1))
def printcoords(event): global dph, lines1, lines2, lastevent, imdates_ordinal, imdates_dt # outputting x and y coords to console if event.inaxes != axv: return elif event.button != 1: return elif not event.dblclick: ## Only double click return else: lastevent = event ## Update last event ii = np.int(np.round(event.ydata)) jj = np.int(np.round(event.xdata)) ### Plot on image window ii1h = ii - 0.5; ii2h = ii + 1 - 0.5 jj1h = jj - 0.5; jj2h = jj + 1 - 0.5 pax.set_data([jj1h, jj2h, jj2h, jj1h, jj1h], [ii1h, ii1h, ii2h, ii2h, ii1h]) pax2.set_data(jj, ii) pv.canvas.draw() axts.cla() axts.grid(zorder=0) axts.set_axisbelow(True) axts.set_xlabel('Date') axts.set_ylabel('Cumulative Displacement (mm)') axts.set_ylim(dmin, dmax) ### Get values of noise indices and incidence angle noisetxt = '' for key in mapdict_data: # val_temp = mapdict_data[key] val = mapdict_data[key][ii, jj] # val = val_temp[0,ii,jj] unit = mapdict_unit[key] if key.startswith('Velocity'): continue elif key.startswith('n_') or key == 'mask': noisetxt = noisetxt + '{}: {:d} {}\n'.format(key, int(val), unit) else: noisetxt = noisetxt + '{}: {:.2f} {}\n'.format(key, float(val), unit) try: # Only support from Matplotlib 3.1! axts.xaxis.set_major_formatter(mdates.ConciseDateFormatter(loc_ts)) except: axts.xaxis.set_major_formatter(mdates.DateFormatter('%Y/%m/%d')) for label in axts.get_xticklabels(): label.set_rotation(20) label.set_horizontalalignment('right') ### If not masked ### tscuml file vel1p = vel[0, ii, jj] intercept1p = intercept[0,ii,jj] dph = tscuml[:,ii,jj] err = np.ones(dph.shape) * np.std(dph) # using constant err value errp = mapdict_data['Error'][ii,jj] # for error reading ## fit function lines1 = [0, 0, 0, 0] xvalues = np.arange(imdates_ordinal[0], imdates_ordinal[-1], 10) xdates = [dt.fromordinal(pp) for pp in xvalues] # Mask to exclude nan elements mask = ~np.isnan(dph) # remove nan elements from both arrays imo = np.asarray(imdates_ordinal) imo = imo[mask] imt = np.asarray(imdates_dt) imt = imt[mask] dph = dph[mask] err = err[mask] for model, vis in enumerate(visibilities): if len(dph) > 1: yvalues = calc_model(dph, imo, xvalues, model, vel1p, intercept1p) if model == 0: lines1[model], = axts.plot(xdates, yvalues, 'r-', label=label2, visible=vis, alpha=0.6, zorder=3) axts.legend() else: lines1[model], = axts.plot(xdates, yvalues, 'r-', visible=vis, alpha=0.6, zorder=3) axts.scatter(imt, dph, label=label1, c='b', alpha=0.6, zorder=5) # axts.errorbar(imt, dph, yerr=err, label=label1, fmt='.', color='black', ecolor='blue', elinewidth=0.5, capsize=2) axts.set_title('Velocity = {:.1f} +/- {:.1f} [mm/yr] @({}, {})'.format(vel1p, errp, ii, jj), fontsize=10) # axts.set_ylim(-100,100) ### Y axis if ylen: vlim = [np.nanmedian(dph) - ylen / 2, np.nanmedian(dph) + ylen / 2] axts.set_ylim(vlim) ### Legend axts.legend() pts.canvas.draw()
def plot_size_decay_in_recovery(tracked_cells_df, sav_dir, start_end_NEW_cell=[3, 8], min_survive_frames=3, use_scaled=1, y_lim=10000, last_plot_week=0, ax_title_size=16, x_label='Weeks after recovery', intial_week=1, figsize=(6, 5)): start_frame = start_end_NEW_cell[0] end_frame = start_end_NEW_cell[1] plt.figure(figsize=figsize); list_arrs = []; list_arrs_unscaled = []; list_z = []; list_indices = []; for frame in np.unique(tracked_cells_df.FRAME): if frame >= start_frame and frame <= end_frame: all_sizes_cur_frame, all_sizes_scaled, all_z, all_indices = get_sizes_and_z_cur_frame(tracked_cells_df, frame, use_scaled=use_scaled) for idx in range(len(all_sizes_cur_frame)): sizes_cur_cell = all_sizes_cur_frame[idx] sizes_scaled = all_sizes_scaled[idx] z_cur_cell = all_z[idx] cur_indices = all_indices[idx] if len(sizes_cur_cell) >= min_survive_frames: ### MUST BE ALIVE FOR AT LEAST 4 frames from random import uniform t = uniform(0.,2.) col = (t/2.0, t/2.0, t/2.0) # different gray colors plt.plot(np.arange(intial_week, len(sizes_cur_cell[:last_plot_week]) + intial_week), sizes_cur_cell[:last_plot_week], linewidth=0.5, color=col, alpha=0.2) plt.ylim(intial_week - 1, y_lim) plt.tight_layout() list_arrs.append(list(sizes_scaled[:last_plot_week])) list_arrs_unscaled.append(list(sizes_cur_cell[:last_plot_week])) list_z.append(list(z_cur_cell[:last_plot_week])) list_indices.append(list(cur_indices[:last_plot_week])) ### sort z dist as well row_lengths = [] for row in list_arrs_unscaled: row_lengths.append(len(row)) max_length = max(row_lengths) for row in list_arrs_unscaled: while len(row) < max_length: row.append(np.nan) balanced_array = np.array(list_arrs_unscaled) ### find mean and std of sizes size_arr_unscaled = np.stack(balanced_array) ### find mean and std of sizes mean = np.nanmedian(size_arr_unscaled, axis=0) std = np.nanstd(size_arr_unscaled, axis=0) ### find sem all_n = [] for col_idx in range(len(balanced_array[0, :])): all_n.append(len(np.where(~np.isnan(balanced_array[:,col_idx]))[0])) sem = std/np.sqrt(all_n) ### plot plt.plot(np.arange(intial_week, len(mean) + intial_week), mean, color='r', linestyle='dotted', linewidth=2) plt.scatter(np.arange(intial_week, len(mean) + intial_week), mean, color='r', marker='o', s=30) plt.errorbar(np.arange(intial_week, len(mean) + intial_week), mean, yerr=sem, color='r', fmt='none', capsize=10, capthick=1) plt.xlabel(x_label, fontsize=ax_title_size) plt.ylabel('Soma volume ($\u03bcm^3$)', fontsize=ax_title_size) ax = plt.gca() rs = ax.spines["right"]; rs.set_visible(False) ts = ax.spines["top"]; ts.set_visible(False) from matplotlib.ticker import MaxNLocator ### force integer tick sizes ax.xaxis.set_major_locator(MaxNLocator(integer=True)) plt.tight_layout() plt.savefig(sav_dir + x_label + '_cell sizes changing statistics.png') ### sort list of lists so can be converted to array for mean and std row_lengths = [] for row in list_arrs: row_lengths.append(len(row)) max_length = max(row_lengths) for row in list_arrs: while len(row) < max_length: row.append(np.nan) balanced_array = np.array(list_arrs) ### find mean and std of sizes size_arr = np.stack(balanced_array) mean = np.nanmedian(size_arr, axis=0) std = np.nanstd(size_arr, axis=0) ### find sem all_n = [] for col_idx in range(len(balanced_array[0, :])): all_n.append(len(np.where(~np.isnan(balanced_array[:,col_idx]))[0])) sem = std/np.sqrt(all_n) return mean, sem, size_arr, size_arr_unscaled, list_indices
def main(): """ This is the main function that does all the processing. It calls the required functions to process the raw image. Raw images are saved in .sav (IDL) format and read through this script. """ data_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\Radiance_Cal_VIS' telemetry_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\2018.06.27' image_dir = os.path.join(data_dir, '32_ms') save_dir = os.path.join(image_dir, 'processed_image') image_save_dir = os.path.join(image_dir, 'saved_plots/final') telemetry_dir = os.path.join(telemetry_dir, 'processed/h5') linearity = 0 # option to turn on/off linearity correction smear = 1 cross_talk = 1 all_image = [] var_all = [] signal_all = [] dark_current = 1 if not os.path.exists(save_dir): os.makedirs(save_dir) # data_path_all = sorted([each for each in os.listdir(image_dir) # if each.startswith('2017_07_30_00_24_59_38064') # and each.endswith('.sav')]) # data_path_all = sorted( [each for each in os.listdir(image_dir) if each.endswith('.sav')]) # #dark_data = data_path_all[1:len(data_path_all):2] print('Total data = ', len(data_path_all)) # resultFile = open (os.path.join(save_dir,'file_name_all.csv'),'w') # # for results in data_path_all[0:15]: # resultFile.write(results+"\n") # resultFile.close() if dark_current: dark_current = calculate_dark_current(data_dir) else: dark_current = 1 count = 0 #peak_loc_all = [] for data_path in data_path_all: data_file = os.path.join(image_dir, data_path) print(data_path) # cc telemetry_file_name = data_path.split('.')[0] #################### Telemetry Statistics############################### # Now lets parse the telemetry file to get the required information### #Note; The function can return the CCD temp and FPE temp. But these are #not needed on the image processing algorithm.. atleast for the time being telemetry_file = os.path.join(telemetry_dir, telemetry_file_name + '.h5') if os.path.exists(telemetry_file): coadds, int_time, fpe_temp, fpa_temp = parse_telemetry_file( telemetry_dir, telemetry_file_name) print('FPE Temp. = ', round(fpe_temp, 2), 'FPA Temp = ', round(fpa_temp, 2)) print('Integ. Time =', int_time) else: print('Telemetry file missing') coadds = 10 int_time = 6000 fpe_temp = 43 fpa_temp = -21.5 ##############Image Processing begins here############################## full_frame = read_idl_file(data_file) #check_image = create_image_active_region(full_frame) raw_image = create_final_image(np.array(full_frame)) # print('Max. Val. Raw = ', np.max(raw_image)) quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D'] ##########################OFFSET REMOVAL############################### # Input : Full_Frame TEMPO IMAGE # Otput : Bias Subtracted Active Region. The active region dimensions are #now 1028*1024. 2 pixel lines from SMEAR overclocks are now used for #to store storage summation information # For dark data, only offset removal is needed to compute the dark current # For light data, additional processing such as linearity, smear and #cross talk is needed bias_removed_quads = perform_bias_subtraction(full_frame) #print('Max. Val. Offset Removed = ', np.max(bias_removed_quads)) text1 = telemetry_file_name+'.img' +' (Raw Data)\n Int. time:' + str(round(int_time, 3))+ \ 'ms, Co-adds:' +str(int(coadds))+\ ', FPE temp:'+ str(round(fpe_temp, 1))+'C, ' + \ ' FPA temp: ' + str(round(fpa_temp, 1))+'C' # Now let us save the raw image raw_image_save = os.path.join(image_save_dir, 'raw_image') if not os.path.exists(raw_image_save): os.makedirs(raw_image_save) plot_save_dir = raw_image_save + '/' + data_path + '.png' #prnu_spectrometer[prnu_spectrometer < 0.9] = 0.9 #prnu_spectrometer[prnu_spectrometer > 1.2] = 1.2 #print(np.min(check_image)) #create_image(check_image, text1, plot_save_dir) #--------------------------------------------------------------------- #########################NON LINEARITY CORRECTION####################### # Input : Bias Subtracted Active regions # Output : Linearized Active Region Quads if linearity: linearized_a = apply_linearity_correction( bias_removed_quads[0, :, :], quads[0], coadds) linearized_b = apply_linearity_correction( bias_removed_quads[1, :, :], quads[1], coadds) linearized_c = apply_linearity_correction( bias_removed_quads[2, :, :], quads[2], coadds) linearized_d = apply_linearity_correction( bias_removed_quads[3, :, :], quads[3], coadds) linearized_quads = np.array( [linearized_a, linearized_b, linearized_c, linearized_d]) #print(np.mean(linearized_quads)) else: linearized_quads = bias_removed_quads #---------------------------------------------------------------------- ##########################SMEAR REMOVAL################################ # Input : linearized quads ( all quads together) # Output : SMEAR offset corrected Quad #### lets' create the masked array with outlier mask################ # The outlier mask is array of list of 4 quads. #print('Max. Val. Linearized = ', np.max(linearized_quads)) outlier_mask = read_outlier_mask() # Note : all the arrays after this step are masked arrays if smear: smear_removed_quads = perform_smear_removal( linearized_quads, int_time, outlier_mask) else: smear_removed_quads = linearized_quads #---------------------------------------------------------------------- ##########################CROSS-TALK REMOVAL########################### # Input : smear removed quads (all quads together) # Output : cross talk removed quads #print('Max. Val. Smeared = ', np.max(smear_removed_quads)) if cross_talk: cross_talk_removed_quads = remove_cross_talk( np.array(smear_removed_quads)) else: cross_talk_removed_quads = smear_removed_quads #---------------------------------------------------------------------- #print('Max. Val. Cross Talked = ', np.max(cross_talk_removed_quads)) processed_image = create_final_image( np.array(cross_talk_removed_quads)) processed_image = processed_image - dark_current #prnu_map = parse_prnu_file() prnu_map = read_prnu_files() prnu_spectrometer = create_final_image(np.array(prnu_map)) prnu_spectrometer[prnu_spectrometer > 1.03] = 1.02 prnu_spectrometer[prnu_spectrometer < 0.97] = 0.98 outlier_spectrometer = create_final_image(np.array(outlier_mask)) nx_quad, ny_quad = processed_image.shape outlier_spectrometer = np.reshape(outlier_spectrometer, (nx_quad * ny_quad, 1)) #outlier_detectors = np.array(np.where([outlier_spectrometer == 1])) #print('outliers =', outlier_detectors.shape[1]) outlier_spectrometer = np.reshape(outlier_spectrometer, (nx_quad, ny_quad)) #create_image(outlier_spectrometer,'outliers = '+str(outlier_detectors.shape[1]),'b') processed_image = processed_image / (coadds * prnu_spectrometer) processed_image[processed_image >= 0.90 * 16383] = np.NAN #processed_image = processed_image*180/int_time #print(np.nanmax(processed_image)) #processed_image[processed_image>=0.85*16383] = 16383 text1 = telemetry_file_name+'.img, ' + 'Int. time:' + str(round(int_time,2))+ \ 'ms\n Co-adds:' +str(int(coadds))+\ ', FPE temp:'+ str(round(fpe_temp, 1))+'C, ' + \ ' FPA temp: ' + str(round(fpa_temp, ))+'C' processed_image_save = os.path.join(image_save_dir, 'processed_image') if not os.path.exists(processed_image_save): os.makedirs(processed_image_save) plot_save_dir = processed_image_save + '/' + data_path + '.png' #processed_image = processed_image[:, 500:2000] #processed_image[processed_image>6000] = 6000 #processed_image = uniform_filter(processed_image, size=(15, 15), mode='mirror') #create_image(raw_image, text1, plot_save_dir) all_image.append(processed_image[:, 6:2041]) #print(np.array(all_image).shape) #cc count = count + 1 all_image = 16.5 * np.array(all_image) # Gain is 16.5 from RMM var_all = np.nanvar(all_image, axis=0) signal_all = np.nanmedian(all_image, axis=0) variance_file_name = image_save_dir + '/' + 'variance_all_electrons_32ms.csv' mean_file_name = image_save_dir + '/' + 'mean_all_electrons_32ms.csv' np.savetxt(variance_file_name, var_all, fmt='%1.3f', delimiter=",") np.savetxt(mean_file_name, signal_all, fmt='%1.3f', delimiter=",")
def draw_mask_map(image, seg_map, mask_deep, stars, r_core=None, r_out=None, vmin=None, vmax=None, pad=0, save=False, save_dir='./'): """ Visualize mask map """ from matplotlib import patches mu = np.nanmedian(image) std = mad_std(image) if vmin is None: vmin = mu - std if vmax is None: vmax = mu + 10 * std fig, (ax1, ax2, ax3) = plt.subplots(ncols=3, nrows=1, figsize=(20, 6)) im1 = ax1.imshow(image, cmap='gray', norm=LogNorm(), vmin=vmin, vmax=1e4) ax1.set_title("Image") n_label = seg_map.max() ax2.imshow(seg_map, vmin=1, vmax=n_label - 2, cmap=make_rand_cmap(n_label)) ax2.set_title("Deep Mask") image2 = image.copy() image2[mask_deep] = 0 im3 = ax3.imshow(image2, norm=LogNorm(), vmin=vmin, vmax=vmax, aspect='auto') ax3.set_title("'Sky'") colorbar(im3) if r_core is not None: if np.ndim(r_core) == 0: r_core = [r_core, r_core] star_pos_A = stars.star_pos_verybright + pad star_pos_B = stars.star_pos_medbright + pad aper = CircularAperture(star_pos_A, r=r_core[0]) aper.plot(color='lime', lw=2, label="", alpha=0.9, axes=ax3) aper = CircularAperture(star_pos_B, r=r_core[1]) aper.plot(color='c', lw=2, label="", alpha=0.7, axes=ax3) if r_out is not None: aper = CircularAperture(star_pos_A, r=r_out[0]) aper.plot(color='lime', lw=1.5, label="", alpha=0.9, axes=ax3) aper = CircularAperture(star_pos_B, r=r_out[1]) aper.plot(color='c', lw=1.5, label="", alpha=0.7, axes=ax3) patch_size = image.shape[0] - pad * 2 rec = patches.Rectangle((pad, pad), patch_size, patch_size, facecolor='none', edgecolor='w', linewidth=2, linestyle='--', alpha=0.8) ax3.add_patch(rec) plt.tight_layout() if save: plt.savefig(os.path.join(save_dir, "Mask_dual.png"), dpi=120) plt.show() plt.close() else: plt.show()
def replace_missing_values(feature_table, replacement_dict=None, replacement_method=MEAN_VALUE_REPLACEMENT_METHOD): """For each feature, replaces missing values with mean or median. N = number of examples M = number of features (input variables) For both input and output, `replacement_dict` is a dictionary with the following keys. Only one of the last two keys is present. replacement_dict['feature_name']: Name of feature. replacement_dict['original_mean']: Original mean (before replacement of missing values). replacement_dict['original_median']: Original median (before replacement of missing values). If `replacement_dict` is None, will assume that `feature_table` contains training data and use `replacement_method`. If `replacement_dict` is specified, will assume that `feature_table` contains non-training data, in which case missing values will be replaced with means (or medians) from training data. :param feature_table: pandas DataFrame with N rows and M columns. Column names are feature names. :param replacement_dict: Dictionary with keys listed above. :param replacement_method: String ("mean" or "median"). :return: feature_table_no_missing_values: Same as input, except that missing values have been replaced. :return: replacement_dict: Dictionary with keys listed above. :raises: ValueError: if any column of feature_table has < 2 real values (not NaN). """ num_real_values_by_feature = numpy.sum(numpy.invert( numpy.isnan(feature_table.as_matrix())), axis=0) if numpy.any(num_real_values_by_feature < 2): raise ValueError('Each column of feature_table must have >= 2 real ' 'values (not NaN).') if replacement_dict is None: error_checking.assert_is_string(replacement_method) if replacement_method not in VALID_REPLACEMENT_METHODS: error_string = ( '\n\n{0:s}\n\nValid replacement methods (listed above) do not ' 'include "{1:s}".').format(VALID_REPLACEMENT_METHODS, replacement_method) raise ValueError(error_string) if replacement_method == MEAN_VALUE_REPLACEMENT_METHOD: replacement_dict = { FEATURE_NAMES_KEY: list(feature_table), ORIGINAL_MEANS_KEY: numpy.nanmean(feature_table.as_matrix(), axis=0) } else: replacement_dict = { FEATURE_NAMES_KEY: list(feature_table), ORIGINAL_MEDIANS_KEY: numpy.nanmedian(feature_table.as_matrix(), axis=0) } else: replacement_dict = _reorder_standardization_or_replacement_dict( replacement_dict, list(feature_table)) feature_names = list(feature_table) num_features = len(feature_names) feature_table_no_missing_values = None for j in range(num_features): these_values = copy.deepcopy(feature_table[feature_names[j]].values) nan_indices = numpy.where(numpy.isnan(these_values))[0] if ORIGINAL_MEANS_KEY in replacement_dict: these_values[nan_indices] = replacement_dict[ORIGINAL_MEANS_KEY][j] else: these_values[nan_indices] = replacement_dict[ORIGINAL_MEDIANS_KEY][ j] if feature_table_no_missing_values is None: feature_table_no_missing_values = pandas.DataFrame.from_dict( {feature_names[j]: these_values}) else: feature_table_no_missing_values = ( feature_table_no_missing_values.assign( **{feature_names[j]: these_values})) return feature_table_no_missing_values, replacement_dict
def main(argv=None): #%% Check argv if argv == None: argv = sys.argv start = time.time() ver=1.2; date=20200228; author="Y. Morishita" print("\n{} ver{} {} {}".format(os.path.basename(argv[0]), ver, date, author), flush=True) print("{} {}".format(os.path.basename(argv[0]), ' '.join(argv[1:])), flush=True) #%% Set default ifgdir = [] tsadir = [] loop_thre = 1.5 cmap_noise = 'viridis' cmap_noise_r = 'viridis_r' #%% Read options try: try: opts, args = getopt.getopt(argv[1:], "hd:t:l:", ["help"]) except getopt.error as msg: raise Usage(msg) for o, a in opts: if o == '-h' or o == '--help': print(__doc__) return 0 elif o == '-d': ifgdir = a elif o == '-t': tsadir = a elif o == '-l': loop_thre = float(a) if not ifgdir: raise Usage('No data directory given, -d is not optional!') elif not os.path.isdir(ifgdir): raise Usage('No {} dir exists!'.format(ifgdir)) elif not os.path.exists(os.path.join(ifgdir, 'slc.mli.par')): raise Usage('No slc.mli.par file exists in {}!'.format(ifgdir)) except Usage as err: print("\nERROR:", file=sys.stderr, end='') print(" "+str(err.msg), file=sys.stderr) print("\nFor help, use -h or --help.\n", file=sys.stderr) return 2 print("\nloop_thre : {} rad".format(loop_thre), flush=True) #%% Directory setting ifgdir = os.path.abspath(ifgdir) if not tsadir: tsadir = os.path.join(os.path.dirname(ifgdir), 'TS_'+os.path.basename(ifgdir)) if not os.path.isdir(tsadir): print('\nNo {} exists!'.format(tsadir), file=sys.stderr) return 1 tsadir = os.path.abspath(tsadir) loopdir = os.path.join(tsadir, '12loop') if not os.path.exists(loopdir): os.mkdir(loopdir) loop_pngdir = os.path.join(loopdir ,'good_loop_png') bad_loop_pngdir = os.path.join(loopdir,'bad_loop_png') bad_loop_cand_pngdir = os.path.join(loopdir,'bad_loop_cand_png') if os.path.exists(loop_pngdir): shutil.move(loop_pngdir+'/', loop_pngdir+'_old') #move to old dir if os.path.exists(bad_loop_pngdir): for png in glob.glob(bad_loop_pngdir+'/*.png'): shutil.move(png, loop_pngdir+'_old') #move to old dir shutil.rmtree(bad_loop_pngdir) if os.path.exists(bad_loop_cand_pngdir): for png in glob.glob(bad_loop_cand_pngdir+'/*.png'): shutil.move(png, loop_pngdir+'_old') #move to old dir shutil.rmtree(bad_loop_cand_pngdir) os.mkdir(loop_pngdir) os.mkdir(bad_loop_pngdir) os.mkdir(bad_loop_cand_pngdir) ifg_rasdir = os.path.join(tsadir, '12ifg_ras') if os.path.isdir(ifg_rasdir): shutil.rmtree(ifg_rasdir) os.mkdir(ifg_rasdir) bad_ifgrasdir = os.path.join(tsadir, '12bad_ifg_ras') if os.path.isdir(bad_ifgrasdir): shutil.rmtree(bad_ifgrasdir) os.mkdir(bad_ifgrasdir) bad_ifg_candrasdir = os.path.join(tsadir, '12bad_ifg_cand_ras') if os.path.isdir(bad_ifg_candrasdir): shutil.rmtree(bad_ifg_candrasdir) os.mkdir(bad_ifg_candrasdir) no_loop_ifgrasdir = os.path.join(tsadir, '12no_loop_ifg_ras') if os.path.isdir(no_loop_ifgrasdir): shutil.rmtree(no_loop_ifgrasdir) os.mkdir(no_loop_ifgrasdir) infodir = os.path.join(tsadir, 'info') if not os.path.exists(infodir): os.mkdir(infodir) resultsdir = os.path.join(tsadir, 'results') if not os.path.exists(resultsdir): os.mkdir(resultsdir) netdir = os.path.join(tsadir, 'network') #%% Read date, network information and size ### Get dates ifgdates = tools_lib.get_ifgdates(ifgdir) ### Read bad_ifg11 bad_ifg11file = os.path.join(infodir, '11bad_ifg.txt') bad_ifg11 = io_lib.read_ifg_list(bad_ifg11file) ### Remove bad ifgs and images from list ifgdates = list(set(ifgdates)-set(bad_ifg11)) ifgdates.sort() imdates = tools_lib.ifgdates2imdates(ifgdates) n_ifg = len(ifgdates) n_im = len(imdates) ### Get size mlipar = os.path.join(ifgdir, 'slc.mli.par') width = int(io_lib.get_param_par(mlipar, 'range_samples')) length = int(io_lib.get_param_par(mlipar, 'azimuth_lines')) ### Get loop matrix Aloop = loop_lib.make_loop_matrix(ifgdates) n_loop = Aloop.shape[0] ### Extract no loop ifgs ns_loop4ifg = np.abs(Aloop).sum(axis=0) ixs_ifg_no_loop = np.where(ns_loop4ifg==0)[0] no_loop_ifg = [ifgdates[ix] for ix in ixs_ifg_no_loop] #%% 1st loop closure check. First without reference print('\n1st Loop closure check and make png for all possible {} loops...'.format(n_loop), flush=True) bad_ifg_cand = [] good_ifg = [] loop_ph_rms_ifg = [] for i in range(n_loop): if np.mod(i, 100) == 0: print(" {0:3}/{1:3}th loop...".format(i, n_loop), flush=True) unw12, unw23, unw13, ifgd12, ifgd23, ifgd13 = loop_lib.read_unw_loop_ph(Aloop[i, :], ifgdates, ifgdir, length, width) ## Calculate loop phase and check n bias (2pi*n) loop_ph = unw12+unw23-unw13 loop_2pin = int(np.round(np.nanmedian(loop_ph)/(2*np.pi)))*2*np.pi loop_ph = loop_ph-loop_2pin #unbias 2pi x n loop_ph_rms_ifg.append(np.sqrt(np.nanmean(loop_ph**2))) ### List as good or bad candidate if loop_ph_rms_ifg[i] >= loop_thre: #Bad loop including bad ifg. bad_ifg_cand.extend([ifgd12, ifgd23, ifgd13]) else: good_ifg.extend([ifgd12, ifgd23, ifgd13]) ### Output png. If exist in old, move to save time oldpng = os.path.join(loop_pngdir+'_old/', ifgd12[:8]+'_'+ifgd23[:8]+'_'+ifgd13[-8:]+'_loop.png') if os.path.exists(oldpng): ### Just move from old png shutil.move(oldpng, loop_pngdir) else: ### Make png. Take time a little. loop_lib.make_loop_png(ifgd12, ifgd23, ifgd13, unw12, unw23, unw13, loop_ph, loop_pngdir) if os.path.exists(loop_pngdir+'_old/'): shutil.rmtree(loop_pngdir+'_old/') #%% Identify bad ifgs and output text bad_ifg = loop_lib.identify_bad_ifg(bad_ifg_cand, good_ifg) bad_ifgfile = os.path.join(loopdir, 'bad_ifg_loop.txt') with open(bad_ifgfile, 'w') as f: for i in bad_ifg: print('{}'.format(i), file=f) ### Compute n_unw without bad_ifg11 and bad_ifg n_unw = np.zeros((length, width), dtype=np.int16) for ifgd in ifgdates: if ifgd in bad_ifg: continue unwfile = os.path.join(ifgdir, ifgd, ifgd+'.unw') unw = io_lib.read_img(unwfile, length, width) unw[unw == 0] = np.nan # Fill 0 with nan n_unw += ~np.isnan(unw) # Summing number of unnan unw #%% 2nd loop closure check without bad ifgs to define stable ref area print('\n2nd Loop closure check without bad ifgs to define ref area...', flush=True) ns_loop_ph = np.zeros((length, width), dtype=np.float32) ns_bad_loop = np.zeros((length, width), dtype=np.int16) loop_ph_rms_points = np.zeros((length, width), dtype=np.float32) for i in range(n_loop): if np.mod(i, 100) == 0: print(" {0:3}/{1:3}th loop...".format(i, n_loop), flush=True) ### Read unw unw12, unw23, unw13, ifgd12, ifgd23, ifgd13 = loop_lib.read_unw_loop_ph(Aloop[i, :], ifgdates, ifgdir, length, width) ### Skip if bad ifg is included if ifgd12 in bad_ifg or ifgd23 in bad_ifg or ifgd13 in bad_ifg: continue ## Calculate loop phase and rms at points loop_ph = unw12+unw23-unw13 loop_2pin = int(np.round(np.nanmedian(loop_ph)/(2*np.pi)))*2*np.pi loop_ph = loop_ph-loop_2pin #unbias ns_loop_ph = ns_loop_ph + ~np.isnan(loop_ph) loop_ph_sq = loop_ph**2 loop_ph_sq[np.isnan(loop_ph_sq)] = 0 loop_ph_rms_points = loop_ph_rms_points + loop_ph_sq ns_bad_loop = ns_bad_loop+(loop_ph_sq>np.pi**2) #suspected unw error # ns_bad_loop = ns_bad_loop+(np.abs(loop_ph)>loop_thre) ## multiple nan seem to generate RuntimeWarning ns_loop_ph[ns_loop_ph==0] = np.nan # To avoid 0 division loop_ph_rms_points = np.sqrt(loop_ph_rms_points/ns_loop_ph) ### Find stable ref area which have all n_unw and minimum ns_bad_loop and loop_ph_rms_points mask1 = (n_unw==np.nanmax(n_unw)) min_ns_bad_loop = np.nanmin(ns_bad_loop) while True: mask2 = (ns_bad_loop==min_ns_bad_loop) if np.all(~(mask1*mask2)): ## All masked min_ns_bad_loop = min_ns_bad_loop+1 ## Make mask2 again else: break loop_ph_rms_points_masked = loop_ph_rms_points*mask1*mask2 loop_ph_rms_points_masked[loop_ph_rms_points_masked==0] = np.nan refyx = np.where(loop_ph_rms_points_masked==np.nanmin(loop_ph_rms_points_masked)) refy1 = refyx[0][0] # start from 0, not 1 refy2 = refyx[0][0]+1 # shift +1 for python custom. start from 1 end with width refx1 = refyx[1][0] refx2 = refyx[1][0]+1 ### Save 12ref.txt reffile = os.path.join(infodir, '12ref.txt') with open(reffile, 'w') as f: print('{0}:{1}/{2}:{3}'.format(refx1, refx2, refy1, refy2), file=f) ### Save loop_ph_rms_masked and png loop_ph_rms_maskedfile = os.path.join(loopdir, 'loop_ph_rms_masked') loop_ph_rms_points_masked.tofile(loop_ph_rms_maskedfile) cmax = np.nanpercentile(loop_ph_rms_points_masked, 95) pngfile = loop_ph_rms_maskedfile+'.png' title = 'RMS of loop phase (rad)' plot_lib.make_im_png(loop_ph_rms_points_masked, pngfile, cmap_noise_r, title, None, cmax) ### Check ref exist in unw. If not, list as noref_ifg noref_ifg = [] for ifgd in ifgdates: if ifgd in bad_ifg: continue unwfile = os.path.join(ifgdir, ifgd, ifgd+'.unw') unw_ref = io_lib.read_img(unwfile, length, width)[refy1:refy2, refx1:refx2] unw_ref[unw_ref == 0] = np.nan # Fill 0 with nan if np.all(np.isnan(unw_ref)): noref_ifg.append(ifgd) bad_ifgfile = os.path.join(loopdir, 'bad_ifg_noref.txt') with open(bad_ifgfile, 'w') as f: for i in noref_ifg: print('{}'.format(i), file=f) #%% 3rd loop closure check without bad ifgs wrt ref point print('\n3rd loop closure check taking into account ref phase...', flush=True) bad_ifg_cand2 = [] good_ifg2 = [] loop_ph_rms_ifg2 = [] for i in range(n_loop): if np.mod(i, 100) == 0: print(" {0:3}/{1:3}th loop...".format(i, n_loop), flush=True) ### Read unw unw12, unw23, unw13, ifgd12, ifgd23, ifgd13 = loop_lib.read_unw_loop_ph(Aloop[i, :], ifgdates, ifgdir, length, width) ### Skip if bad ifg is included if ifgd12 in bad_ifg or ifgd23 in bad_ifg or ifgd13 in bad_ifg: loop_ph_rms_ifg2.append('--') continue ### Skip if noref ifg is included if ifgd12 in noref_ifg or ifgd23 in noref_ifg or ifgd13 in noref_ifg: loop_ph_rms_ifg2.append('--') continue ## Skip if no data in ref area in any unw. It is bad data. ref_unw12 = np.nanmean(unw12[refy1:refy2, refx1:refx2]) ref_unw23 = np.nanmean(unw23[refy1:refy2, refx1:refx2]) ref_unw13 = np.nanmean(unw13[refy1:refy2, refx1:refx2]) ## Calculate loop phase taking into account ref phase loop_ph = unw12+unw23-unw13-(ref_unw12+ref_unw23-ref_unw13) loop_ph_rms_ifg2.append(np.sqrt(np.nanmean((loop_ph)**2))) ### List as good or bad candidate if loop_ph_rms_ifg2[i] >= loop_thre: #Bad loop including bad ifg. bad_ifg_cand2.extend([ifgd12, ifgd23, ifgd13]) else: good_ifg2.extend([ifgd12, ifgd23, ifgd13]) #%% Identify additional bad ifgs and output text bad_ifg2 = loop_lib.identify_bad_ifg(bad_ifg_cand2, good_ifg2) bad_ifgfile = os.path.join(loopdir, 'bad_ifg_loopref.txt') with open(bad_ifgfile, 'w') as f: for i in bad_ifg2: print('{}'.format(i), file=f) #%% Output all bad ifg list and identify remaining candidate of bad ifgs ### Merge bad ifg, bad_ifg2, noref_ifg bad_ifg_all = list(set(bad_ifg+bad_ifg2+noref_ifg)) # Remove multiple bad_ifg_all.sort() ifgdates_good = list(set(ifgdates)-set(bad_ifg_all)) ifgdates_good.sort() bad_ifgfile = os.path.join(infodir, '12bad_ifg.txt') with open(bad_ifgfile, 'w') as f: for i in bad_ifg_all: print('{}'.format(i), file=f) ### Identify removed image and output file imdates_good = tools_lib.ifgdates2imdates(ifgdates_good) imdates_bad = list(set(imdates)-set(imdates_good)) imdates_bad.sort() bad_imfile = os.path.join(infodir, '12removed_image.txt') with open(bad_imfile, 'w') as f: for i in imdates_bad: print('{}'.format(i), file=f) ### Remaining candidate of bad ifg bad_ifg_cand_res = list(set(bad_ifg_cand2)-set(bad_ifg_all)) bad_ifg_cand_res.sort() bad_ifg_candfile = os.path.join(infodir, '12bad_ifg_cand.txt') with open(bad_ifg_candfile, 'w') as f: for i in bad_ifg_cand_res: print('{}'.format(i), file=f) #%% 4th loop to be used to calc n_loop_err and n_ifg_noloop print('\n4th loop to compute statistics...', flush=True) ns_loop_err = np.zeros((length, width), dtype=np.int16) for i in range(n_loop): if np.mod(i, 100) == 0: print(" {0:3}/{1:3}th loop...".format(i, n_loop), flush=True) ### Read unw unw12, unw23, unw13, ifgd12, ifgd23, ifgd13 = loop_lib.read_unw_loop_ph(Aloop[i, :], ifgdates, ifgdir, length, width) ### Skip if bad ifg is included if ifgd12 in bad_ifg_all or ifgd23 in bad_ifg_all or ifgd13 in bad_ifg_all: continue ## Compute ref ref_unw12 = np.nanmean(unw12[refy1:refy2, refx1:refx2]) ref_unw23 = np.nanmean(unw23[refy1:refy2, refx1:refx2]) ref_unw13 = np.nanmean(unw13[refy1:refy2, refx1:refx2]) ## Calculate loop phase taking into account ref phase loop_ph = unw12+unw23-unw13-(ref_unw12+ref_unw23-ref_unw13) ## Count number of loops with suspected unwrap error (>pi) loop_ph[np.isnan(loop_ph)] = 0 #to avoid warning ns_loop_err = ns_loop_err+(np.abs(loop_ph)>np.pi) #suspected unw error #%% Output loop info, move bad_loop_png loop_info_file = os.path.join(loopdir, 'loop_info.txt') f = open(loop_info_file, 'w') print('# loop_thre: {} rad. *: Removed w/o ref, **: Removed w/ ref'.format(loop_thre), file=f) print('# /: Candidates of bad loops but causative ifgs unidentified', file=f) print('# image1 image2 image3 RMS w/oref w/ref', file=f) for i in range(n_loop): ### Find index of ifg ix_ifg12, ix_ifg23 = np.where(Aloop[i, :] == 1)[0] ix_ifg13 = np.where(Aloop[i, :] == -1)[0][0] ifgd12 = ifgdates[ix_ifg12] ifgd23 = ifgdates[ix_ifg23] ifgd13 = ifgdates[ix_ifg13] imd1 = ifgd12[:8] imd2 = ifgd23[:8] imd3 = ifgd23[-8:] ## Move loop_png if bad ifg or bad ifg_cand is included looppngfile = os.path.join(loop_pngdir, '{0}_{1}_{2}_loop.png'.format(imd1, imd2, imd3)) badlooppngfile = os.path.join(bad_loop_pngdir, '{0}_{1}_{2}_loop.png'.format(imd1, imd2, imd3)) badloopcandpngfile = os.path.join(bad_loop_cand_pngdir, '{0}_{1}_{2}_loop.png'.format(imd1, imd2, imd3)) badloopflag1 = ' ' badloopflag2 = ' ' if ifgd12 in bad_ifg or ifgd23 in bad_ifg or ifgd13 in bad_ifg: badloopflag1 = '*' shutil.move(looppngfile, badlooppngfile) elif ifgd12 in bad_ifg2 or ifgd23 in bad_ifg2 or ifgd13 in bad_ifg2: badloopflag2 = '**' shutil.move(looppngfile, badlooppngfile) elif ifgd12 in bad_ifg_cand_res or ifgd23 in bad_ifg_cand_res or ifgd13 in bad_ifg_cand_res: badloopflag1 = '/' if os.path.exists(looppngfile): shutil.move(looppngfile, badloopcandpngfile) if type(loop_ph_rms_ifg2[i]) == np.float32: str_loop_ph_rms_ifg2 = "{:.2f}".format(loop_ph_rms_ifg2[i]) else: ## -- str_loop_ph_rms_ifg2 = loop_ph_rms_ifg2[i] print('{0} {1} {2} {3:5.2f} {4} {5:5s} {6}'.format(imd1, imd2, imd3, loop_ph_rms_ifg[i], badloopflag1, str_loop_ph_rms_ifg2, badloopflag2), file=f) f.close() #%% Saving coh_avg, n_unw, and n_loop_err only for good ifgs print('\nSaving coh_avg, n_unw, and n_loop_err...', flush=True) ### Calc coh avg and n_unw coh_avg = np.zeros((length, width), dtype=np.float32) n_coh = np.zeros((length, width), dtype=np.int16) n_unw = np.zeros((length, width), dtype=np.int16) for ifgd in ifgdates_good: ccfile = os.path.join(ifgdir, ifgd, ifgd+'.cc') if os.path.getsize(ccfile) == length*width: coh = io_lib.read_img(ccfile, length, width, np.uint8) coh = coh.astype(np.float32)/255 else: coh = io_lib.read_img(ccfile, length, width) coh[np.isnan(coh)] = 0 # Fill nan with 0 coh_avg += coh n_coh += (coh!=0) unwfile = os.path.join(ifgdir, ifgd, ifgd+'.unw') unw = io_lib.read_img(unwfile, length, width) unw[unw == 0] = np.nan # Fill 0 with nan n_unw += ~np.isnan(unw) # Summing number of unnan unw coh_avg[n_coh==0] = np.nan n_coh[n_coh==0] = 1 #to avoid zero division coh_avg = coh_avg/n_coh ### Save files n_unwfile = os.path.join(resultsdir, 'n_unw') np.float32(n_unw).tofile(n_unwfile) coh_avgfile = os.path.join(resultsdir, 'coh_avg') coh_avg.tofile(coh_avgfile) n_loop_errfile = os.path.join(resultsdir, 'n_loop_err') np.float32(ns_loop_err).tofile(n_loop_errfile) ### Save png title = 'Average coherence' plot_lib.make_im_png(coh_avg, coh_avgfile+'.png', cmap_noise, title) title = 'Number of used unw data' plot_lib.make_im_png(n_unw, n_unwfile+'.png', cmap_noise, title, n_im) title = 'Number of unclosed loops' plot_lib.make_im_png(ns_loop_err, n_loop_errfile+'.png', cmap_noise_r, title) #%% Link ras ### First, identify suffix of raster image (ras, bmp, or png?) unwfile = os.path.join(ifgdir, ifgdates[0], ifgdates[0]+'.unw') if os.path.exists(unwfile+'.ras'): suffix = '.ras' elif os.path.exists(unwfile+'.bmp'): suffix = '.bmp' elif os.path.exists(unwfile+'.png'): suffix = '.png' for ifgd in ifgdates: rasname = ifgd+'.unw'+suffix rasorg = os.path.join(ifgdir, ifgd, rasname) ### Bad ifgs if ifgd in bad_ifg_all: os.symlink(os.path.relpath(rasorg, bad_ifgrasdir), os.path.join(bad_ifgrasdir, rasname)) ### Remaining bad ifg candidates elif ifgd in bad_ifg_cand_res: os.symlink(os.path.relpath(rasorg, bad_ifg_candrasdir), os.path.join(bad_ifg_candrasdir, rasname)) ### Good ifgs else: os.symlink(os.path.relpath(rasorg, ifg_rasdir), os.path.join(ifg_rasdir, rasname)) if ifgd in no_loop_ifg: os.symlink(os.path.relpath(rasorg, no_loop_ifgrasdir), os.path.join(no_loop_ifgrasdir, rasname)) #%% Plot network ## Read bperp data or dummy bperp_file = os.path.join(ifgdir, 'baselines') if os.path.exists(bperp_file): bperp = io_lib.read_bperp_file(bperp_file, imdates) else: #dummy bperp = np.random.random(n_im).tolist() pngfile = os.path.join(netdir, 'network12_all.png') plot_lib.plot_network(ifgdates, bperp, [], pngfile) pngfile = os.path.join(netdir, 'network12.png') plot_lib.plot_network(ifgdates, bperp, bad_ifg_all, pngfile) pngfile = os.path.join(netdir, 'network12_nobad.png') plot_lib.plot_network(ifgdates, bperp, bad_ifg_all, pngfile, plot_bad=False) ### Network info ## Identify gaps G = inv_lib.make_sb_matrix(ifgdates_good) ixs_inc_gap = np.where(G.sum(axis=0)==0)[0] ## Connected network ix1 = 0 connected_list = [] for ix2 in np.append(ixs_inc_gap, len(imdates_good)-1): #append for last image imd1 = imdates_good[ix1] imd2 = imdates_good[ix2] dyear = (dt.datetime.strptime(imd2, '%Y%m%d').toordinal() - dt.datetime.strptime(imd1, '%Y%m%d').toordinal())/365.25 n_im_connect = ix2-ix1+1 connected_list.append([imdates_good[ix1], imdates_good[ix2], dyear, n_im_connect]) ix1 = ix2+1 # Next connection #%% Caution about no_loop ifg, remaining large RMS loop and gap ### no_loop ifg if len(no_loop_ifg)!=0: no_loop_ifgfile = os.path.join(infodir, '12no_loop_ifg.txt') with open(no_loop_ifgfile, 'w') as f: print("\nThere are {} ifgs without loop, recommend to check manually in no_loop_ifg_ras12".format(len(no_loop_ifg)), flush=True) for ifgd in no_loop_ifg: print('{}'.format(ifgd), flush=True) print('{}'.format(ifgd), file=f) ### Remaining candidates of bad ifgs if len(bad_ifg_cand_res)!=0: print("\nThere are {} remaining candidates of bad ifgs but not identified.".format(len(bad_ifg_cand_res)), flush=True) print("Check 12bad_ifg_cand_ras and loop/bad_loop_cand_png.", flush=True) # for ifgd in bad_ifg_cand_res: # print('{}'.format(ifgd)) print('\n{0}/{1} ifgs are discarded from further processing.'.format(len(bad_ifg_all), n_ifg), flush=True) for ifgd in bad_ifg_all: print('{}'.format(ifgd), flush=True) ### Gap gap_infofile = os.path.join(infodir, '12network_gap_info.txt') with open(gap_infofile, 'w') as f: if ixs_inc_gap.size!=0: print("Gaps between:", file=f) print("\nGaps in network between:", flush=True) for ix in ixs_inc_gap: print("{} {}".format(imdates_good[ix], imdates_good[ix+1]), file=f) print("{} {}".format(imdates_good[ix], imdates_good[ix+1]), flush=True) print("\nConnected network (year, n_image):", file=f) print("\nConnected network (year, n_image):", flush=True) for list1 in connected_list: print("{0}-{1} ({2:.2f}, {3})".format(list1[0], list1[1], list1[2], list1[3]), file=f) print("{0}-{1} ({2:.2f}, {3})".format(list1[0], list1[1], list1[2], list1[3]), flush=True) print('\nIf you want to change the bad ifgs to be discarded, re-run with different thresholds before next step.', flush=True) #%% Finish elapsed_time = time.time()-start hour = int(elapsed_time/3600) minite = int(np.mod((elapsed_time/60),60)) sec = int(np.mod(elapsed_time,60)) print("\nElapsed time: {0:02}h {1:02}m {2:02}s".format(hour,minite,sec)) print('\n{} Successfully finished!!\n'.format(os.path.basename(argv[0]))) print('Output directory: {}\n'.format(os.path.relpath(tsadir)))
#transects = SDS_tools.transects_from_geojson(geojson_file) # option 3: create the transects by manually providing the coordinates of two points #transects = dict([]) #transects['Transect 1'] = np.array([[342836, 6269215], [343315, 6269071]]) #transects['Transect 2'] = np.array([[342482, 6268466], [342958, 6268310]]) #transects['Transect 3'] = np.array([[342185, 6267650], [342685, 6267641]]) # intersect the transects with the 2D shorelines to obtain time-series of cross-shore distance # (also saved a .csv file with the time-series, dates are in UTC time) settings['along_dist'] = 25 cross_distance = SDS_transects.compute_intersection(output, transects, settings) # plot the time-series from matplotlib import gridspec fig = plt.figure() gs = gridspec.GridSpec(len(cross_distance),1) gs.update(left=0.05, right=0.95, bottom=0.05, top=0.95, hspace=0.05) for i,key in enumerate(cross_distance.keys()): if np.all(np.isnan(cross_distance[key])): continue ax = fig.add_subplot(gs[i,0]) ax.grid(linestyle=':', color='0.5') ax.set_ylim([-50,50]) ax.plot(output['dates'], cross_distance[key]- np.nanmedian(cross_distance[key]), '-^', markersize=6) ax.set_ylabel('distance [m]', fontsize=12) ax.text(0.5,0.95,'Transect ' + key, bbox=dict(boxstyle="square", ec='k',fc='w'), ha='center', va='top', transform=ax.transAxes, fontsize=14) mng = plt.get_current_fig_manager() mng.window.showMaximized() fig.set_size_inches([15.76, 8.52])
from astropy.table import Table data_path = '/Users/krogager/Data/NOT/MALS/test_pynot/raw/' raw_frames = [ 'ALzh010097.fits', 'ALzh010098.fits', 'ALzh010099.fits', 'ALzh010100.fits', ] red_fname = '/Users/krogager/Data/NOT/MALS/test_pynot/imaging/GRB160801A/GRB160801A_r_SDSS.fits' seg_fname = '/Users/krogager/Data/NOT/MALS/test_pynot/imaging/GRB160801A/GRB160801A_r_SDSS_seg.fits' red = fits.getdata(red_fname) hdr = fits.getheader(red_fname) med_r = np.nanmedian(red) std_r = np.median(np.fabs(red - med_r)) vmin_r = med_r - 5 * std_r vmax_r = med_r + 10 * std_r wcs = WCS(hdr) cmap = plt.cm.afmhot_r plt.close('all') dpi = 100 for num, fname in enumerate(raw_frames): raw = fits.getdata(data_path + fname) med = np.nanmedian(raw) std = np.median(np.fabs(raw - med))