def calc_norm_summary_tables(accuracy_tbl, time_tbl):
"""
Calculate normalized performance/ranking summary, as numpy
matrices as usual for convenience, and matrices of additional
statistics (min, max, percentiles, etc.)
Here normalized means relative to the best which gets a 1, all
others get the ratio resulting from dividing by the performance of
the best.
"""
# Min across all minimizers, i.e. for each fit problem what is the lowest chi-squared and the lowest time
min_sum_err_sq = np.nanmin(accuracy_tbl, 1)
min_runtime = np.nanmin(time_tbl, 1)
# create normalised tables
norm_acc_rankings = accuracy_tbl / min_sum_err_sq[:, None]
norm_runtimes = time_tbl / min_runtime[:, None]
summary_cells_acc = np.array([np.nanmin(norm_acc_rankings, 0),
np.nanmax(norm_acc_rankings, 0),
nanmean(norm_acc_rankings, 0),
nanmedian(norm_acc_rankings, 0)
])
summary_cells_runtime = np.array([np.nanmin(norm_runtimes, 0),
np.nanmax(norm_runtimes, 0),
nanmean(norm_runtimes, 0),
nanmedian(norm_runtimes, 0)
])
return norm_acc_rankings, norm_runtimes, summary_cells_acc, summary_cells_runtime
def triangleMAPs(savefilename,basename):
with open(savefilename,'rb') as savefile:
bf= numpy.array(pickle.load(savefile))
samples= numpy.array(pickle.load(savefile))
bf_g15= numpy.array(pickle.load(savefile))
samples_g15= numpy.array(pickle.load(savefile))
bf_zero= numpy.array(pickle.load(savefile))
samples_zero= numpy.array(pickle.load(savefile))
labels= []
for jj in range(samples.shape[2]):
labels.append(r"$\mathrm{param}\ %i$" % jj)
maps= define_rcsample.MAPs()
for ii, map in enumerate(maps.map()):
if ii >= len(bf): break
tfeh= numpy.nanmedian(map['FE_H'])
tafe= numpy.nanmedian(map[define_rcsample._AFETAG])
for tbf,tsamples,ext in zip([bf,bf_g15,bf_zero],
[samples,samples_g15,samples_zero],
['fid','g15','zero']):
try:
triangle.corner(tsamples[ii,].T,quantiles=[0.16, 0.5, 0.84],
labels=labels,
show_titles=True,title_args={"fontsize": 12},
bins=21)
except ValueError: pass
else:
bovy_plot.bovy_text(r'$[\mathrm{{Fe/H}}] = {feh:.1f},$'\
.format(feh=tfeh)+'\n'
+r'$[\alpha/\mathrm{{Fe}}] = {afe:.2f}$'\
.format(afe=tafe),
top_left=True,size=16.)
bovy_plot.bovy_end_print(basename+"_%i_%s.png" % (ii,ext))
return None
def find_bounds(model):
"""
Return the median upper and lower bound of the metabolic model.
Bounds can vary from model to model. Cobrapy defaults to (-1000, 1000) but
this may not be the case for merged or autogenerated models. In these
cases, this function is used to iterate over all the bounds of all the
reactions and find the median bound values in the model, which are
then used as the 'most common' bounds.
Parameters
----------
model : cobra.Model
The metabolic model under investigation.
"""
lower_bounds = np.asarray([rxn.lower_bound for rxn in model.reactions],
dtype=float)
upper_bounds = np.asarray([rxn.upper_bound for rxn in model.reactions],
dtype=float)
lower_bound = np.nanmedian(lower_bounds[lower_bounds != 0.0])
upper_bound = np.nanmedian(upper_bounds[upper_bounds != 0.0])
if np.isnan(lower_bound):
LOGGER.warning("Could not identify a median lower bound.")
lower_bound = -1000.0
if np.isnan(upper_bound):
LOGGER.warning("Could not identify a median upper bound.")
upper_bound = 1000.0
return lower_bound, upper_bound
def collapse_cube(w1, w2):
""" Collapse a MUSE data cube.
Arguments
cube : MUSE data cube name containing both data and stat extensions.
iext : Initial extension to be used. Default is one for combined cubes.
"""
fits = "slice_w{0}_{1}.fits".format(w1, w2)
outfits = "collapsed_w{0}_{1}.fits".format(w1, w2)
data = pf.getdata(fits, 0)
error = pf.getdata(fits, 1)
h = pf.getheader(fits, 0)
h2 = pf.getheader(fits, 1)
h["NAXIS"] = 2
del h["NAXIS3"]
h2["NAXIS"] = 2
del h2["NAXIS3"]
print "Starting collapsing process..."
start = time.time()
w = wavelength_array(fits)
# newdata = np.trapz(data, dx=np.diff(w)[0], axis=0)
# newdata = np.nansum(data, axis=0) * np.diff(w)[0]
newdata = np.nanmedian(data, axis=0)
noise = 1.482602 / np.sqrt(6.) * np.nanmedian(np.abs(2.* data - \
np.roll(data, 2, axis=0) - np.roll(data, -2, axis=0)), \
axis=0)
end = time.time()
print "Collapsing lasted {0} minutes.".format((end - start)/60.)
hdu = pf.PrimaryHDU(newdata, h)
hdu2 = pf.ImageHDU(noise, h2)
hdulist = pf.HDUList([hdu, hdu2])
hdulist.writeto(outfits, clobber=True)
return
def make_NRCS_image( nobj, bandname, fn='', dir='.', max=np.nan, min=np.nan,
**kwargs):
if not fn:
if 'reduced' in bandname:
fn = bandname[:9]+'.png'
else:
fn = bandname+'.png'
resize(nobj)
try:
s0 = 10.0*np.log10(nobj[bandname])
except:
n_obj.undo()
raise
s0[np.where(np.isinf(s0))]=np.nan
#if nobj.fileName[-2:]=='nc':
# s0 = flipdim(nobj,s0)
caption='dB'
if np.isnan(min):
min = np.nanmedian(s0,axis=None)-2.0*np.nanstd(s0,axis=None)
if np.isnan(max):
max = np.nanmedian(s0,axis=None)+2.0*np.nanstd(s0,axis=None)
nansatFigure(s0, min, max, dir, fn)
nobj.undo()
return fn
def plot_divergences(Ns, Ks, ivars):
divs = np.array([divergence(ivar, Truth) for ivar in ivars])
small = (Ns * Ks) < 300
med = ((Ns * Ks) > 3000) * ((Ns * Ks) < 5000)
big = (Ns * Ks) > 60000
Ksteps = 2. ** np.arange(0, 9)
mediansmalldivs = np.array([_hoggmedian((divs[small])[np.isclose(Ks[small], Kstep)]) for Kstep in Ksteps])
medianmeddivs = np.array([_hoggmedian((divs[med])[np.isclose(Ks[med], Kstep)]) for Kstep in Ksteps])
medianbigdivs = np.array([_hoggmedian((divs[big])[np.isclose(Ks[big], Kstep)]) for Kstep in Ksteps])
plt.clf()
plt.axhline(np.median(divs[small]), color="k", alpha=0.25)
plt.axhline(np.median(divs[med] ), color="k", alpha=0.25)
plt.axhline(np.median(divs[big] ), color="k", alpha=0.25)
plt.plot(Ks[small], divs[small], "k_", ms= 6, alpha=0.5)
plt.plot(Ks[med], divs[med], "k_", ms=12, alpha=0.5)
plt.plot(Ks[big], divs[big], "k_", ms=18, alpha=0.5)
good = np.isfinite(mediansmalldivs)
plt.plot(Ksteps[good], mediansmalldivs[good], "k_", ms= 6, mew=4)
plt.plot(Ksteps, medianmeddivs, "k_", ms=12, mew=4)
plt.plot(Ksteps, medianbigdivs, "k_", ms=18, mew=4)
plt.loglog()
plt.xlim(np.min(Ks) / 1.5, np.max(Ks) * 1.5)
plt.ylim(np.nanmedian(divs[big]) / 30., np.nanmedian(divs[small]) * 30.)
plt.xlabel("number of photons per image $K$")
plt.ylabel("divergence from the Truth")
hogg_savefig("divergences.png")
return None
def _make_tuples(self, key):
# Get behavior filename
behavior_path = (experiment.Session() & key).fetch1('behavior_path')
local_path = lab.Paths().get_local_path(behavior_path)
filename = (experiment.Scan.BehaviorFile() & key).fetch1('filename')
full_filename = os.path.join(local_path, filename)
# Read file
data = h5.read_behavior_file(full_filename)
# Get counter timestamps and convert to seconds
ts = h5.ts2sec(data['ts'], is_packeted=True)
# Read temperature (if available) and invalidate points with unreliable timestamps
temp_raw = data.get('temperature', None)
if temp_raw is None:
raise PipelineException('Scan {animal_id}-{session}-{scan_idx} does not have '
'temperature data'.format(**key))
temp_raw[np.isnan(ts)] = float('nan')
# Read temperature and smooth it
temp_celsius = (temp_raw * 100 - 32) / 1.8 # F to C
sampling_rate = int(round(1 / np.nanmedian(np.diff(ts)))) # samples per second
smooth_temp = signal.low_pass_filter(temp_celsius, sampling_rate, cutoff_freq=1,
filter_size=2 * sampling_rate)
# Resample at 1 Hz
downsampled_ts = ts[::sampling_rate]
downsampled_temp = smooth_temp[::sampling_rate]
# Insert
self.insert1({**key, 'temp_time': downsampled_ts,
'temperatures': downsampled_temp,
'median_temperature': np.nanmedian(downsampled_temp)})
self.notify(key)
def flaremeter(data):
''' Obtain median of data across baselines, polarizations, and frequencies to create a
time series indicated whether a flare has occurred. Values returned will be close
to unity if no flare. Returns:
tlevel: Array of levels at each time, nominally near unity
bflag: Array of flags indicating nominal background (where True) or
elevated background (where False) indicating possible flare
'''
nbl,npol,nf,nt = data.shape
tlevel = np.zeros(nt,'float')
background = np.sqrt(np.abs(data[:,0,:,:])**2 + np.abs(data[:,1,:,:])**2)
init_bg = np.nanmedian(background,2) # Initially take background as median over entire time range
bflag = np.ones(nt,'bool') # flags indicating "good" background times (not in flare)
for i in range(nt):
good, = np.where(bflag[:i] == True) # List of indexes of good background times up to current time
ngood = len(good) # Truncate list of indexes to last 100 elements (or fewer)
if ngood > 100:
good = good[ngood-100:]
# Calculate median over good background times
bg = np.nanmedian(background[:,:,good],2)
else:
# If there haven't been 100 times with good backgrounds yet, just use the initial one.
# This is supposed to avoid startup transients.
bg = init_bg
# Generate levels for each baseline and frequency for this time
level = np.sqrt(abs(data[:,0,:,i])**2 + abs(data[:,1,:,i])**2)/bg
# Take median over baseline and frequency to give a single number for this time
tlevel[i] = np.nanmedian(level)
if tlevel[i] > 1.05:
# If the level of the current time is higher than 1.05, do not include this time in future backgrounds
bflag[i] = False
return tlevel, bflag
def subtract(args):
for im_name in args.input1:
if args.overwrite:
new_name = im_name
elif args.suffix:
new_name = utilities.add_suffix_prefix(im_name, suffix=args.suffix)
# Read image, separate data and header
im = fits.open(im_name)
data = im[0].data
hdr = im[0].header
# Extract the overscan region. Notice that what we call x,y are the second and first axes
y0, y1, x0, x1 = args.region
overscan = data.copy()[x0:x1, y0:y1]
# Average over the short axis
if overscan.shape[0] < overscan.shape[1]:
average = numpy.nanmedian(overscan, axis=0)
# Fit a polynomial and return the fitted values
fitted_overscan = fit_pol(average, args.deg)
data[:, y0:y1] -= fitted_overscan
else:
average = numpy.nanmedian(overscan, axis=1)
# Fit a polynomial and return the fitted values
fitted_overscan = fit_pol(average, args.deg)
data[x0:x1, :] = (data[x0:x1, :].T - fitted_overscan).T
# Write to the output file
hdr.add_comment("Overscan region subtracted. Region: [{0}:{1},{2}:{3}]".format(x0, x1, y0, y1))
fits.writeto(new_name, data, hdr, clobber=True)
return None
def cutoff(self, df, z_score=3.0):
"""
Cut off extreme values using Median Absolute Deviation
Parameters
----------
df : pd.DataFrame
Returns
-------
pd.DataFrame
"""
df = self._align_univariate(df)
df = self._mask_non_index_member(df)
axis = 1
x = df.values
median = np.nanmedian(x, axis=axis).reshape(-1, 1)
diff = x - median
diff_abs = np.abs(diff)
mad = np.nanmedian(diff_abs, axis=axis).reshape(-1, 1)
mask = diff_abs > z_score * mad
x[mask] = 0
x = x + z_score * mad * np.sign(diff * mask) + mask * median
return pd.DataFrame(index=df.index, columns=df.columns, data=x)
def test_multiFringes(self):
"""Test that multi-fringe results are handled correctly by the task.
"""
self.config.filters.append("_unknown_")
self.config.large = 16
task = FringeTask(name="multiFringeMock", config=self.config)
config = isrMock.IsrMockConfig()
config.fringeScale = [750.0, 240.0, 220.0]
config.fringeX0 = [100.0, 150.0, 200.0]
config.fringeY0 = [0.0, 200.0, 0.0]
dataRef = isrMock.FringeDataRefMock(config=config)
exp = dataRef.get("raw")
medianBefore = np.nanmedian(exp.getImage().getArray())
fringes = task.readFringes(dataRef, assembler=None)
solution, rms = task.run(exp, **fringes.getDict())
medianAfter = np.nanmedian(exp.getImage().getArray())
stdAfter = np.nanstd(exp.getImage().getArray())
self.assertLess(medianAfter, medianBefore)
self.assertFloatsAlmostEqual(medianAfter, 3002.233, atol=1e-4)
self.assertFloatsAlmostEqual(stdAfter, 3549.9375, atol=1e-4)
deviation = np.abs(solution - config.fringeScale)
self.assertTrue(np.all(deviation / rms < 1.0))
def sigmaclip(data, factor, replacement=None, median=False, maxiter = 100):
std = np.std(data)
iteration=0
if median: center = np.nanmedian(data)
else: center = np.nanmean(data)
if not replacement: replacement = np.nan
elif replacement == 'mean': replacement = center
indx = (data>(center+std*factor))+(data<(center-std*factor))
while np.sum(indx) > 0 and iteration < maxiter:
#print indx, np.sum(indx)
#pl.plot(data)
#pl.plot([0,len(data)],[center+std*factor,center+std*factor])
#pl.plot([0,len(data)],[center-std*factor,center-std*factor])
data[indx] = replacement
std = np.std(data)
if median: center = np.nanmedian(data)
else: center = np.nanmean(data)
if not replacement: replacement = np.nan
elif replacement == 'mean': replacement = center
indx = (data>(center+std*factor))+(data<(center-std*factor))
#print indx, np.sum(indx)
#pl.plot(data,'ko')
#pl.show()
iteration+=1
return data
def Scatter(y, win=13, remove_outliers=False):
'''
Return the scatter in ppm based on the median running standard deviation
for a window size of :py:obj:`win` = 13 cadences (for K2, this
is ~6.5 hours, as in VJ14).
:param ndarray y: The array whose CDPP is to be computed
:param int win: The window size in cadences. Default `13`
:param bool remove_outliers: Clip outliers at 5 sigma before computing \
the CDPP? Default `False`
'''
if remove_outliers:
# Remove 5-sigma outliers from data
# smoothed on a 1 day timescale
if len(y) >= 50:
ys = y - Smooth(y, 50)
else:
ys = y
M = np.nanmedian(ys)
MAD = 1.4826 * np.nanmedian(np.abs(ys - M))
out = []
for i, _ in enumerate(y):
if (ys[i] > M + 5 * MAD) or (ys[i] < M - 5 * MAD):
out.append(i)
out = np.array(out, dtype=int)
y = np.delete(y, out)
if len(y):
return 1.e6 * np.nanmedian([np.std(yi) / np.sqrt(win)
for yi in Chunks(y, win, all=True)])
else:
return np.nan
def savePredictors(indata):
''' compute velocity quantiles for each inter-saccade event
'''
import numpy as np
data,coderid,gazeLag=indata
try:
preds=[]
coords=[]
data.extractBasicEvents()
data.driftCorrection()
if coderid!=-1:
importFailed=data.importComplexEvents(coderid=coderid)
if importFailed: return [[data.vp,-1]]
g=data.getGaze();totdur=float(g.shape[0])
hz=100#hz of the coordinate output
padstart=20# insert nans at start to allow lag
if gazeLag>=0:
gg=data.getGaze(hz=hz)[:,np.newaxis,[7,8]]
tr=data.getTraj(hz=hz)
mnn=min(gg.shape[0],tr.shape[0])
tt=data.getGaze(hz=hz)[:mnn,0]
gg=gg[:mnn,:,:];tr=tr[:mnn,:,:]
vel=data.getVelocity()
for si in range(len(data.sev)):
if si+2<len(data.sev): e=data.sev[si+1][0]
else: e=-1
s=data.sev[si][1];d=e-s
if g[s,0]-gazeLag<0 and g[e,0]-gazeLag<=0: continue
preds.append([data.vp,data.block,data.trial,
data.sev[si][-1],si,s,d,s/totdur,d/totdur])
if gazeLag>=0:
sel1=np.logical_and(tt>(g[s,0]),tt<(g[e,0]))
sel2=np.logical_and(tt>(g[s,0]-gazeLag),tt<(g[e,0]-gazeLag))
#print '1',sel1.sum(),sel2.sum()
trt=tr[sel2,:,:];
#print '2',trt.shape
trt=trt[:min(sel2.sum(),sel1.sum()),:,:]
#print '3',trt.shape
temp=np.ones((sel1.sum(),14,2))*np.nan
#print '4',temp.shape
temp[-sel2.sum():,:,:]=trt
coords.append(np.concatenate([gg[sel1,:,:],temp],axis=1))
tps=[s,s+d/4.,s+d/2.,s+3*d/4.,e]
tps=np.int32(np.round(tps))
for ti in range(len(tps)-1):
preds[-1].append(np.nanmedian(vel[tps[ti]:tps[ti+1]]))
dist=np.nanmedian(data.dist[tps[ti]:tps[ti+1],:],0)
di=np.argsort(dist)[:4]#take four nearest agents
preds[-1].extend(dist[di])
dev=np.abs(data.dev[tps[ti]:tps[ti+1],:])
dev=np.nanmedian(dev[:,di],0)
preds[-1].extend(dev)
if len(preds):
if gazeLag>=0: return [preds,coords]
else: return [preds,[]]
else: return [[[data.vp,-1]],[]]
except:
print 'Error at vp %d b %d t %d'%(data.vp,data.block,data.trial)
raise
def values_to_sizes(values, size, size_range):
maxval, minval = np.nanmax(values), np.nanmin(values)
try:
medval = np.nanmedian(values)
except KeyError:
medval = np.nanmedian(values.values)
sizes = (values-medval)/(maxval-medval)*size_range + size
return sizes
def plot_lc_white(self, ax=None):
# ax.plot(self.time, self.flux_r, lw=1)
# ax.plot(self.time, self.trtime, lw=1)
# ax.plot(self.time, self.trposi+4*(np.percentile(self.flux_r, [99])[0]-1), lw=1)
ax.plot(self.time, self.flux_r-self.trposi+np.nanmedian(self.trposi)
-self.trtime+np.nanmedian(self.trtime), '.')
[ax.axvline(self.bls.tc+i*self._rbls['bls_period'], alpha=0.25, ls='--', lw=1) for i in range(35)]
setp(ax,xlim=self.time[[0,-1]], xlabel='Time', ylabel='Normalised flux')
def get_bb_ratio(bb_height, bb_width, quality, zp_r):
"""Returns the Bright Band ratio of each PR bin
With *SR*, we refer to precipitation radars based on space-born platforms
such as TRMM or GPM.
This function basically applies the Bright Band (BB) information as
provided by the corresponding SR datasets per beam, namely BB height and
width, as well as quality flags of the SR beams. A BB ratio of <= 0
indicates that a bin is located below the melting layer (ML), >=1
above the ML, and in between 0 and 1 inside the ML.
Parameters
----------
bb_height : :class:`numpy:numpy.ndarray`
Array of shape (nscans, nbeams) containing the SR beams' BB heights
in meters.
bb_width : :class:`numpy:numpy.ndarray`
Array of shape (nscans, nbeams) containing the SR beams' BB widths
in meters.
quality : :class:`numpy:numpy.ndarray`
Array of shape (nscans, nbeams) containing the SR beams' BB quality
index.
zp_r : :class:`numpy:numpy.ndarray`
Array of SR bin altitudes of shape (nscans, nbeams, nbins).
Returns
-------
ratio : :class:`numpy:numpy.ndarray`
Array of shape (nscans, nbeams, nbins) containing the BB ratio of
every SR bin.
- ratio <= 0: below ml
- 0 < ratio < 1: between ml
- 1 <= ratio: above ml
ibb : :class:`numpy:numpy.ndarray`
Boolean array containing the indices of SR bins connected to the
BB.
"""
# parameters for bb detection
ibb = (bb_height > 0) & (bb_width > 0) & (quality == 1)
# set non-bb-pixels to np.nan
bb_height = bb_height.copy()
bb_height[~ibb] = np.nan
bb_width = bb_width.copy()
bb_width[~ibb] = np.nan
# get median of bb-pixels
bb_height_m = np.nanmedian(bb_height)
bb_width_m = np.nanmedian(bb_width)
# approximation of melting layer top and bottom
zmlt = bb_height_m + bb_width_m / 2.
zmlb = bb_height_m - bb_width_m / 2.
# get ratio connected to brightband height
ratio = (zp_r - zmlb) / (zmlt - zmlb)
return ratio, ibb
def SysRem(time, flux, err, ncbv=5, niter=50, sv_win=999,
sv_order=3, **kwargs):
'''
Applies :py:obj:`SysRem` to a given set of light curves.
:param array_like time: The time array for all of the light curves
:param array_like flux: A 2D array of the fluxes for each of the light \
curves, shape `(nfluxes, ntime)`
:param array_like err: A 2D array of the flux errors for each of the \
light curves, shape `(nfluxes, ntime)`
:param int ncbv: The number of signals to recover. Default 5
:param int niter: The number of :py:obj:`SysRem` iterations to perform. \
Default 50
:param int sv_win: The Savitsky-Golay filter window size. Default 999
:param int sv_order: The Savitsky-Golay filter order. Default 3
'''
nflx, tlen = flux.shape
# Get normalized fluxes
med = np.nanmedian(flux, axis=1).reshape(-1, 1)
y = flux - med
# Compute the inverse of the variances
invvar = 1. / err ** 2
# The CBVs for this set of fluxes
cbvs = np.zeros((ncbv, tlen))
# Recover `ncbv` components
for n in range(ncbv):
# Initialize the weights and regressors
c = np.zeros(nflx)
a = np.ones(tlen)
f = y * invvar
# Perform `niter` iterations
for i in range(niter):
# Compute the `c` vector (the weights)
c = np.dot(f, a) / np.dot(invvar, a ** 2)
# Compute the `a` vector (the regressors)
a = np.dot(c, f) / np.dot(c ** 2, invvar)
# Remove this component from all light curves
y -= np.outer(c, a)
# Save this regressor after smoothing it a bit
if sv_win >= len(a):
sv_win = len(a) - 1
if sv_win % 2 == 0:
sv_win -= 1
cbvs[n] = savgol_filter(a - np.nanmedian(a), sv_win, sv_order)
return cbvs
def draw(self):
#return np.fmax(self.ads.draw(), self.cds.draw())
adens = self.ads.draw()
cdens = self.cds.draw()
dens = cdens.copy()
for ii in range(dens.shape[0]):
if np.nanmedian(adens[ii]) > np.nanmedian(cdens[ii]):
dens[ii] = adens[ii]
return dens
def snr(flux, axis=0):
""" Calculates the S/N ratio of a spectra.
Translated from the IDL routine der_snr.pro """
signal = np.nanmedian(flux, axis=axis)
noise = 1.482602 / np.sqrt(6.) * np.nanmedian(np.abs(2.*flux - \
np.roll(flux, 2, axis=axis) - np.roll(flux, -2, axis=axis)), \
axis=axis)
return signal, noise, signal / noise
def _getMagnitudes(g):
if any([not np.isnan(x) for x in g.Mag]):
mag = np.nanmedian(g.Mag)
else:
mag = np.NaN
if any([not np.isnan(x) for x in g.ProEnMag]):
PEmag = np.nanmedian(g.ProEnMag)
else:
PEmag = np.NaN
return mag, PEmag
def populate_temp_elev_dict(dic, grid_temp, grid_elev, single_value=True):
r"""Populate a grid with average values from the dictionary.
Takes values with temperature and elevation information
and assigns them to a grid position. For each grid averages
of the values within the grid are calculated.
Parameters
----------
dic : dict
Dictionary containing lists of the values and their
temperature and elevation.
grid_temp, grid_elev : int
Dimension of the grid.
single_value : bool
Used to determine, whether there is a single value, as
is the case with tsi data or an average of data, as is
the case for psf data. Default is `True`.
Returns
-------
dict
A dictionary of lists as defined by `generate_temp_elev_dict`.
"""
ext = _get_temp_elev_dict_ext(dic)
g = generate_temp_elev_dict()
for __temp in range(grid_temp):
for __elev in range(grid_elev):
bound = _get_temp_elev_bound(ext, grid_temp, grid_elev, __temp, __elev)
std, mean, med, var, val = [[],[],[],[],[]]
for k, v in enumerate(dic['temp']):
if dic['temp'][k] >= bound[0] and dic['temp'][k] <= bound[1]:
if dic['elev'][k] >= bound[2] and dic['temp'][k] <= bound[3]:
if single_value:
val.append(dic['val'][k])
else:
std.append(dic['std'][k])
mean.append(dic['mean'][k])
med.append(dic['med'][k])
var.append(dic['var'][k])
g['temp'].append((bound[0] + bound[1])/2)
g['elev'].append((bound[2] + bound[3])/2)
if single_value:
g['std'].append(np.nanstd(val))
g['mean'].append(np.nanmean(val))
g['med'].append(np.nanmedian(val))
g['var'].append(np.nanvar(val))
g['err'].append(np.nanstd(val)/math.sqrt(len(val)))
else:
g['std'].append(np.nanmedian(std))
g['mean'].append(np.nanmedian(mean))
g['med'].append(np.nanmedian(med))
g['var'].append(np.nanmedian(var))
g['err'].append(np.nanstd(mean)/math.sqrt(len(std)))
return g
def mad(arr):
"""
Median average deviation.
The parameter `arr` is any list-like object (list, tuple, numpy.ndarray)
"""
if not IsListlike(arr):
raise ValueError("The input to mad must be a list-like object!")
median = np.nanmedian(arr)
arr = np.array(arr)
return np.nanmedian(np.abs(arr - median))
def mad(arr):
"""
Median Absolute Deviation: a "Robust" version of standard deviation.
Indices variabililty of the sample.
https://en.wikipedia.org/wiki/Median_absolute_deviation
:param arr: numpy array value
:return: return mad
"""
# arr = np.ma.array(arr).compressed() # should be faster to not use masked arrays.
med = np.nanmedian(arr)
return 1.4826 * np.nanmedian(np.absolute(arr - med))
def nanmad(a, axis=None):
"""
Calculates the Median Absolute Deviation from an array.
"""
a = np.array(a, copy=False)
a_median = np.nanmedian(a, axis=axis)
# re-broadcast the output median array to subtract it
if axis is not None:
a_median = np.expand_dims(a_median, axis=axis)
# calculated the median average deviation
return np.nanmedian(np.abs(a - a_median), axis=axis)
def mad(arr):
"""
Median average deviation
:param arr: A list-like object
:return:
"""
if not IsListlike(arr):
raise ValueError("The input to mad must be a list-like object!")
median = np.nanmedian(arr)
arr = np.array(arr)
return np.nanmedian(np.abs(arr - median))
def mad_based_outlier(data, thresh=3.5):
data = numpy.array(data)
if len(data.shape) == 1:
data = data[:,None]
median = numpy.nanmedian(data)
diff = numpy.nansum((data - median)**2, dtype=float, axis=-1)
diff = numpy.sqrt(diff)
med_abs_deviation = numpy.nanmedian(diff)
modified_z_score = 0.6745 * diff / med_abs_deviation
return modified_z_score > thresh
def response_curve(x1, _ticker='SP500', f=None, md=True, sigd=False):
univ_ib_gd = cr_cret.retrieve(univ_ib_eqidx_ext + 'ExchOpen')[_ticker].values
univ_ib_vl = cr_vol_all_adj.retrieve(univ_ib_eqidx_ext + 'vol_pb_120')[_ticker].values
univ_ib_s1 = cr_sig_mr_sg.retrieve(univ_ib_eqidx_ext + x1)[_ticker].values
univ_ib_cl = cr_cret.retrieve(univ_ib_eqidx_ext + 'Close')[_ticker].values
z = list(np.where(univ_ib_gd.astype('int') == 1)[0])
univ_ib_s1 = univ_ib_s1[z]
univ_ib_cl = univ_ib_cl[z]
univ_ib_vl = univ_ib_vl[z]
univ_ib_cl = filt.ret(univ_ib_cl)/filt.lag(univ_ib_vl, 1)
univ_ib_s1 = filt.lag(univ_ib_s1, 1)/filt.lag(univ_ib_vl, 1)
univ_ib_cl, univ_ib_s1 = reduce_nonnan(univ_ib_cl, univ_ib_s1)
_bins = 20
_range = np.maximum(np.percentile(univ_ib_s1, 99), -np.percentile(univ_ib_s1, 1))
_delta = _range/_bins
if f is not None:
pyl.figure(f)
else:
pyl.figure(1)
for i in range(0, 16):
if i == 0:
uis1 = univ_ib_s1
uic1 = univ_ib_cl
else:
uis1 = filt.lag(univ_ib_s1, i)
uic1 = filt.sma(univ_ib_cl, i+1)
uis1, uic1 = reduce_nonnan(uis1, uic1)
uis1_b = np.linspace(-_range, _range, num=_bins+1)
uic1_b = np.zeros(_bins+1)*np.nan
for j in range(0, _bins+1):
# j = 1
if j==0:
tmp__ = np.where(uis1 <= uis1_b[j]+_delta)[0]
elif j == _bins+1:
tmp__ = np.where(uis1 > uis1_b[j]-_delta)[0]
else:
tmp__ = np.where((uis1 <= uis1_b[j]+_delta) & (uis1 > uis1_b[j]-_delta))[0]
if tmp__.shape[0] > 0:
if md:
if not sigd:
uic1_b[j] = np.nanmedian(uic1[tmp__]) #/np.nanstd(uic1[tmp__])
else:
uic1_b[j] = np.nanmedian(uic1[tmp__])/np.nanstd(uic1[tmp__])
else:
if not sigd:
uic1_b[j] = np.nanmean(uic1[tmp__]) #/np.nanstd(uic1[tmp__])
else:
uic1_b[j] = np.nanmean(uic1[tmp__])/np.nanstd(uic1[tmp__])
pyl.subplot(4, 4, i+1)
pyl.plot(uis1_b, uic1_b)
def summarize(fitted, rchi2, measurement, rchi2_limit=1.5):
"""
Create summaries of the input measurement
:param fitted:
:param rchi2:
:param measurement:
:param summary:
:param rchi2_limit:
:return:
"""
nlon = measurement.shape[1]
mean = np.zeros(shape=nlon)
std = np.zeros_like(mean)
median = np.zeros_like(mean)
mad = np.zeros_like(mean)
n_found = np.zeros_like(mean)
for i in range(0, nlon):
# Find where the successful fits were
successful_fit = fitted[:, i]
# Reduced chi-squared
rc2 = rchi2[:, i]
# Successful fit
f = np.logical_and(successful_fit, rc2 < rchi2_limit)
# Indices of the successful fits
trial_index = np.nonzero(f)
# Number of successful trials
n_found[i] = np.sum(f)
m = measurement[trial_index, i]
mean[i] = np.nansum(m) / (1.0 * n_found[i])
std[i] = np.std(m)
median[i] = np.nanmedian(m)
mad[i] = np.nanmedian(np.abs(m - median[i]))
mean_mean = np.nanmean(mean)
mean_std = np.nanmean(std)
median_median = np.nanmedian(median)
median_mad = np.nanmedian(mad)
return ("mean value (standard deviation)", mean, std, mean_mean, mean_std),\
("median value (median absolute deviation)", median, mad, median_median, median_mad),\
("n_found", n_found)
def test_empty(self):
mat = np.zeros((0, 3))
for axis in [0, None]:
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
assert_(np.isnan(np.nanmedian(mat, axis=axis)).all())
assert_(len(w) == 1)
assert_(issubclass(w[0].category, RuntimeWarning))
for axis in [1]:
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
assert_equal(np.nanmedian(mat, axis=axis), np.zeros([]))
assert_(len(w) == 0)
elif product == "LS8_OLI_LASRC":
validValues = [322, 386, 834, 898, 1346, 324, 388, 836, 900, 1348]
cloud_mask = isin(nbar["pixel_qa"].values, validValues)
for band in bands:
datos = np.where(
np.logical_and(nbar.data_vars[band] != nodata, cloud_mask),
nbar.data_vars[band], np.nan)
allNan = ~np.isnan(datos)
if normalized:
m = np.nanmean(datos.reshape((datos.shape[0], -1)), axis=1)
st = np.nanstd(datos.reshape((datos.shape[0], -1)), axis=1)
datos = np.true_divide(
(datos - m[:, np.newaxis, np.newaxis]),
st[:, np.newaxis, np.newaxis]) * np.nanmean(st) + np.nanmean(m)
medians1[band] = np.nanmedian(datos, 0)
medians1[band][np.sum(allNan, 0) < minValid] = np.nan
del datos
# In[7]:
from sklearn.externals import joblib
# In[21]:
#Preprocesar:
nmed = None
nan_mask = None
for band in bands:
b = medians1[band].ravel()
if nan_mask is None:
def plotsupp2(supp_value_frame, analysis_path):
'''
用于输出csv文件和图
:param supp_value_frame: pd.DataFrame 供应商方面数据
:param analysis_path: string 文件路径
:return: None 仅用于作图
'''
# 输出第一张表
grouped = supp_value_frame.groupby('supp_name')
summary = []
for supp_name, group in grouped:
caigousum = group["pur_bill_id"].drop_duplicates().count()
skusum = group["item_sku_id"].drop_duplicates().count()
vltmean = np.mean(group["vlt"])
vltstd = np.std(group["vlt"])
vltcv = vltstd / vltmean
manzu = np.sum(group["actual_pur_qtty"]) / np.sum(group["originalnum"])
manzu2 = np.nansum(group["actual_pur_qtty"]) / np.nansum(group["plan_pur_qtty"])
summary.append({'supp_name':supp_name, 'caigousum': caigousum,
'skusum': skusum, 'vltmean': vltmean, 'vltstd': vltstd,
'vltcv': vltcv, 'manzu':manzu, 'manzu2': manzu2})
table1 = pd.DataFrame.from_dict(summary)
calname = ['caigousum','skusum','vltmean','vltstd','vltcv','manzu']
for i, colname in enumerate(calname):
min_num = np.min(table1[colname])
max_num = np.max(table1[colname])
cal_val = np.array(map(lambda x: (x - min_num)/(max_num - min_num)*9 + 1,table1[colname].values))
if i == 0:
sum_val = cal_val
else:
sum_val += cal_val
sum_val /= 6
table2 = pd.concat([table1, pd.DataFrame(sum_val.T, columns=["rank"])], axis=1)
table2 = table2.loc[:,['supp_name','caigousum','skusum','vltmean','vltstd','vltcv','manzu', 'rank']]
table2 = table2.sort_values(['rank'], ascending=False)
table2.to_csv(analysis_path + os.sep + 'supp_info.csv')
# 画vlt图
binsn = np.append(np.arange(0, 0.6, 0.05), 1)
plothistper(table2['vltcv'], binsn, u'vltcv', u'个数', u'vltcv分布', analysis_path + '//report//vltcv', intshu=False)
# 画满足率
binsn2 = np.arange(0.3, 1.1, 0.1)
plothistper(table1['manzu'], binsn2, u'满足率', u'个数', u'实际满足率分布', analysis_path + '//report//manzu', intshu=False)
plothistper(table1[table1['manzu2'].notnull()]["manzu2"], binsn2, u'满足率', u'个数', u'实际相对回告满足率分布', analysis_path + '//report//manzu2', intshu=False)
print "实际满足率的基本统计信息，最小值:{0:.2f}%,中位数:{1:.2f}%,均值:{2:.2f}%,最大值:{3:.2f}%。".format(
np.nanmin(table1[table1['manzu']>0.01]['manzu'].values),np.nanmedian(table1['manzu'].values),
np.nanmean(table1['manzu'].values),np.nanmax(table1['manzu'].values))
print "实际相对回告满足率分布的基本统计信息，最小值:{0:.2f}%,中位数:{1:.2f}%,均值:{2:.2f}%,最大值:{3:.2f}%。".format(
np.nanmin(table1[table1['manzu2']>0.01]['manzu2'].values),np.nanmedian(table1['manzu2'].values),
np.nanmean(table1['manzu2'].values),np.nanmax(table1['manzu2'].values))
# 画 vltcv * 满足率 图
fig, ax = plt.subplots(figsize=(12, 8))
# ax.grid()
ax.set_xlabel(u"vlt波动性")
ax.set_ylabel(u"平均满足率")
ax.set_xlim(0.05, 0.5)
ax.set_ylim(0.2, 1.1)
plt.scatter(table1['vltcv'], table1['manzu'], color='#0070C0')
plt.plot([0.3, 0.3], [0.2, 1.1], '--', color='red')
plt.plot([0.05, 0.5], [0.8, 0.8], '--', color='red')
plt.annotate('(1)', xy=(0.16, 0.9), fontsize=20, color='red')
plt.annotate('(2)', xy=(0.39, 0.9), fontsize=20, color='red')
plt.annotate('(3)', xy=(0.16, 0.5), fontsize=20, color='red')
plt.annotate('(4)', xy=(0.39, 0.5), fontsize=20, color='red')
plt.title(u'vlt稳定性与订单满足率散点图')
plt.savefig(analysis_path + '//report//vltcv_manzu')
def nanmedian(values, axis=None, skipna=True, mask=None):
"""
Parameters
----------
values : ndarray
axis: int, optional
skipna : bool, default True
mask : ndarray[bool], optional
nan-mask if known
Returns
-------
result : float
Unless input is a float array, in which case use the same
precision as the input array.
Examples
--------
>>> import pandas.core.nanops as nanops
>>> s = pd.Series([1, np.nan, 2, 2])
>>> nanops.nanmedian(s)
2.0
"""
def get_median(x):
mask = notna(x)
if not skipna and not mask.all():
return np.nan
return np.nanmedian(x[mask])
values, mask, dtype, dtype_max, _ = _get_values(values, skipna, mask=mask)
if not is_float_dtype(values):
values = values.astype("f8")
if mask is not None:
values[mask] = np.nan
if axis is None:
values = values.ravel()
notempty = values.size
# an array from a frame
if values.ndim > 1:
# there's a non-empty array to apply over otherwise numpy raises
if notempty:
if not skipna:
return _wrap_results(
np.apply_along_axis(get_median, axis, values), dtype)
# fastpath for the skipna case
return _wrap_results(np.nanmedian(values, axis), dtype)
# must return the correct shape, but median is not defined for the
# empty set so return nans of shape "everything but the passed axis"
# since "axis" is where the reduction would occur if we had a nonempty
# array
shp = np.array(values.shape)
dims = np.arange(values.ndim)
ret = np.empty(shp[dims != axis])
ret.fill(np.nan)
return _wrap_results(ret, dtype)
# otherwise return a scalar value
return _wrap_results(get_median(values) if notempty else np.nan, dtype)
def multilook_data(data, lks_y=1, lks_x=1, method='mean'):
"""Modified from Praveen on StackOverflow:
link: https://stackoverflow.com/questions/34689519/how-to-coarser-the-2-d-array-data-resolution
Parameters: data : 2D / 3D np.array in real or complex
lks_y : int, number of multilook in y/azimuth direction
lks_x : int, number of multilook in x/range direction
method : str, multilook method, mean, median or nearest
Returns: coarse_data : 2D / 3D np.array after multilooking in last two dimension
"""
method_list = ['mean', 'median', 'nearest']
if method not in method_list:
msg = 'un-supported multilook method: {}. '.format(method)
msg += 'Available methods: {}'.format(method_list)
raise ValueError(msg)
# do nothing if no multilook is applied
lks_y = int(lks_y)
lks_x = int(lks_x)
if lks_y * lks_x == 1:
return data
dtype = data.dtype
shape = np.array(data.shape, dtype=float)
ysize = int(shape[0] / lks_y)
xsize = int(shape[1] / lks_x)
if len(shape) == 2:
if method in ['mean', 'median']:
# crop data to the exact multiple of the multilook number
new_shape = np.floor(shape /
(lks_y, lks_x)).astype(int) * (lks_y, lks_x)
crop_data = data[:new_shape[0], :new_shape[1]]
# reshape to more dimensions and collapse the extra dimensions with mean
temp = crop_data.reshape(
(new_shape[0] // lks_y, lks_y, new_shape[1] // lks_x, lks_x))
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=RuntimeWarning)
if method == 'mean':
coarse_data = np.nanmean(temp, axis=(1, 3))
elif method == 'median':
coarse_data = np.nanmedian(temp, axis=(1, 3))
elif method == 'nearest':
coarse_data = data[int(lks_y / 2)::lks_y, int(lks_x / 2)::lks_x]
# fix size discrepency from average method
if coarse_data.shape != (ysize, xsize):
coarse_data = coarse_data[:ysize, :xsize]
elif len(shape) == 3:
if method in ['mean', 'median']:
# crop data to the exact multiple of the multilook number
new_shape = np.floor(
shape / (1, lks_y, lks_x)).astype(int) * (1, lks_y, lks_x)
crop_data = data[:new_shape[0], :new_shape[1], :new_shape[2]]
# reshape to more dimensions and collapse the extra dimensions with mean
temp = crop_data.reshape((new_shape[0], new_shape[1] // lks_y,
lks_y, new_shape[2] // lks_x, lks_x))
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=RuntimeWarning)
if method == 'mean':
coarse_data = np.nanmean(temp, axis=(2, 4))
elif method == 'median':
coarse_data = np.nanmedian(temp, axis=(2, 4))
elif method == 'nearest':
coarse_data = data[:, int(lks_y / 2)::lks_y, int(lks_x / 2)::lks_x]
# fix size discrepency from average method
if coarse_data.shape[-2:] != (ysize, xsize):
coarse_data = coarse_data[:, :ysize, :xsize]
# ensure output data type
coarse_data = np.array(coarse_data, dtype=dtype)
return coarse_data
def get_median(x):
mask = notna(x)
if not skipna and not mask.all():
return np.nan
return np.nanmedian(x[mask])
# In[73]:
#check why "local" is an object data type as it is supposed to be binary and does not contain any "?" in the expanded view above
print(ad_df.loc[:, "local"].unique())
# In[74]:
# Coerce to numeric and impute medians for height continuous column
ad_df.loc[:,
"height continuous"] = pd.to_numeric(ad_df.loc[:,
"height continuous"],
errors='coerce')
HasNan = np.isnan(ad_df.loc[:, "height continuous"])
ad_df.loc[HasNan,
"height continuous"] = np.nanmedian(ad_df.loc[:,
"height continuous"])
# In[75]:
plt.hist(ad_df.loc[:, "height continuous"])
# In[76]:
## The high limit for acceptable values is the mean plus 2 standard deviations
LimitHi = ad_df.loc[:, "height continuous"].mean() + 2 * (
ad_df.loc[:, "height continuous"].std())
print(LimitHi)
# In[77]:
#Replace outliers
def draw_mask_map_strip(image,
seg_comb,
mask_comb,
stars,
ma_example=None,
r_core=None,
vmin=None,
vmax=None,
pad=0,
save=False,
save_dir='./'):
""" Visualize mask map w/ strips """
from matplotlib import patches
star_pos_A = stars.star_pos_verybright + pad
star_pos_B = stars.star_pos_medbright + pad
if ma_example is not None:
mask_strip, mask_cross = ma_example
mu = np.nanmedian(image)
std = mad_std(image)
if vmin is None:
vmin = mu - std
if vmax is None:
vmax = mu + 10 * std
fig, (ax1, ax2, ax3) = plt.subplots(ncols=3, nrows=1, figsize=(20, 6))
mask_strip[mask_cross.astype(bool)] = 0.5
ax1.imshow(mask_strip, cmap="gray_r")
ax1.plot(star_pos_A[0][0], star_pos_A[0][1], "r*", ms=18)
ax1.set_title("Strip/Cross")
n_label = seg_comb.max()
ax2.imshow(seg_comb,
vmin=1,
vmax=n_label - 3,
cmap=make_rand_cmap(n_label))
ax2.plot(star_pos_A[:, 0], star_pos_A[:, 1], "r*", ms=18)
ax2.set_title("Mask Comb.")
image3 = image.copy()
image3[mask_comb] = 0
im3 = ax3.imshow(image3,
norm=LogNorm(),
aspect='auto',
vmin=vmin,
vmax=vmax)
ax3.plot(star_pos_A[:, 0], star_pos_A[:, 1], "r*", ms=18)
ax3.set_title("'Sky'")
colorbar(im3)
if r_core is not None:
if np.ndim(r_core) == 0:
r_core = [r_core, r_core]
aper = CircularAperture(star_pos_A, r=r_core[0])
aper.plot(color='lime', lw=2, label="", alpha=0.9, axes=ax3)
aper = CircularAperture(star_pos_B, r=r_core[1])
aper.plot(color='c', lw=2, label="", alpha=0.7, axes=ax3)
size = image.shape[0] - pad * 2
rec = patches.Rectangle((pad, pad),
size,
size,
facecolor='none',
edgecolor='w',
linewidth=2,
linestyle='--',
alpha=0.8)
ax3.add_patch(rec)
plt.tight_layout()
if save:
plt.savefig(os.path.join(save_dir, "Mask_strip.png"), dpi=120)
plt.show()
plt.close()
else:
plt.show()
def map_to_gamma_template(in_file,
out_file=None,
no_neg=False,
smooth_arcmin=0.0,
rm_median=False,
flag_dist_arcmin=0.0,
normalize=True):
"""
PURPOSE: This function extracts a map from a fits fil and modifies it
applying smoothing, baseline removal, flag of negative values,
and masking. The new image is then writtin in a new file.
INPUT: - in_file (string): input file full name
- out_file (string): (output file full name)
- no_deg (bool): set negative values to zero (default is no)
- smooth_arcmin (float): Gaussian smoothing FWHM, in arcmin, to apply
- rm_median (bool): remove the median of the map (default is no)
- flag_dist_arcmin (float): set to zero pixels beyond this limit,
from the center
- normalize (bool): set this keyword to normalize the map to 1 (i.e.
setting the integral sum(map)*reso^2 to 1)
OUTPUT: - The new map is written in a new file
- image (2d numpy array): the new map
- header : the corresponding map header
"""
#---------- Data extraction
data = fits.open(in_file)[0]
image = data.data
header = data.header
wcs = WCS(header)
reso_x = abs(wcs.wcs.cdelt[0])
reso_y = abs(wcs.wcs.cdelt[1])
Npixx = image.shape[0]
Npixy = image.shape[1]
#---------- Data modification
wnan = np.isnan(image)
image[wnan] = 0.0
winf = np.isinf(image)
image[winf] = 0.0
if smooth_arcmin >= 0:
sigma_sm = smooth_arcmin / 60.0 / np.array(
[reso_x, reso_x]) / (2 * np.sqrt(2 * np.log(2)))
image = ndimage.gaussian_filter(image, sigma=sigma_sm)
if rm_median:
image = image - np.nanmedian(image)
if no_neg:
image[image < 0] = 0.0
if flag_dist_arcmin > 0:
ra_map, dec_map = get_radec_map(data.header)
ra_ctr = np.mean(ra_map)
dec_ctr = np.mean(dec_map)
distmap = greatcircle(ra_map, dec_map, ra_ctr, dec_ctr)
image[distmap > flag_dist_arcmin / 60.0] = 0.0
if normalize == True:
norm = np.sum(image) * reso_x * reso_y * (np.pi / 180.0)**2
image = image / norm
#---------- Write FITS
if out_file is not None:
hdu = fits.PrimaryHDU(header=header)
hdu.data = image
hdu.writeto(out_file, overwrite=True)
return image, header
def check_pointing_using_2mass_catalog(filename, out_dir='./'):
"""Check that stars from an external 2MASS catalog are present at the expected
locations within filename.
Parameters
----------
filename : str
Name of fits file containing the observation to check. This file must
contain a valid GWCS (i.e. it must have gone through the assign_wcs
pipeline step.)
out_dir : str
Name of directory into which source catalogs are written
Returns
-------
med_dist : float
Median offset between expected and measured source locations, in arcsec
dev_dist : float
Standard deviation of the offset between expected and measured source
locations, in arcsec
mean_unc : float
Mean uncertainty in the source locoations in the 2MASS catalog, in arcsec
d2d_arcsec : list
Offsets between expected and measured source locations for all matched
sources, in arcsec
"""
print("Working on: {}".format(os.path.basename(filename)))
model = ImageModel(filename)
pix_scale = model.meta.wcsinfo.cdelt1 * 3600.
# Read in catalog from 2MASS. We'll be using the positions in this
# catalog as "truth"
catalog_2mass = os.path.join(os.path.dirname(__file__),
'car19_2MASS_source_catalog.txt')
cat_2mass = ascii.read(catalog_2mass)
ra_unc = cat_2mass['err_maj'] * u.arcsec
dec_unc = cat_2mass['err_min'] * u.arcsec
total_unc = np.sqrt(ra_unc**2 + dec_unc**2)
# Calculate the pixel corresponding to the RA, Dec value of the stars
# in the 2MASS catalog
world2det = model.meta.wcs.get_transform('world', 'detector')
star_x, star_y = world2det(cat_2mass['ra'], cat_2mass['dec'])
cat_2mass['x'] = star_x
cat_2mass['y'] = star_y
# Remove stars that are not on the detector
ydim, xdim = model.data.shape
good = ((star_x > 0) & (star_x < xdim) & (star_y > 0) & (star_y < ydim))
print('{} 2MASS sources should be present on the detector.'.format(
np.sum(good)))
if np.sum(good) == 0:
print('Skipping.\n')
return None, None, None, None
cat_2mass = cat_2mass[good]
# Find sources in the data
sub_fwhm = get_fwhm(model.meta.instrument.filter)
plot_file = '{}_source_map.png'.format(os.path.basename(filename))
plot_file = os.path.join(out_dir, plot_file)
sources = find_sources(model.data,
threshold=50,
fwhm=sub_fwhm,
plot_name=plot_file)
# Calculate RA, Dec of the sources in the new catalog
det2world = model.meta.wcs.get_transform('detector', 'world')
found_ra, found_dec = det2world(sources['xcentroid'].data,
sources['ycentroid'].data)
sources['calc_RA'] = found_ra
sources['calc_Dec'] = found_dec
# Match catalogs
found_sources = SkyCoord(ra=found_ra * u.degree, dec=found_dec * u.degree)
from_2mass = SkyCoord(ra=cat_2mass['ra'] * u.degree,
dec=cat_2mass['dec'] * u.degree)
idx, d2d, d3d = from_2mass.match_to_catalog_3d(found_sources)
# Print table of sources
d2d_arcsec = d2d.to(u.arcsec).value
cat_2mass['image_x'] = sources[idx]['xcentroid']
cat_2mass['image_y'] = sources[idx]['ycentroid']
cat_2mass['image_ra'] = sources[idx]['calc_RA']
cat_2mass['image_dec'] = sources[idx]['calc_Dec']
cat_2mass['delta_sky'] = d2d_arcsec
cat_2mass['delta_pix'] = d2d_arcsec / pix_scale
for colname in ['x', 'y', 'image_x', 'image_y', 'delta_pix']:
cat_2mass[colname].info.format = '7.3f'
print(cat_2mass['x', 'y', 'image_x', 'image_y', 'delta_pix'])
# Save the table of sources
table_file = 'comp_found_sources_to_2MASS_{}.txt'.format(
os.path.basename(filename))
table_file = os.path.join(out_dir, table_file)
print(
'Table of source location comparison saved to: {}'.format(table_file))
ascii.write(cat_2mass, table_file, overwrite=True)
# Get info on median offsets
med_dist = np.nanmedian(d2d_arcsec)
dev_dist = np.nanstd(d2d_arcsec)
print((
"Median distance between sources in 2MASS catalog and those found in the "
"data: {0:1.2f} arcsec = {1:1.2f} pixels".format(
med_dist, med_dist / pix_scale)))
mean_unc = np.mean(total_unc)
mean_unc_pix = np.mean(total_unc.value) / pix_scale
print((
"Mean uncertainty in the source locations within the 2MASS catalog: {0:1.2f} = {1:1.2f} "
"pixels\n\n".format(mean_unc, mean_unc_pix)))
return med_dist, dev_dist, mean_unc, d2d_arcsec
observations = data['Classification'].loc[data['Obj ID'] == obj]
if any(observations == 'EB'):
print('At least one EB found!')
objTable = data.loc[data['Obj ID'] == obj]
classification = objTable['Classification'].tolist()
files = objTable['Filename'].copy()
i = 0
# Plot all observations on the same light curve
plt.figure(figsize=(20, 6))
for file in files:
fitsTable = fits.open(file, memmap=True)
curveTable = Table(fitsTable[1].data).to_pandas()
curveData = curveTable.loc[curveTable['QUALITY'] == 0].dropna(
subset=['TIME']).dropna(subset=['PDCSAP_FLUX']).copy()
fluxMed = np.nanmedian(curveData['PDCSAP_FLUX'])
curveData['REL_FLUX'] = curveData['PDCSAP_FLUX'].div(fluxMed)
curveData['REL_FLUX_ERR'] = curveData['PDCSAP_FLUX_ERR'].div(
fluxMed)
# Determine EB/non-EB and colorize accordingly
if classification[i] == 'EB':
color = 'tab:purple'
else:
color = 'tab:gray'
figName = obj + '.png'
plt.scatter(curveData['TIME'],
curveData['REL_FLUX'],
color=color,
s=.2)
def pixel_value(self):
"""The median pixel value of the ROI."""
masked_img = self.circle_mask()
return np.nanmedian(masked_img)
train["Dependents"] = train["Dependents"].replace("3+", 3)
train["Dependents"] = train["Dependents"].replace("0", 0)
new_dependents = np.where(train["Dependents"].isnull(), 0, train["Dependents"])
train["Dependents"] = new_dependents
train["Dependents"] = pd.to_numeric(train["Dependents"])
train["Dependents"].dtypes
train["Dependents"].isnull().sum()
train["Self_Employed"].value_counts()
new_Self_Employed = np.where(train["Self_Employed"].isnull(), "No",
train["Self_Employed"])
train["Self_Employed"] = new_Self_Employed
train["Self_Employed"].isnull().sum()
train["LoanAmount"] = np.where(train["LoanAmount"].isnull(),
np.nanmedian(train["LoanAmount"]),
train["LoanAmount"])
train["LoanAmount"].isnull().sum()
train["Loan_Amount_Term"].value_counts()
train["Loan_Amount_Term"] = np.where(train["Loan_Amount_Term"].isnull(), 360,
train["Loan_Amount_Term"])
train["Loan_Amount_Term"].isnull().sum()
train["Credit_History"].value_counts()
train["Credit_History"] = np.where(train["Credit_History"].isnull(), 1.0,
train["Credit_History"])
train["Credit_History"].isnull().sum()
check_missing = train.isnull().sum()
try:
bbig = np.concatenate(bbig, axis=0)
ffin = np.concatenate(ffin, axis=0)
aarr = np.concatenate(aarr, axis=0)
taarr = np.concatenate(taarr, axis=0)
nnz = np.concatenate(nnz, axis=0)
except ValueError:
print('return')
bla = np.nansum(aarr, 0) / np.nansum(nnz, 0)
bla1 = np.nansum(ffin, 0)
blab = np.nansum(bbig, 0) / np.nansum(ffin,0) * 100
tbla = np.nanmedian(taarr, 0)
out.append((bla,bla1,blab, tbla))
######### 2d plots
f = plt.figure(figsize=(12, 5), dpi=400)
ll = [30, 60, 90, 120, 180] # keys
lll = [15, 30, 60, 90, 120,120]
for ind, k in enumerate(ll):
def readLogFile(fnameOrFolder,
subtractDark=False,
skip_first=0,
asDataStorage=True,
last=None,
srcur_min=30):
""" read id9 style logfile;
last before data will be used as keys ...
only srcur>srcur_min will be kept
subtractDark is not needed for data collected with waxscollect
"""
if os.path.isdir(fnameOrFolder):
fname = findLogFile(fnameOrFolder)
else:
fname = fnameOrFolder
log.info("Reading id9 logfile: %s" % fname)
data = utils.files.readLogFile(fname,skip_first=skip_first,last=last,\
output = "array",converters=dict(delay=_delayToNum))
# work on darks if needed
if subtractDark:
## find darks
with open(fname, "r") as f:
lines = f.readlines()
lines = [line.strip() for line in lines]
# look only for comment lines
lines = [line for line in lines if line[0] == "#"]
for line in lines:
if line.find("pd1 dark/sec") >= 0: darks['pd1ic'] = _findDark(line)
if line.find("pd2 dark/sec") >= 0: darks['pd2ic'] = _findDark(line)
if line.find("pd3 dark/sec") >= 0: darks['pd3ic'] = _findDark(line)
## subtract darks
for diode in ['pd1ic', 'pd2ic', 'pd3ic', 'pd4ic']:
if diode in darks:
data[diode] = data[diode] - darks[diode] * data['timeic']
# srcur filter
if "currentmA" in data.dtype.names:
idx_cur = data['currentmA'] > srcur_min
if (idx_cur.sum() < idx_cur.shape[0] * 0.5):
log.warn("Minimum srcur filter has kept only %.1f%%" %
(idx_cur.sum() / idx_cur.shape[0] * 100))
log.warn("Minimum srcur: %.2f, median(srcur): %.2f" %
(srcur_min, np.nanmedian(data["currentmA"])))
data = data[idx_cur]
else:
log.warn("Could not find currentmA in logfile, skipping filtering")
info = DataStorage()
# usually folders are named sample/run
if os.path.isdir(fnameOrFolder):
folder = fnameOrFolder
else:
folder = os.path.dirname(fnameOrFolder)
dirs = folder.split(os.path.sep)
ylabel = ".".join(dirs[-2:])
info.name = ".".join(dirs[-2:])
try:
reprate = readReprate(fname)
info.reprate = reprate
ylabel += " %.2f Hz" % reprate
except:
print("Could not read rep rate info")
try:
time_info = timesToInfo(data['time'])
info.duration = time_info
ylabel += "\n" + time_info
except:
print("Could not read time duration info")
info.ylabel = ylabel
if asDataStorage:
data = DataStorage(
dict((name, data[name]) for name in data.dtype.names))
return data, info
def median_performance_reached_in_last_ntrials(mid, df, n_trials=500):
mouse_df = df[df['animal_id'] == mid]
max_performance_reached = np.nanmedian(mouse_df['cumulative_performance'][-n_trials:])
return max_performance_reached
# run orthologs loop
print("MODE: re-cluster orthogroups with ETE(SO)+MCL")
logging.info("Analyse orthogroups in %s" % (phy_fo))
orthogroup_loop()
print("DONE")
elif ani == "opti":
# run optimisation loop
print("MODE: inflation optimisation with %i random phylogenies" % nopt)
logging.info("Find optimal inflation")
inf_lis, mod_lis = optimisation_loop(nopt=nopt)
# plot optimisation histograms
print("# median inflation is %f" % np.nanmedian(inf_lis))
with PdfPages('%s.optimise_inflation.pdf' % out_fn) as pdf:
# inflation
plt.figure(figsize=(4,3))
plt.title("Hist inflation I\nmedian = %f" % np.nanmedian(inf_lis))
plt.hist(inf_lis[~np.isnan(inf_lis)])
pdf.savefig(bbox_inches='tight')
# modularity
plt.figure(figsize=(4,3))
plt.title("Hist modularity Q\nmedian = %f" % np.nanmedian(mod_lis))
plt.hist(mod_lis[~np.isnan(mod_lis)])
pdf.savefig(bbox_inches='tight')
# close pdf
plt.close()
print("DONE")
nicest=hdulist2[0].data[:,:]
meanxcousedata = (~np.isnan(mom012))&(nicest>0)&(mom012>3*mom0rms)
tex=hdulist3[0].data[:,:]
av = nicest/9.4e20/2.
print 'average NH',np.nanmean(nicest[meanxcousedata]),'average mom0',np.nanmean(mom012[meanxcousedata]),'mean Xco',np.nanmean(nicest[meanxcousedata])/np.nanmean(mom012[meanxcousedata])
hdulist1.close()
hdulist2.close()
hdulist3.close()
rawxco = nicest/mom012
hdulist4[0].data = nicest/mom012
hdulist4[0].data[~meanxcousedata] = np.nan
#hdulist4.writeto('XCO.fits', output_verify='exception', clobber=True, checksum=False)
print 'nanmin XCO',np.nanmin(hdulist4[0].data),'nanmax XCO',np.nanmax(hdulist4[0].data),'nanmedian XCO',np.nanmedian(hdulist4[0].data)
def plothist(data,pdfname):
xx = data[~np.isnan(data)]
#xx = xxx[xxx<1.e2]
x = xx[xx>0]
p=plt.figure(figsize=(7,6))
fig, ax = plt.subplots(1,1)
# the histogram of the data
n, bins, patches = plt.hist(x, 100, histtype='step', color='k')
#print n,bins
#plt.xscale('log')
#plt.yscale('log')
plt.xlabel(r'$X_{\rm CO}~({\rm cm}^{-2}~{\rm (K~km~s^{-1})}^{-1})$')
plt.ylabel('counts')
plt.xlim(0,4.e20)
data = pd.concat([Train_data['SalePrice'], Train_data['GrLivArea']], axis=1)
data.plot.scatter(x='GrLivArea', y='SalePrice', ylim=(0,800000));
#Seperating the character and numeric columns for data visualization
Train_data_char = Train_data[Character_columns_Train]
Train_data_num = Train_data[Numeric_columns_Train]
#correlation matrix
corrmat = Train_data.corr()
f, ax = plt.subplots(figsize=(15, 12))
sns.heatmap(corrmat, vmax=.8, square=True);
#Preprocessing the data by combining both Test and Train data
Train_data['isTrain'] = pd.Series(1, index = Train_data.index)
Test_data['isTrain'] = pd.Series(0, index = Test_data.index)
house = pd.concat([Train_data, Test_data])
house['MasVnrArea'] = house.MasVnrArea.replace(np.NaN, np.mean(house['MasVnrArea'])) #Replacing null values with mean values
house['LotFrontage'] = house.LotFrontage.replace(np.NaN, np.nanmedian(house['LotFrontage'])) #Replacing null values with median values
house['Street'] = house['Street'].astype('category')
#Label Encoder implement here*****
house[Character_columns_Train] = house[Character_columns_Train].astype('category')
house[Character_columns_Train] = house[Character_columns_Train].apply(lambda x: x.cat.codes)
pcp_ens = np.zeros(
(np.shape(pcp)[0], np.shape(pcp)[1], np.shape(pcp)[2],
ens_num)) # create ens array for one member
tmean_ens = np.zeros(
(np.shape(pcp)[0], np.shape(pcp)[1], np.shape(pcp)[2], ens_num))
trange_ens = np.zeros(
(np.shape(pcp)[0], np.shape(pcp)[1], np.shape(pcp)[2], ens_num))
pcp_ens[:, :, :, i] = pcp
tmean_ens[:, :, :, i] = tmean
trange_ens[:, :, :, i] = trange
#=================================================================================
# calculate time-series mean values of mean and std. (time,y,x)
pcp_ens_mean = np.nanmean(pcp_ens, axis=3)
pcp_ens_median = np.nanmedian(pcp_ens, axis=3)
pcp_ens_std = np.std(pcp_ens, axis=3)
pcp_ens_lb = np.percentile(pcp_ens, lb_perct, axis=3)
pcp_ens_ub = np.percentile(pcp_ens, ub_perct, axis=3)
del pcp_ens
tmean_ens_mean = np.nanmean(tmean_ens, axis=3)
tmean_ens_median = np.nanmedian(tmean_ens, axis=3)
tmean_ens_std = np.std(tmean_ens, axis=3)
tmean_ens_lb = np.percentile(tmean_ens, lb_perct, axis=3)
tmean_ens_ub = np.percentile(tmean_ens, ub_perct, axis=3)
del tmean_ens
trange_ens_mean = np.nanmean(trange_ens, axis=3)
trange_ens_median = np.nanmedian(trange_ens, axis=3)
trange_ens_std = np.std(trange_ens, axis=3)
def measure_azimuthal_profile(data, star, add_noise=False, model=False):
radii_data = make_radii(data, star)
# Mask region outside data's PSF subtraction zone and inside IWA.
r_mask = 9
data[radii_data >= 135] = np.nan
data[radii_data < r_mask] = np.nan
# # Scale data brightness by sb_scale.
# data *= sb_scale
# OFF! Model flux calibration.
# data *= conv_WtoJy*sb_scale*data # [mJy arcsec^-2]
# Add Gaussian noise to image if desired.
if add_noise:
data += np.random.normal(scale=0.1, size=data.shape)
# Convolve model with Gaussian simulating instrument PSF.
if model:
data = filters.gaussian_filter(data.copy(), 1.2) #1.31)
fontSize = 16
# Clean up the edges of the data, which often have artifacts.
# Do this by smoothing the data to expand the NaN's already present outside
# the edges, ignoring the inner mask, and creating a new mask from this.
mask_edges = 2.
if mask_edges:
mask = np.ma.masked_invalid(filters.gaussian_filter(data,
mask_edges)).mask
mask[star[0] - r_mask * 4:star[0] + r_mask * 4,
star[1] - r_mask * 4:star[1] + r_mask * 4] = False
data = np.ma.masked_array(data, mask=mask).filled(np.nan)
# Brightness cuts.
cut_colors = ['C4', 'C9', 'C2', 'C8', 'C4', 'C9', 'C2', 'C8']
bw = 3 # width of slice [pix]
fs = 6 # filter size [pix]
data_rot45 = rot_array(data, star,
-45 * np.pi / 180) # data rotated -45 deg
sl_pa0 = np.sum(data[:, star[1] - (bw // 2):star[1] + (bw // 2) + 1],
axis=1)[::-1] # data PA=0 slice
sl_pa45 = np.sum(data_rot45[:,
star[1] - (bw // 2):star[1] + (bw // 2) + 1],
axis=1)[::-1] # data PA=45 slice
sl_pa90 = np.sum(data[star[0] - (bw // 2):star[1] + (bw // 2) + 1],
axis=0) # data PA=90 slice
sl_pa135 = np.sum(data_rot45[star[0] - (bw // 2):star[1] + (bw // 2) + 1],
axis=0) # data PA=135 slice
sl_list = np.array([sl_pa0, sl_pa45, sl_pa90, sl_pa135])
pa_sls = [0, 45, 90, 135]
# Estimate 'noise' as median of all slices.
sl_noise = np.nanmedian(sl_list, axis=0)
sl_rms = np.sqrt(np.nanmedian(sl_list**2, axis=0)) # 'root median square'
sl_med = np.sqrt(np.nanmedian(sl_list**2))
# Find S/N ratio.
# sl_snr = sl_list/filters.median_filter(sl_rms, fs)
sl_snr = np.array(
[filters.median_filter(sl_list[ii], fs)
for ii in range(len(sl_list))]) / sl_med
# Plot the image with lines marking PA angles measured.
fig0 = plt.figure(3)
plt.clf()
plt.subplot(111)
plt.subplots_adjust(bottom=0.1, right=0.99, top=0.91, left=0.05)
im = plt.imshow(data, extent=[-140, 140, -140, 140])
plt.clim(np.percentile(np.nan_to_num(data), 1.5),
np.percentile(np.nan_to_num(data), 99.99))
# Plot lines tracing PA=0, 45, 90, 135.
ext = im.get_extent() # [-x, +x, -y, +y]
plt.plot([0, 0], [ext[2], ext[3]], 'r--', alpha=0.5, linewidth=1)
plt.plot([ext[3], ext[2]], [ext[0], ext[1]], 'r--', alpha=0.5, linewidth=1)
plt.plot([ext[2], ext[3]], [0, 0], 'r--', alpha=0.5, linewidth=1)
plt.plot([ext[1], ext[0]], [ext[3], ext[2]], 'r--', alpha=0.5, linewidth=1)
plt.text(5, ext[3] - 15, '-0', fontsize=12)
plt.text(ext[0] + 15, ext[3] - 15, '-45', fontsize=12)
plt.text(ext[0] + 5, 5, '-90', fontsize=12)
plt.text(ext[0] + 15, ext[2] + 5, '-135', fontsize=12)
plt.title(fn + "\nPA = 0 is north up", fontsize=10)
plt.draw()
# Plot surface brightness v. projected separation.
fig1 = plt.figure(5)
plt.clf()
plt.subplot(111)
plt.subplots_adjust(bottom=0.13, right=0.98, top=0.91, left=0.15)
plt.axvspan(xmin=-r_mask, xmax=r_mask, color='k', alpha=0.1)
plt.axvline(x=0, c='0.4', linestyle='--', linewidth=1.)
plt.axhline(y=0, c='0.4', linestyle='--', linewidth=1.)
label_sls = 4 * ['data'] #4*['mod'] + 4*['data']
for ii, sl in enumerate(sl_list):
plt.plot(range(0, data.shape[0]) - star[0],
sl,
label='%s %d' % (label_sls[ii], pa_sls[ii]),
c=cut_colors[ii],
marker='.',
linestyle='None',
alpha=0.5)
plt.plot(range(0, data.shape[0]) - star[0],
filters.median_filter(sl_noise, fs),
label='med',
c='k',
linestyle=':',
linewidth=2.)
plt.plot(range(0, data.shape[0]) - star[0],
sl_rms,
label='rms',
c='m',
linestyle='-',
linewidth=1.5)
plt.ylabel("Surface Brightness")
plt.xlabel("Projected Separation (px)")
plt.title('%s\nsl %d px, filt %d px' % (fn, bw, fs), fontsize=10)
plt.legend(ncol=2,
fontsize=fontSize - 4,
labelspacing=0.2,
columnspacing=1,
borderpad=0.4,
handletextpad=0.3,
handlelength=1,
framealpha=0.5)
plt.draw()
# Plot S/N-like things.
fig2 = plt.figure(6)
plt.clf()
plt.subplot(111)
plt.subplots_adjust(bottom=0.13, right=0.98, top=0.91, left=0.15)
plt.axvspan(xmin=-r_mask, xmax=r_mask, color='k', alpha=0.1)
plt.axvline(x=0, c='0.4', linestyle='--', linewidth=1.)
plt.axhline(y=0, c='0.4', linestyle='--', linewidth=1.)
# for ii, sl in enumerate(sl_list[4:]):
# plt.plot(range(0,data_Qr.shape[0])-star[0], sl_dQr_snr[ii], label='%d' % pa_sls[ii], c=cut_colors[ii], linestyle='-', linewidth=1.5)
for ii, sl in enumerate(sl_list):
plt.plot(range(0, data.shape[0]) - star[0],
sl_snr[ii],
label='%d' % pa_sls[ii],
c=cut_colors[ii],
linestyle='-',
linewidth=1.5)
plt.axhline(y=3,
c='0.2',
linestyle='--',
linewidth=1.5,
label='"3$\sigma$"',
zorder=0)
plt.ylim(-5, 20)
plt.ylabel("Pseudo S/N")
plt.xlabel("Projected Separation (px)")
plt.title('%s\nsl %d px, filt %d px' % (fn, bw, fs), fontsize=10)
plt.legend(ncol=2,
fontsize=fontSize - 4,
labelspacing=0.2,
columnspacing=1,
borderpad=0.4,
handletextpad=0.3,
handlelength=1,
framealpha=0.5)
plt.draw()
# Plot folded S/N-like things.
fig3 = plt.figure(7)
plt.clf()
plt.subplot(111)
plt.subplots_adjust(bottom=0.13, right=0.98, top=0.91, left=0.15)
plt.axvspan(xmin=0, xmax=r_mask, color='k', alpha=0.1)
plt.axvline(x=0, c='0.4', linestyle='--', linewidth=1.)
plt.axhline(y=0, c='0.4', linestyle='--', linewidth=1.)
# for ii, sl in enumerate(sl_list[4:]):
# plt.plot(range(0,data_Qr.shape[0])-star[0], sl_dQr_snr[ii], label='%d' % pa_sls[ii], c=cut_colors[ii], linestyle='-', linewidth=1.5)
folded_snr = np.array([
sl_snr[ii][0:sl_snr[ii].shape[0] // 2][::-1] +
sl_snr[ii][1 + sl_snr[ii].shape[0] // 2:] for ii in range(len(sl_list))
])
folded_mean = np.nanmean(folded_snr, axis=0)
for ii, sl in enumerate(sl_list[:4]):
plt.plot(range(0, data.shape[0] // 2),
folded_snr[ii],
label='%d' % pa_sls[ii],
c=cut_colors[ii],
linestyle='-',
linewidth=1.5)
plt.plot(range(0, data.shape[0] // 2),
folded_mean,
label='Mean',
c='k',
linestyle='-',
linewidth=2.5)
plt.axhline(y=3,
c='0.2',
linestyle='--',
linewidth=1.5,
label='"3$\sigma$"',
zorder=0)
plt.ylim(-5, 20)
plt.ylabel("Folded Pseudo S/N")
plt.xlabel("Projected Separation (px)")
plt.title('%s\nsl %d px, filt %d px' % (fn, bw, fs), fontsize=10)
plt.legend(ncol=2,
fontsize=fontSize - 4,
labelspacing=0.2,
columnspacing=1,
borderpad=0.4,
handletextpad=0.3,
handlelength=1,
framealpha=0.5)
plt.draw()
# Compile list of hits.
snr_hits = np.where(sl_snr >= 3)
folded_snr_hits = np.where(folded_snr >= 3)
folded_mean_hits = np.where(folded_mean >= 3)
N_snr_hits = snr_hits[0].shape[0]
N_folded_snr_hits = folded_snr_hits[0].shape[0]
N_folded_mean_hits = folded_mean_hits[0].shape[0]
print("\nS/N > 3: %d (%d%%)" %
(N_snr_hits, 100 * N_snr_hits / sl_snr.size))
print("Folded S/N > 3: %d (%d%%)" %
(N_folded_snr_hits, 100 * N_folded_snr_hits / folded_snr.size))
print("Folded Mean S/N > 3: %d (%d%%)" %
(N_folded_mean_hits, 100 * N_folded_mean_hits / folded_mean.size))
print("Max slice S/N: %.1f" % np.nanmax(sl_snr))
# Plot flux contours in 3D.
# data3d = data.copy()
# Smooth slightly and trim negative values for better contouring.
data3d = filters.gaussian_filter(np.nan_to_num(data.copy()), 2.)
data3d[data3d < 0.] = np.nan
fig3d = plt.figure(8)
plt.clf()
ax = fig3d.add_subplot(111, projection='3d')
yy, xx = np.mgrid[:data3d.shape[0], :data3d.shape[1]]
yy -= star[0]
xx -= star[1]
ax.plot([0, 0], [0, 0], [0, np.nanmax(data3d)], 'k--')
nlevels = 20
c3d = ax.contour(xx, yy, data3d, levels=nlevels)
ax.set_xlabel('X')
ax.set_ylabel('Y')
ax.set_zlabel('Flux')
ax.set_title('%s\n%d contours' % (fn, nlevels), fontsize=10, pad=40)
plt.draw()
# fig3 = plt.figure(33)
# ax1 = plt.subplot(111)
# # ax1.imshow((radii**2)*conv_WtoJy*I_scale*mod_hdu[0].data[0,0,0]/1000., vmin=vmin, vmax=vmax, cmap=colmap_devs) #viridis)
# # ax1.imshow(np.log10(conv_WtoJy*I_scale*mod_hdu[0].data[0,0,0]), vmin=vmin*2., vmax=vmax*2., cmap=colmap_devs) #viridis)
# from matplotlib.colors import SymLogNorm
# ax1.imshow(conv_WtoJy*I_scale*mod_hdu[0].data[0,0,i_inc], vmin=0., vmax=vmax*10., cmap=colmap,
# norm=SymLogNorm(linthresh=0.05, linscale=1.0, vmin=0., vmax=vmax*10.))
# ax1.set_xlim(star[1]-wdw[1], star[1]+wdw[1])
# ax1.set_ylim(star[0]-wdw[0], star[0]+wdw[0])
# fpm_circle = plt.Circle(star[::-1], 8.68, color='k', fill=False)
# ax1.add_artist(fpm_circle)
# # plt.title('%.2f x Raw Model * r^2' % I_scale)
# plt.title('%.2f x Raw Model I' % I_scale)
# ax1.tick_params(which='both', direction='in', color=ctick)
# ax1.yaxis.set_ticks(np.arange(90, 191, 50))
# ax1.xaxis.set_ticks(np.arange(90, 191, 50))
# # ax1.yaxis.set_minor_locator(minor_locator)
# # ax1.xaxis.set_minor_locator(minor_locator)
# ax1.grid(axis='both', which='both', c='0.6', alpha=0.5)
#
# plt.draw()
# if save:
# fig0.savefig(os.path.expanduser('~/Research/data/gpi/auto_disk/%s_autodisk_imagecuts' % fn) + '.png', dpi=200, transparent=False, format='png')
# fig2.savefig(os.path.expanduser('~/Research/data/gpi/auto_disk/%s_autodisk_slsnr' % fn) + '.png', dpi=200, transparent=False, format='png')
# fig3.savefig(os.path.expanduser('~/Research/data/gpi/auto_disk/%s_autodisk_foldedsnr' % fn) + '.png', dpi=200, transparent=False, format='png')
#
# # Compute the disk brightness; feed in units of ADU/s.
# # Assume data are already in ADU/coadd (default GPI pipeline output).
# itime = data_hdu[hdu_ind].header['ITIME'] # [s] integration time per coadd
# auto_disk_brightness(data/itime, data_Ur/itime, star)
# pdb.set_trace()
if output:
return [
np.nanmax(sl_snr), N_snr_hits, 100 * N_snr_hits / sl_snr.size,
N_folded_snr_hits, 100 * N_folded_snr_hits / folded_snr.size,
N_folded_mean_hits, 100 * N_folded_mean_hits / folded_mean.size
]
else:
return
def _perform(self):
# image sections for each amp
bsec, dsec, tsec, direc = self.action.args.map_ccd
namps = len(bsec)
# polynomial fit order
if namps == 4:
porder = 2
else:
porder = 7
# header keyword to update
key = 'OSCANSUB'
keycom = 'Overscan subtracted?'
# is it performed?
performed = False
# loop over amps
# plts = [] # plots for each amp
for ia in range(namps):
# get gain
gain = self.action.args.ccddata.header['GAIN%d' % (ia + 1)]
# check if we have enough data to fit
if (bsec[ia][3] -
bsec[ia][2]) > self.config.instrument.minoscanpix:
# pull out an overscan vector
x0 = bsec[ia][2] + self.config.instrument.oscanbuf
x1 = bsec[ia][3] - self.config.instrument.oscanbuf
y0 = bsec[ia][0]
y1 = bsec[ia][1] + 1
osvec = np.nanmedian(self.action.args.ccddata.data[y0:y1,
x0:x1],
axis=1)
nsam = x1 - x0
xx = np.arange(len(osvec), dtype=np.float)
# fit it, avoiding first 50 px
if direc[ia]:
# forward read skips first 50 px
oscoef = np.polyfit(xx[50:], osvec[50:], porder)
else:
# reverse read skips last 50 px
oscoef = np.polyfit(xx[:-50], osvec[:-50], porder)
# generate fitted overscan vector for full range
osfit = np.polyval(oscoef, xx)
# calculate residuals
resid = (osvec - osfit) * math.sqrt(nsam) * gain / 1.414
sdrs = float("%.3f" % np.std(resid))
self.logger.info("Amp%d Read noise from oscan in e-: %.3f" %
((ia + 1), sdrs))
self.action.args.ccddata.header['OSCNRN%d' % (ia + 1)] = \
(sdrs, "amp%d RN in e- from oscan" % (ia + 1))
if self.config.instrument.plot_level >= 1:
x = np.arange(len(osvec))
p = figure(title=self.action.args.plotlabel +
'OSCAN amp %d' % (ia + 1),
x_axis_label='overscan px',
y_axis_label='counts',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.circle(x, osvec, legend_label="Data")
p.line(x,
osfit,
line_color='red',
line_width=3,
legend_label="Fit")
bokeh_plot(p, self.context.bokeh_session)
# plts.append(p)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
# subtract it
for ix in range(dsec[ia][2], dsec[ia][3] + 1):
self.action.args.ccddata.data[y0:y1, ix] = \
self.action.args.ccddata.data[y0:y1, ix] - osfit
performed = True
else:
self.logger.info("not enough overscan px to fit amp %d")
# if self.config.instrument.plot_level >= 1 and len(plts) > 0:
# bokeh_plot(gridplot(plts, ncols=(2 if namps > 2 else 1),
# plot_width=500, plot_height=300,
# toolbar_location=None),
# self.context.bokeh_session)
# if self.config.instrument.plot_level >= 2:
# input("Next? <cr>: ")
# else:
# time.sleep(self.config.instrument.plot_pause)
self.action.args.ccddata.header[key] = (performed, keycom)
log_string = SubtractOverscan.__module__
self.action.args.ccddata.header['HISTORY'] = log_string
self.logger.info(log_string)
return self.action.args
O_E__JK.append(O_E__K), OplusE__JK.append(
OplusE__K), O_E_grad__JK.append(O_E_grad__K)
print("DEBUG O_Eshapes (-,+,-_-grad):\t{}, {}, {}".format(
np.array(O_E__K).shape,
np.array(OplusE__K).shape,
np.array(O_E_grad__K).shape))
slitdiffnorm__K = np.array(O_E_grad__K) / np.array(OplusE__K)
# Determine the Stokes parameter and polarization degrees
if filtername == "b_HIGH":
offsetangle = 2. * 1.54
elif filtername == "v_HIGH":
offsetangle = 2. * 1.8
UQPphi, sigma_UQPphi = polfun.detpol(slitdiffnorm__K,
np.nanmedian(np.sqrt(OplusE__K),
axis=0),
offsxy0__45=[Qoffsx, Qoffsy],
offsxy22_5__67_5=[Uoffsx, Uoffsy],
offsxy0__22_5=[QUoffsx, QUoffsy],
corran=offsetangle)
# Append results to list
UQPphi_J.append(UQPphi), sigma_UQPphi_J.append(sigma_UQPphi)
# Save results to fits files
polfun.savefits(UQPphi[1], imsavedir + "/tpl{}".format(j + 1),
"Q_i{}j{}".format(i + 1, j + 1))
polfun.savefits(UQPphi[0], imsavedir + "/tpl{}".format(j + 1),
"U_i{}j{}".format(i + 1, j + 1))
polfun.savefits(UQPphi[2], imsavedir + "/tpl{}".format(j + 1),
"P_i{}j{}".format(i + 1, j + 1))
polfun.savefits(sigma_UQPphi[1], imsavedir + "/tpl{}".format(j + 1),
"sigmaQ_i{}j{}".format(i + 1, j + 1))
def printcoords(event):
global dph, lines1, lines2, lastevent, imdates_ordinal, imdates_dt
# outputting x and y coords to console
if event.inaxes != axv:
return
elif event.button != 1:
return
elif not event.dblclick: ## Only double click
return
else:
lastevent = event ## Update last event
ii = np.int(np.round(event.ydata))
jj = np.int(np.round(event.xdata))
### Plot on image window
ii1h = ii - 0.5; ii2h = ii + 1 - 0.5
jj1h = jj - 0.5; jj2h = jj + 1 - 0.5
pax.set_data([jj1h, jj2h, jj2h, jj1h, jj1h], [ii1h, ii1h, ii2h, ii2h, ii1h])
pax2.set_data(jj, ii)
pv.canvas.draw()
axts.cla()
axts.grid(zorder=0)
axts.set_axisbelow(True)
axts.set_xlabel('Date')
axts.set_ylabel('Cumulative Displacement (mm)')
axts.set_ylim(dmin, dmax)
### Get values of noise indices and incidence angle
noisetxt = ''
for key in mapdict_data:
# val_temp = mapdict_data[key]
val = mapdict_data[key][ii, jj]
# val = val_temp[0,ii,jj]
unit = mapdict_unit[key]
if key.startswith('Velocity'):
continue
elif key.startswith('n_') or key == 'mask':
noisetxt = noisetxt + '{}: {:d} {}\n'.format(key, int(val), unit)
else:
noisetxt = noisetxt + '{}: {:.2f} {}\n'.format(key, float(val), unit)
try: # Only support from Matplotlib 3.1!
axts.xaxis.set_major_formatter(mdates.ConciseDateFormatter(loc_ts))
except:
axts.xaxis.set_major_formatter(mdates.DateFormatter('%Y/%m/%d'))
for label in axts.get_xticklabels():
label.set_rotation(20)
label.set_horizontalalignment('right')
### If not masked
### tscuml file
vel1p = vel[0, ii, jj]
intercept1p = intercept[0,ii,jj]
dph = tscuml[:,ii,jj]
err = np.ones(dph.shape) * np.std(dph) # using constant err value
errp = mapdict_data['Error'][ii,jj] # for error reading
## fit function
lines1 = [0, 0, 0, 0]
xvalues = np.arange(imdates_ordinal[0], imdates_ordinal[-1], 10)
xdates = [dt.fromordinal(pp) for pp in xvalues]
# Mask to exclude nan elements
mask = ~np.isnan(dph)
# remove nan elements from both arrays
imo = np.asarray(imdates_ordinal)
imo = imo[mask]
imt = np.asarray(imdates_dt)
imt = imt[mask]
dph = dph[mask]
err = err[mask]
for model, vis in enumerate(visibilities):
if len(dph) > 1:
yvalues = calc_model(dph, imo, xvalues, model, vel1p, intercept1p)
if model == 0:
lines1[model], = axts.plot(xdates, yvalues, 'r-', label=label2, visible=vis, alpha=0.6, zorder=3)
axts.legend()
else:
lines1[model], = axts.plot(xdates, yvalues, 'r-', visible=vis, alpha=0.6, zorder=3)
axts.scatter(imt, dph, label=label1, c='b', alpha=0.6, zorder=5)
# axts.errorbar(imt, dph, yerr=err, label=label1, fmt='.', color='black', ecolor='blue', elinewidth=0.5, capsize=2)
axts.set_title('Velocity = {:.1f} +/- {:.1f} [mm/yr] @({}, {})'.format(vel1p, errp, ii, jj), fontsize=10)
# axts.set_ylim(-100,100)
### Y axis
if ylen:
vlim = [np.nanmedian(dph) - ylen / 2, np.nanmedian(dph) + ylen / 2]
axts.set_ylim(vlim)
### Legend
axts.legend()
pts.canvas.draw()
def plot_size_decay_in_recovery(tracked_cells_df, sav_dir, start_end_NEW_cell=[3, 8], min_survive_frames=3, use_scaled=1, y_lim=10000, last_plot_week=0, ax_title_size=16, x_label='Weeks after recovery', intial_week=1, figsize=(6, 5)):
start_frame = start_end_NEW_cell[0]
end_frame = start_end_NEW_cell[1]
plt.figure(figsize=figsize);
list_arrs = []; list_arrs_unscaled = []; list_z = []; list_indices = [];
for frame in np.unique(tracked_cells_df.FRAME):
if frame >= start_frame and frame <= end_frame:
all_sizes_cur_frame, all_sizes_scaled, all_z, all_indices = get_sizes_and_z_cur_frame(tracked_cells_df, frame, use_scaled=use_scaled)
for idx in range(len(all_sizes_cur_frame)):
sizes_cur_cell = all_sizes_cur_frame[idx]
sizes_scaled = all_sizes_scaled[idx]
z_cur_cell = all_z[idx]
cur_indices = all_indices[idx]
if len(sizes_cur_cell) >= min_survive_frames: ### MUST BE ALIVE FOR AT LEAST 4 frames
from random import uniform
t = uniform(0.,2.)
col = (t/2.0, t/2.0, t/2.0) # different gray colors
plt.plot(np.arange(intial_week, len(sizes_cur_cell[:last_plot_week]) + intial_week), sizes_cur_cell[:last_plot_week], linewidth=0.5, color=col, alpha=0.2)
plt.ylim(intial_week - 1, y_lim)
plt.tight_layout()
list_arrs.append(list(sizes_scaled[:last_plot_week]))
list_arrs_unscaled.append(list(sizes_cur_cell[:last_plot_week]))
list_z.append(list(z_cur_cell[:last_plot_week]))
list_indices.append(list(cur_indices[:last_plot_week]))
### sort z dist as well
row_lengths = []
for row in list_arrs_unscaled:
row_lengths.append(len(row))
max_length = max(row_lengths)
for row in list_arrs_unscaled:
while len(row) < max_length:
row.append(np.nan)
balanced_array = np.array(list_arrs_unscaled)
### find mean and std of sizes
size_arr_unscaled = np.stack(balanced_array)
### find mean and std of sizes
mean = np.nanmedian(size_arr_unscaled, axis=0)
std = np.nanstd(size_arr_unscaled, axis=0)
### find sem
all_n = []
for col_idx in range(len(balanced_array[0, :])):
all_n.append(len(np.where(~np.isnan(balanced_array[:,col_idx]))[0]))
sem = std/np.sqrt(all_n)
### plot
plt.plot(np.arange(intial_week, len(mean) + intial_week), mean, color='r', linestyle='dotted', linewidth=2)
plt.scatter(np.arange(intial_week, len(mean) + intial_week), mean, color='r', marker='o', s=30)
plt.errorbar(np.arange(intial_week, len(mean) + intial_week), mean, yerr=sem, color='r', fmt='none', capsize=10, capthick=1)
plt.xlabel(x_label, fontsize=ax_title_size)
plt.ylabel('Soma volume ($\u03bcm^3$)', fontsize=ax_title_size)
ax = plt.gca()
rs = ax.spines["right"]; rs.set_visible(False)
ts = ax.spines["top"]; ts.set_visible(False)
from matplotlib.ticker import MaxNLocator ### force integer tick sizes
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
plt.tight_layout()
plt.savefig(sav_dir + x_label + '_cell sizes changing statistics.png')
### sort list of lists so can be converted to array for mean and std
row_lengths = []
for row in list_arrs:
row_lengths.append(len(row))
max_length = max(row_lengths)
for row in list_arrs:
while len(row) < max_length:
row.append(np.nan)
balanced_array = np.array(list_arrs)
### find mean and std of sizes
size_arr = np.stack(balanced_array)
mean = np.nanmedian(size_arr, axis=0)
std = np.nanstd(size_arr, axis=0)
### find sem
all_n = []
for col_idx in range(len(balanced_array[0, :])):
all_n.append(len(np.where(~np.isnan(balanced_array[:,col_idx]))[0]))
sem = std/np.sqrt(all_n)
return mean, sem, size_arr, size_arr_unscaled, list_indices
def main():
"""
This is the main function that does all the processing. It calls the required
functions to process the raw image. Raw images are saved in .sav (IDL)
format and read through this script.
"""
data_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\Radiance_Cal_VIS'
telemetry_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\2018.06.27'
image_dir = os.path.join(data_dir, '32_ms')
save_dir = os.path.join(image_dir, 'processed_image')
image_save_dir = os.path.join(image_dir, 'saved_plots/final')
telemetry_dir = os.path.join(telemetry_dir, 'processed/h5')
linearity = 0 # option to turn on/off linearity correction
smear = 1
cross_talk = 1
all_image = []
var_all = []
signal_all = []
dark_current = 1
if not os.path.exists(save_dir):
os.makedirs(save_dir)
# data_path_all = sorted([each for each in os.listdir(image_dir)
# if each.startswith('2017_07_30_00_24_59_38064')
# and each.endswith('.sav')])
#
data_path_all = sorted(
[each for each in os.listdir(image_dir) if each.endswith('.sav')])
# #dark_data = data_path_all[1:len(data_path_all):2]
print('Total data = ', len(data_path_all))
# resultFile = open (os.path.join(save_dir,'file_name_all.csv'),'w')
#
# for results in data_path_all[0:15]:
# resultFile.write(results+"\n")
# resultFile.close()
if dark_current:
dark_current = calculate_dark_current(data_dir)
else:
dark_current = 1
count = 0
#peak_loc_all = []
for data_path in data_path_all:
data_file = os.path.join(image_dir, data_path)
print(data_path)
# cc
telemetry_file_name = data_path.split('.')[0]
#################### Telemetry Statistics###############################
# Now lets parse the telemetry file to get the required information###
#Note; The function can return the CCD temp and FPE temp. But these are
#not needed on the image processing algorithm.. atleast for the time being
telemetry_file = os.path.join(telemetry_dir,
telemetry_file_name + '.h5')
if os.path.exists(telemetry_file):
coadds, int_time, fpe_temp, fpa_temp = parse_telemetry_file(
telemetry_dir, telemetry_file_name)
print('FPE Temp. = ', round(fpe_temp, 2), 'FPA Temp = ',
round(fpa_temp, 2))
print('Integ. Time =', int_time)
else:
print('Telemetry file missing')
coadds = 10
int_time = 6000
fpe_temp = 43
fpa_temp = -21.5
##############Image Processing begins here##############################
full_frame = read_idl_file(data_file)
#check_image = create_image_active_region(full_frame)
raw_image = create_final_image(np.array(full_frame))
# print('Max. Val. Raw = ', np.max(raw_image))
quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']
##########################OFFSET REMOVAL###############################
# Input : Full_Frame TEMPO IMAGE
# Otput : Bias Subtracted Active Region. The active region dimensions are
#now 1028*1024. 2 pixel lines from SMEAR overclocks are now used for
#to store storage summation information
# For dark data, only offset removal is needed to compute the dark current
# For light data, additional processing such as linearity, smear and
#cross talk is needed
bias_removed_quads = perform_bias_subtraction(full_frame)
#print('Max. Val. Offset Removed = ', np.max(bias_removed_quads))
text1 = telemetry_file_name+'.img' +' (Raw Data)\n Int. time:' + str(round(int_time, 3))+ \
'ms, Co-adds:' +str(int(coadds))+\
', FPE temp:'+ str(round(fpe_temp, 1))+'C, ' + \
' FPA temp: ' + str(round(fpa_temp, 1))+'C'
# Now let us save the raw image
raw_image_save = os.path.join(image_save_dir, 'raw_image')
if not os.path.exists(raw_image_save):
os.makedirs(raw_image_save)
plot_save_dir = raw_image_save + '/' + data_path + '.png'
#prnu_spectrometer[prnu_spectrometer < 0.9] = 0.9
#prnu_spectrometer[prnu_spectrometer > 1.2] = 1.2
#print(np.min(check_image))
#create_image(check_image, text1, plot_save_dir)
#---------------------------------------------------------------------
#########################NON LINEARITY CORRECTION#######################
# Input : Bias Subtracted Active regions
# Output : Linearized Active Region Quads
if linearity:
linearized_a = apply_linearity_correction(
bias_removed_quads[0, :, :], quads[0], coadds)
linearized_b = apply_linearity_correction(
bias_removed_quads[1, :, :], quads[1], coadds)
linearized_c = apply_linearity_correction(
bias_removed_quads[2, :, :], quads[2], coadds)
linearized_d = apply_linearity_correction(
bias_removed_quads[3, :, :], quads[3], coadds)
linearized_quads = np.array(
[linearized_a, linearized_b, linearized_c, linearized_d])
#print(np.mean(linearized_quads))
else:
linearized_quads = bias_removed_quads
#----------------------------------------------------------------------
##########################SMEAR REMOVAL################################
# Input : linearized quads ( all quads together)
# Output : SMEAR offset corrected Quad
#### lets' create the masked array with outlier mask################
# The outlier mask is array of list of 4 quads.
#print('Max. Val. Linearized = ', np.max(linearized_quads))
outlier_mask = read_outlier_mask()
# Note : all the arrays after this step are masked arrays
if smear:
smear_removed_quads = perform_smear_removal(
linearized_quads, int_time, outlier_mask)
else:
smear_removed_quads = linearized_quads
#----------------------------------------------------------------------
##########################CROSS-TALK REMOVAL###########################
# Input : smear removed quads (all quads together)
# Output : cross talk removed quads
#print('Max. Val. Smeared = ', np.max(smear_removed_quads))
if cross_talk:
cross_talk_removed_quads = remove_cross_talk(
np.array(smear_removed_quads))
else:
cross_talk_removed_quads = smear_removed_quads
#----------------------------------------------------------------------
#print('Max. Val. Cross Talked = ', np.max(cross_talk_removed_quads))
processed_image = create_final_image(
np.array(cross_talk_removed_quads))
processed_image = processed_image - dark_current
#prnu_map = parse_prnu_file()
prnu_map = read_prnu_files()
prnu_spectrometer = create_final_image(np.array(prnu_map))
prnu_spectrometer[prnu_spectrometer > 1.03] = 1.02
prnu_spectrometer[prnu_spectrometer < 0.97] = 0.98
outlier_spectrometer = create_final_image(np.array(outlier_mask))
nx_quad, ny_quad = processed_image.shape
outlier_spectrometer = np.reshape(outlier_spectrometer,
(nx_quad * ny_quad, 1))
#outlier_detectors = np.array(np.where([outlier_spectrometer == 1]))
#print('outliers =', outlier_detectors.shape[1])
outlier_spectrometer = np.reshape(outlier_spectrometer,
(nx_quad, ny_quad))
#create_image(outlier_spectrometer,'outliers = '+str(outlier_detectors.shape[1]),'b')
processed_image = processed_image / (coadds * prnu_spectrometer)
processed_image[processed_image >= 0.90 * 16383] = np.NAN
#processed_image = processed_image*180/int_time
#print(np.nanmax(processed_image))
#processed_image[processed_image>=0.85*16383] = 16383
text1 = telemetry_file_name+'.img, ' + 'Int. time:' + str(round(int_time,2))+ \
'ms\n Co-adds:' +str(int(coadds))+\
', FPE temp:'+ str(round(fpe_temp, 1))+'C, ' + \
' FPA temp: ' + str(round(fpa_temp, ))+'C'
processed_image_save = os.path.join(image_save_dir, 'processed_image')
if not os.path.exists(processed_image_save):
os.makedirs(processed_image_save)
plot_save_dir = processed_image_save + '/' + data_path + '.png'
#processed_image = processed_image[:, 500:2000]
#processed_image[processed_image>6000] = 6000
#processed_image = uniform_filter(processed_image, size=(15, 15), mode='mirror')
#create_image(raw_image, text1, plot_save_dir)
all_image.append(processed_image[:, 6:2041])
#print(np.array(all_image).shape)
#cc
count = count + 1
all_image = 16.5 * np.array(all_image) # Gain is 16.5 from RMM
var_all = np.nanvar(all_image, axis=0)
signal_all = np.nanmedian(all_image, axis=0)
variance_file_name = image_save_dir + '/' + 'variance_all_electrons_32ms.csv'
mean_file_name = image_save_dir + '/' + 'mean_all_electrons_32ms.csv'
np.savetxt(variance_file_name, var_all, fmt='%1.3f', delimiter=",")
np.savetxt(mean_file_name, signal_all, fmt='%1.3f', delimiter=",")
def draw_mask_map(image,
seg_map,
mask_deep,
stars,
r_core=None,
r_out=None,
vmin=None,
vmax=None,
pad=0,
save=False,
save_dir='./'):
""" Visualize mask map """
from matplotlib import patches
mu = np.nanmedian(image)
std = mad_std(image)
if vmin is None:
vmin = mu - std
if vmax is None:
vmax = mu + 10 * std
fig, (ax1, ax2, ax3) = plt.subplots(ncols=3, nrows=1, figsize=(20, 6))
im1 = ax1.imshow(image, cmap='gray', norm=LogNorm(), vmin=vmin, vmax=1e4)
ax1.set_title("Image")
n_label = seg_map.max()
ax2.imshow(seg_map, vmin=1, vmax=n_label - 2, cmap=make_rand_cmap(n_label))
ax2.set_title("Deep Mask")
image2 = image.copy()
image2[mask_deep] = 0
im3 = ax3.imshow(image2,
norm=LogNorm(),
vmin=vmin,
vmax=vmax,
aspect='auto')
ax3.set_title("'Sky'")
colorbar(im3)
if r_core is not None:
if np.ndim(r_core) == 0:
r_core = [r_core, r_core]
star_pos_A = stars.star_pos_verybright + pad
star_pos_B = stars.star_pos_medbright + pad
aper = CircularAperture(star_pos_A, r=r_core[0])
aper.plot(color='lime', lw=2, label="", alpha=0.9, axes=ax3)
aper = CircularAperture(star_pos_B, r=r_core[1])
aper.plot(color='c', lw=2, label="", alpha=0.7, axes=ax3)
if r_out is not None:
aper = CircularAperture(star_pos_A, r=r_out[0])
aper.plot(color='lime', lw=1.5, label="", alpha=0.9, axes=ax3)
aper = CircularAperture(star_pos_B, r=r_out[1])
aper.plot(color='c', lw=1.5, label="", alpha=0.7, axes=ax3)
patch_size = image.shape[0] - pad * 2
rec = patches.Rectangle((pad, pad),
patch_size,
patch_size,
facecolor='none',
edgecolor='w',
linewidth=2,
linestyle='--',
alpha=0.8)
ax3.add_patch(rec)
plt.tight_layout()
if save:
plt.savefig(os.path.join(save_dir, "Mask_dual.png"), dpi=120)
plt.show()
plt.close()
else:
plt.show()
def replace_missing_values(feature_table,
replacement_dict=None,
replacement_method=MEAN_VALUE_REPLACEMENT_METHOD):
"""For each feature, replaces missing values with mean or median.
N = number of examples
M = number of features (input variables)
For both input and output, `replacement_dict` is a dictionary with the
following keys. Only one of the last two keys is present.
replacement_dict['feature_name']: Name of feature.
replacement_dict['original_mean']: Original mean (before replacement of
missing values).
replacement_dict['original_median']: Original median (before replacement of
missing values).
If `replacement_dict` is None, will assume that `feature_table` contains
training data and use `replacement_method`. If `replacement_dict` is
specified, will assume that `feature_table` contains non-training data, in
which case missing values will be replaced with means (or medians) from
training data.
:param feature_table: pandas DataFrame with N rows and M columns. Column
names are feature names.
:param replacement_dict: Dictionary with keys listed above.
:param replacement_method: String ("mean" or "median").
:return: feature_table_no_missing_values: Same as input, except that missing
values have been replaced.
:return: replacement_dict: Dictionary with keys listed above.
:raises: ValueError: if any column of feature_table has < 2 real values
(not NaN).
"""
num_real_values_by_feature = numpy.sum(numpy.invert(
numpy.isnan(feature_table.as_matrix())),
axis=0)
if numpy.any(num_real_values_by_feature < 2):
raise ValueError('Each column of feature_table must have >= 2 real '
'values (not NaN).')
if replacement_dict is None:
error_checking.assert_is_string(replacement_method)
if replacement_method not in VALID_REPLACEMENT_METHODS:
error_string = (
'\n\n{0:s}\n\nValid replacement methods (listed above) do not '
'include "{1:s}".').format(VALID_REPLACEMENT_METHODS,
replacement_method)
raise ValueError(error_string)
if replacement_method == MEAN_VALUE_REPLACEMENT_METHOD:
replacement_dict = {
FEATURE_NAMES_KEY:
list(feature_table),
ORIGINAL_MEANS_KEY:
numpy.nanmean(feature_table.as_matrix(), axis=0)
}
else:
replacement_dict = {
FEATURE_NAMES_KEY:
list(feature_table),
ORIGINAL_MEDIANS_KEY:
numpy.nanmedian(feature_table.as_matrix(), axis=0)
}
else:
replacement_dict = _reorder_standardization_or_replacement_dict(
replacement_dict, list(feature_table))
feature_names = list(feature_table)
num_features = len(feature_names)
feature_table_no_missing_values = None
for j in range(num_features):
these_values = copy.deepcopy(feature_table[feature_names[j]].values)
nan_indices = numpy.where(numpy.isnan(these_values))[0]
if ORIGINAL_MEANS_KEY in replacement_dict:
these_values[nan_indices] = replacement_dict[ORIGINAL_MEANS_KEY][j]
else:
these_values[nan_indices] = replacement_dict[ORIGINAL_MEDIANS_KEY][
j]
if feature_table_no_missing_values is None:
feature_table_no_missing_values = pandas.DataFrame.from_dict(
{feature_names[j]: these_values})
else:
feature_table_no_missing_values = (
feature_table_no_missing_values.assign(
**{feature_names[j]: these_values}))
return feature_table_no_missing_values, replacement_dict
def main(argv=None):
#%% Check argv
if argv == None:
argv = sys.argv
start = time.time()
ver=1.2; date=20200228; author="Y. Morishita"
print("\n{} ver{} {} {}".format(os.path.basename(argv[0]), ver, date, author), flush=True)
print("{} {}".format(os.path.basename(argv[0]), ' '.join(argv[1:])), flush=True)
#%% Set default
ifgdir = []
tsadir = []
loop_thre = 1.5
cmap_noise = 'viridis'
cmap_noise_r = 'viridis_r'
#%% Read options
try:
try:
opts, args = getopt.getopt(argv[1:], "hd:t:l:", ["help"])
except getopt.error as msg:
raise Usage(msg)
for o, a in opts:
if o == '-h' or o == '--help':
print(__doc__)
return 0
elif o == '-d':
ifgdir = a
elif o == '-t':
tsadir = a
elif o == '-l':
loop_thre = float(a)
if not ifgdir:
raise Usage('No data directory given, -d is not optional!')
elif not os.path.isdir(ifgdir):
raise Usage('No {} dir exists!'.format(ifgdir))
elif not os.path.exists(os.path.join(ifgdir, 'slc.mli.par')):
raise Usage('No slc.mli.par file exists in {}!'.format(ifgdir))
except Usage as err:
print("\nERROR:", file=sys.stderr, end='')
print(" "+str(err.msg), file=sys.stderr)
print("\nFor help, use -h or --help.\n", file=sys.stderr)
return 2
print("\nloop_thre : {} rad".format(loop_thre), flush=True)
#%% Directory setting
ifgdir = os.path.abspath(ifgdir)
if not tsadir:
tsadir = os.path.join(os.path.dirname(ifgdir), 'TS_'+os.path.basename(ifgdir))
if not os.path.isdir(tsadir):
print('\nNo {} exists!'.format(tsadir), file=sys.stderr)
return 1
tsadir = os.path.abspath(tsadir)
loopdir = os.path.join(tsadir, '12loop')
if not os.path.exists(loopdir): os.mkdir(loopdir)
loop_pngdir = os.path.join(loopdir ,'good_loop_png')
bad_loop_pngdir = os.path.join(loopdir,'bad_loop_png')
bad_loop_cand_pngdir = os.path.join(loopdir,'bad_loop_cand_png')
if os.path.exists(loop_pngdir):
shutil.move(loop_pngdir+'/', loop_pngdir+'_old') #move to old dir
if os.path.exists(bad_loop_pngdir):
for png in glob.glob(bad_loop_pngdir+'/*.png'):
shutil.move(png, loop_pngdir+'_old') #move to old dir
shutil.rmtree(bad_loop_pngdir)
if os.path.exists(bad_loop_cand_pngdir):
for png in glob.glob(bad_loop_cand_pngdir+'/*.png'):
shutil.move(png, loop_pngdir+'_old') #move to old dir
shutil.rmtree(bad_loop_cand_pngdir)
os.mkdir(loop_pngdir)
os.mkdir(bad_loop_pngdir)
os.mkdir(bad_loop_cand_pngdir)
ifg_rasdir = os.path.join(tsadir, '12ifg_ras')
if os.path.isdir(ifg_rasdir): shutil.rmtree(ifg_rasdir)
os.mkdir(ifg_rasdir)
bad_ifgrasdir = os.path.join(tsadir, '12bad_ifg_ras')
if os.path.isdir(bad_ifgrasdir): shutil.rmtree(bad_ifgrasdir)
os.mkdir(bad_ifgrasdir)
bad_ifg_candrasdir = os.path.join(tsadir, '12bad_ifg_cand_ras')
if os.path.isdir(bad_ifg_candrasdir): shutil.rmtree(bad_ifg_candrasdir)
os.mkdir(bad_ifg_candrasdir)
no_loop_ifgrasdir = os.path.join(tsadir, '12no_loop_ifg_ras')
if os.path.isdir(no_loop_ifgrasdir): shutil.rmtree(no_loop_ifgrasdir)
os.mkdir(no_loop_ifgrasdir)
infodir = os.path.join(tsadir, 'info')
if not os.path.exists(infodir): os.mkdir(infodir)
resultsdir = os.path.join(tsadir, 'results')
if not os.path.exists(resultsdir): os.mkdir(resultsdir)
netdir = os.path.join(tsadir, 'network')
#%% Read date, network information and size
### Get dates
ifgdates = tools_lib.get_ifgdates(ifgdir)
### Read bad_ifg11
bad_ifg11file = os.path.join(infodir, '11bad_ifg.txt')
bad_ifg11 = io_lib.read_ifg_list(bad_ifg11file)
### Remove bad ifgs and images from list
ifgdates = list(set(ifgdates)-set(bad_ifg11))
ifgdates.sort()
imdates = tools_lib.ifgdates2imdates(ifgdates)
n_ifg = len(ifgdates)
n_im = len(imdates)
### Get size
mlipar = os.path.join(ifgdir, 'slc.mli.par')
width = int(io_lib.get_param_par(mlipar, 'range_samples'))
length = int(io_lib.get_param_par(mlipar, 'azimuth_lines'))
### Get loop matrix
Aloop = loop_lib.make_loop_matrix(ifgdates)
n_loop = Aloop.shape[0]
### Extract no loop ifgs
ns_loop4ifg = np.abs(Aloop).sum(axis=0)
ixs_ifg_no_loop = np.where(ns_loop4ifg==0)[0]
no_loop_ifg = [ifgdates[ix] for ix in ixs_ifg_no_loop]
#%% 1st loop closure check. First without reference
print('\n1st Loop closure check and make png for all possible {} loops...'.format(n_loop), flush=True)
bad_ifg_cand = []
good_ifg = []
loop_ph_rms_ifg = []
for i in range(n_loop):
if np.mod(i, 100) == 0:
print(" {0:3}/{1:3}th loop...".format(i, n_loop), flush=True)
unw12, unw23, unw13, ifgd12, ifgd23, ifgd13 = loop_lib.read_unw_loop_ph(Aloop[i, :], ifgdates, ifgdir, length, width)
## Calculate loop phase and check n bias (2pi*n)
loop_ph = unw12+unw23-unw13
loop_2pin = int(np.round(np.nanmedian(loop_ph)/(2*np.pi)))*2*np.pi
loop_ph = loop_ph-loop_2pin #unbias 2pi x n
loop_ph_rms_ifg.append(np.sqrt(np.nanmean(loop_ph**2)))
### List as good or bad candidate
if loop_ph_rms_ifg[i] >= loop_thre: #Bad loop including bad ifg.
bad_ifg_cand.extend([ifgd12, ifgd23, ifgd13])
else:
good_ifg.extend([ifgd12, ifgd23, ifgd13])
### Output png. If exist in old, move to save time
oldpng = os.path.join(loop_pngdir+'_old/', ifgd12[:8]+'_'+ifgd23[:8]+'_'+ifgd13[-8:]+'_loop.png')
if os.path.exists(oldpng):
### Just move from old png
shutil.move(oldpng, loop_pngdir)
else:
### Make png. Take time a little.
loop_lib.make_loop_png(ifgd12, ifgd23, ifgd13, unw12, unw23, unw13, loop_ph, loop_pngdir)
if os.path.exists(loop_pngdir+'_old/'):
shutil.rmtree(loop_pngdir+'_old/')
#%% Identify bad ifgs and output text
bad_ifg = loop_lib.identify_bad_ifg(bad_ifg_cand, good_ifg)
bad_ifgfile = os.path.join(loopdir, 'bad_ifg_loop.txt')
with open(bad_ifgfile, 'w') as f:
for i in bad_ifg:
print('{}'.format(i), file=f)
### Compute n_unw without bad_ifg11 and bad_ifg
n_unw = np.zeros((length, width), dtype=np.int16)
for ifgd in ifgdates:
if ifgd in bad_ifg:
continue
unwfile = os.path.join(ifgdir, ifgd, ifgd+'.unw')
unw = io_lib.read_img(unwfile, length, width)
unw[unw == 0] = np.nan # Fill 0 with nan
n_unw += ~np.isnan(unw) # Summing number of unnan unw
#%% 2nd loop closure check without bad ifgs to define stable ref area
print('\n2nd Loop closure check without bad ifgs to define ref area...', flush=True)
ns_loop_ph = np.zeros((length, width), dtype=np.float32)
ns_bad_loop = np.zeros((length, width), dtype=np.int16)
loop_ph_rms_points = np.zeros((length, width), dtype=np.float32)
for i in range(n_loop):
if np.mod(i, 100) == 0:
print(" {0:3}/{1:3}th loop...".format(i, n_loop), flush=True)
### Read unw
unw12, unw23, unw13, ifgd12, ifgd23, ifgd13 = loop_lib.read_unw_loop_ph(Aloop[i, :], ifgdates, ifgdir, length, width)
### Skip if bad ifg is included
if ifgd12 in bad_ifg or ifgd23 in bad_ifg or ifgd13 in bad_ifg:
continue
## Calculate loop phase and rms at points
loop_ph = unw12+unw23-unw13
loop_2pin = int(np.round(np.nanmedian(loop_ph)/(2*np.pi)))*2*np.pi
loop_ph = loop_ph-loop_2pin #unbias
ns_loop_ph = ns_loop_ph + ~np.isnan(loop_ph)
loop_ph_sq = loop_ph**2
loop_ph_sq[np.isnan(loop_ph_sq)] = 0
loop_ph_rms_points = loop_ph_rms_points + loop_ph_sq
ns_bad_loop = ns_bad_loop+(loop_ph_sq>np.pi**2) #suspected unw error
# ns_bad_loop = ns_bad_loop+(np.abs(loop_ph)>loop_thre)
## multiple nan seem to generate RuntimeWarning
ns_loop_ph[ns_loop_ph==0] = np.nan # To avoid 0 division
loop_ph_rms_points = np.sqrt(loop_ph_rms_points/ns_loop_ph)
### Find stable ref area which have all n_unw and minimum ns_bad_loop and loop_ph_rms_points
mask1 = (n_unw==np.nanmax(n_unw))
min_ns_bad_loop = np.nanmin(ns_bad_loop)
while True:
mask2 = (ns_bad_loop==min_ns_bad_loop)
if np.all(~(mask1*mask2)): ## All masked
min_ns_bad_loop = min_ns_bad_loop+1 ## Make mask2 again
else:
break
loop_ph_rms_points_masked = loop_ph_rms_points*mask1*mask2
loop_ph_rms_points_masked[loop_ph_rms_points_masked==0] = np.nan
refyx = np.where(loop_ph_rms_points_masked==np.nanmin(loop_ph_rms_points_masked))
refy1 = refyx[0][0] # start from 0, not 1
refy2 = refyx[0][0]+1 # shift +1 for python custom. start from 1 end with width
refx1 = refyx[1][0]
refx2 = refyx[1][0]+1
### Save 12ref.txt
reffile = os.path.join(infodir, '12ref.txt')
with open(reffile, 'w') as f:
print('{0}:{1}/{2}:{3}'.format(refx1, refx2, refy1, refy2), file=f)
### Save loop_ph_rms_masked and png
loop_ph_rms_maskedfile = os.path.join(loopdir, 'loop_ph_rms_masked')
loop_ph_rms_points_masked.tofile(loop_ph_rms_maskedfile)
cmax = np.nanpercentile(loop_ph_rms_points_masked, 95)
pngfile = loop_ph_rms_maskedfile+'.png'
title = 'RMS of loop phase (rad)'
plot_lib.make_im_png(loop_ph_rms_points_masked, pngfile, cmap_noise_r, title, None, cmax)
### Check ref exist in unw. If not, list as noref_ifg
noref_ifg = []
for ifgd in ifgdates:
if ifgd in bad_ifg:
continue
unwfile = os.path.join(ifgdir, ifgd, ifgd+'.unw')
unw_ref = io_lib.read_img(unwfile, length, width)[refy1:refy2, refx1:refx2]
unw_ref[unw_ref == 0] = np.nan # Fill 0 with nan
if np.all(np.isnan(unw_ref)):
noref_ifg.append(ifgd)
bad_ifgfile = os.path.join(loopdir, 'bad_ifg_noref.txt')
with open(bad_ifgfile, 'w') as f:
for i in noref_ifg:
print('{}'.format(i), file=f)
#%% 3rd loop closure check without bad ifgs wrt ref point
print('\n3rd loop closure check taking into account ref phase...', flush=True)
bad_ifg_cand2 = []
good_ifg2 = []
loop_ph_rms_ifg2 = []
for i in range(n_loop):
if np.mod(i, 100) == 0:
print(" {0:3}/{1:3}th loop...".format(i, n_loop), flush=True)
### Read unw
unw12, unw23, unw13, ifgd12, ifgd23, ifgd13 = loop_lib.read_unw_loop_ph(Aloop[i, :], ifgdates, ifgdir, length, width)
### Skip if bad ifg is included
if ifgd12 in bad_ifg or ifgd23 in bad_ifg or ifgd13 in bad_ifg:
loop_ph_rms_ifg2.append('--')
continue
### Skip if noref ifg is included
if ifgd12 in noref_ifg or ifgd23 in noref_ifg or ifgd13 in noref_ifg:
loop_ph_rms_ifg2.append('--')
continue
## Skip if no data in ref area in any unw. It is bad data.
ref_unw12 = np.nanmean(unw12[refy1:refy2, refx1:refx2])
ref_unw23 = np.nanmean(unw23[refy1:refy2, refx1:refx2])
ref_unw13 = np.nanmean(unw13[refy1:refy2, refx1:refx2])
## Calculate loop phase taking into account ref phase
loop_ph = unw12+unw23-unw13-(ref_unw12+ref_unw23-ref_unw13)
loop_ph_rms_ifg2.append(np.sqrt(np.nanmean((loop_ph)**2)))
### List as good or bad candidate
if loop_ph_rms_ifg2[i] >= loop_thre: #Bad loop including bad ifg.
bad_ifg_cand2.extend([ifgd12, ifgd23, ifgd13])
else:
good_ifg2.extend([ifgd12, ifgd23, ifgd13])
#%% Identify additional bad ifgs and output text
bad_ifg2 = loop_lib.identify_bad_ifg(bad_ifg_cand2, good_ifg2)
bad_ifgfile = os.path.join(loopdir, 'bad_ifg_loopref.txt')
with open(bad_ifgfile, 'w') as f:
for i in bad_ifg2:
print('{}'.format(i), file=f)
#%% Output all bad ifg list and identify remaining candidate of bad ifgs
### Merge bad ifg, bad_ifg2, noref_ifg
bad_ifg_all = list(set(bad_ifg+bad_ifg2+noref_ifg)) # Remove multiple
bad_ifg_all.sort()
ifgdates_good = list(set(ifgdates)-set(bad_ifg_all))
ifgdates_good.sort()
bad_ifgfile = os.path.join(infodir, '12bad_ifg.txt')
with open(bad_ifgfile, 'w') as f:
for i in bad_ifg_all:
print('{}'.format(i), file=f)
### Identify removed image and output file
imdates_good = tools_lib.ifgdates2imdates(ifgdates_good)
imdates_bad = list(set(imdates)-set(imdates_good))
imdates_bad.sort()
bad_imfile = os.path.join(infodir, '12removed_image.txt')
with open(bad_imfile, 'w') as f:
for i in imdates_bad:
print('{}'.format(i), file=f)
### Remaining candidate of bad ifg
bad_ifg_cand_res = list(set(bad_ifg_cand2)-set(bad_ifg_all))
bad_ifg_cand_res.sort()
bad_ifg_candfile = os.path.join(infodir, '12bad_ifg_cand.txt')
with open(bad_ifg_candfile, 'w') as f:
for i in bad_ifg_cand_res:
print('{}'.format(i), file=f)
#%% 4th loop to be used to calc n_loop_err and n_ifg_noloop
print('\n4th loop to compute statistics...', flush=True)
ns_loop_err = np.zeros((length, width), dtype=np.int16)
for i in range(n_loop):
if np.mod(i, 100) == 0:
print(" {0:3}/{1:3}th loop...".format(i, n_loop), flush=True)
### Read unw
unw12, unw23, unw13, ifgd12, ifgd23, ifgd13 = loop_lib.read_unw_loop_ph(Aloop[i, :], ifgdates, ifgdir, length, width)
### Skip if bad ifg is included
if ifgd12 in bad_ifg_all or ifgd23 in bad_ifg_all or ifgd13 in bad_ifg_all:
continue
## Compute ref
ref_unw12 = np.nanmean(unw12[refy1:refy2, refx1:refx2])
ref_unw23 = np.nanmean(unw23[refy1:refy2, refx1:refx2])
ref_unw13 = np.nanmean(unw13[refy1:refy2, refx1:refx2])
## Calculate loop phase taking into account ref phase
loop_ph = unw12+unw23-unw13-(ref_unw12+ref_unw23-ref_unw13)
## Count number of loops with suspected unwrap error (>pi)
loop_ph[np.isnan(loop_ph)] = 0 #to avoid warning
ns_loop_err = ns_loop_err+(np.abs(loop_ph)>np.pi) #suspected unw error
#%% Output loop info, move bad_loop_png
loop_info_file = os.path.join(loopdir, 'loop_info.txt')
f = open(loop_info_file, 'w')
print('# loop_thre: {} rad. *: Removed w/o ref, **: Removed w/ ref'.format(loop_thre), file=f)
print('# /: Candidates of bad loops but causative ifgs unidentified', file=f)
print('# image1 image2 image3 RMS w/oref w/ref', file=f)
for i in range(n_loop):
### Find index of ifg
ix_ifg12, ix_ifg23 = np.where(Aloop[i, :] == 1)[0]
ix_ifg13 = np.where(Aloop[i, :] == -1)[0][0]
ifgd12 = ifgdates[ix_ifg12]
ifgd23 = ifgdates[ix_ifg23]
ifgd13 = ifgdates[ix_ifg13]
imd1 = ifgd12[:8]
imd2 = ifgd23[:8]
imd3 = ifgd23[-8:]
## Move loop_png if bad ifg or bad ifg_cand is included
looppngfile = os.path.join(loop_pngdir, '{0}_{1}_{2}_loop.png'.format(imd1, imd2, imd3))
badlooppngfile = os.path.join(bad_loop_pngdir, '{0}_{1}_{2}_loop.png'.format(imd1, imd2, imd3))
badloopcandpngfile = os.path.join(bad_loop_cand_pngdir, '{0}_{1}_{2}_loop.png'.format(imd1, imd2, imd3))
badloopflag1 = ' '
badloopflag2 = ' '
if ifgd12 in bad_ifg or ifgd23 in bad_ifg or ifgd13 in bad_ifg:
badloopflag1 = '*'
shutil.move(looppngfile, badlooppngfile)
elif ifgd12 in bad_ifg2 or ifgd23 in bad_ifg2 or ifgd13 in bad_ifg2:
badloopflag2 = '**'
shutil.move(looppngfile, badlooppngfile)
elif ifgd12 in bad_ifg_cand_res or ifgd23 in bad_ifg_cand_res or ifgd13 in bad_ifg_cand_res:
badloopflag1 = '/'
if os.path.exists(looppngfile):
shutil.move(looppngfile, badloopcandpngfile)
if type(loop_ph_rms_ifg2[i]) == np.float32:
str_loop_ph_rms_ifg2 = "{:.2f}".format(loop_ph_rms_ifg2[i])
else: ## --
str_loop_ph_rms_ifg2 = loop_ph_rms_ifg2[i]
print('{0} {1} {2} {3:5.2f} {4} {5:5s} {6}'.format(imd1, imd2, imd3, loop_ph_rms_ifg[i], badloopflag1, str_loop_ph_rms_ifg2, badloopflag2), file=f)
f.close()
#%% Saving coh_avg, n_unw, and n_loop_err only for good ifgs
print('\nSaving coh_avg, n_unw, and n_loop_err...', flush=True)
### Calc coh avg and n_unw
coh_avg = np.zeros((length, width), dtype=np.float32)
n_coh = np.zeros((length, width), dtype=np.int16)
n_unw = np.zeros((length, width), dtype=np.int16)
for ifgd in ifgdates_good:
ccfile = os.path.join(ifgdir, ifgd, ifgd+'.cc')
if os.path.getsize(ccfile) == length*width:
coh = io_lib.read_img(ccfile, length, width, np.uint8)
coh = coh.astype(np.float32)/255
else:
coh = io_lib.read_img(ccfile, length, width)
coh[np.isnan(coh)] = 0 # Fill nan with 0
coh_avg += coh
n_coh += (coh!=0)
unwfile = os.path.join(ifgdir, ifgd, ifgd+'.unw')
unw = io_lib.read_img(unwfile, length, width)
unw[unw == 0] = np.nan # Fill 0 with nan
n_unw += ~np.isnan(unw) # Summing number of unnan unw
coh_avg[n_coh==0] = np.nan
n_coh[n_coh==0] = 1 #to avoid zero division
coh_avg = coh_avg/n_coh
### Save files
n_unwfile = os.path.join(resultsdir, 'n_unw')
np.float32(n_unw).tofile(n_unwfile)
coh_avgfile = os.path.join(resultsdir, 'coh_avg')
coh_avg.tofile(coh_avgfile)
n_loop_errfile = os.path.join(resultsdir, 'n_loop_err')
np.float32(ns_loop_err).tofile(n_loop_errfile)
### Save png
title = 'Average coherence'
plot_lib.make_im_png(coh_avg, coh_avgfile+'.png', cmap_noise, title)
title = 'Number of used unw data'
plot_lib.make_im_png(n_unw, n_unwfile+'.png', cmap_noise, title, n_im)
title = 'Number of unclosed loops'
plot_lib.make_im_png(ns_loop_err, n_loop_errfile+'.png', cmap_noise_r, title)
#%% Link ras
### First, identify suffix of raster image (ras, bmp, or png?)
unwfile = os.path.join(ifgdir, ifgdates[0], ifgdates[0]+'.unw')
if os.path.exists(unwfile+'.ras'):
suffix = '.ras'
elif os.path.exists(unwfile+'.bmp'):
suffix = '.bmp'
elif os.path.exists(unwfile+'.png'):
suffix = '.png'
for ifgd in ifgdates:
rasname = ifgd+'.unw'+suffix
rasorg = os.path.join(ifgdir, ifgd, rasname)
### Bad ifgs
if ifgd in bad_ifg_all:
os.symlink(os.path.relpath(rasorg, bad_ifgrasdir), os.path.join(bad_ifgrasdir, rasname))
### Remaining bad ifg candidates
elif ifgd in bad_ifg_cand_res:
os.symlink(os.path.relpath(rasorg, bad_ifg_candrasdir), os.path.join(bad_ifg_candrasdir, rasname))
### Good ifgs
else:
os.symlink(os.path.relpath(rasorg, ifg_rasdir), os.path.join(ifg_rasdir, rasname))
if ifgd in no_loop_ifg:
os.symlink(os.path.relpath(rasorg, no_loop_ifgrasdir), os.path.join(no_loop_ifgrasdir, rasname))
#%% Plot network
## Read bperp data or dummy
bperp_file = os.path.join(ifgdir, 'baselines')
if os.path.exists(bperp_file):
bperp = io_lib.read_bperp_file(bperp_file, imdates)
else: #dummy
bperp = np.random.random(n_im).tolist()
pngfile = os.path.join(netdir, 'network12_all.png')
plot_lib.plot_network(ifgdates, bperp, [], pngfile)
pngfile = os.path.join(netdir, 'network12.png')
plot_lib.plot_network(ifgdates, bperp, bad_ifg_all, pngfile)
pngfile = os.path.join(netdir, 'network12_nobad.png')
plot_lib.plot_network(ifgdates, bperp, bad_ifg_all, pngfile, plot_bad=False)
### Network info
## Identify gaps
G = inv_lib.make_sb_matrix(ifgdates_good)
ixs_inc_gap = np.where(G.sum(axis=0)==0)[0]
## Connected network
ix1 = 0
connected_list = []
for ix2 in np.append(ixs_inc_gap, len(imdates_good)-1): #append for last image
imd1 = imdates_good[ix1]
imd2 = imdates_good[ix2]
dyear = (dt.datetime.strptime(imd2, '%Y%m%d').toordinal() - dt.datetime.strptime(imd1, '%Y%m%d').toordinal())/365.25
n_im_connect = ix2-ix1+1
connected_list.append([imdates_good[ix1], imdates_good[ix2], dyear, n_im_connect])
ix1 = ix2+1 # Next connection
#%% Caution about no_loop ifg, remaining large RMS loop and gap
### no_loop ifg
if len(no_loop_ifg)!=0:
no_loop_ifgfile = os.path.join(infodir, '12no_loop_ifg.txt')
with open(no_loop_ifgfile, 'w') as f:
print("\nThere are {} ifgs without loop, recommend to check manually in no_loop_ifg_ras12".format(len(no_loop_ifg)), flush=True)
for ifgd in no_loop_ifg:
print('{}'.format(ifgd), flush=True)
print('{}'.format(ifgd), file=f)
### Remaining candidates of bad ifgs
if len(bad_ifg_cand_res)!=0:
print("\nThere are {} remaining candidates of bad ifgs but not identified.".format(len(bad_ifg_cand_res)), flush=True)
print("Check 12bad_ifg_cand_ras and loop/bad_loop_cand_png.", flush=True)
# for ifgd in bad_ifg_cand_res:
# print('{}'.format(ifgd))
print('\n{0}/{1} ifgs are discarded from further processing.'.format(len(bad_ifg_all), n_ifg), flush=True)
for ifgd in bad_ifg_all:
print('{}'.format(ifgd), flush=True)
### Gap
gap_infofile = os.path.join(infodir, '12network_gap_info.txt')
with open(gap_infofile, 'w') as f:
if ixs_inc_gap.size!=0:
print("Gaps between:", file=f)
print("\nGaps in network between:", flush=True)
for ix in ixs_inc_gap:
print("{} {}".format(imdates_good[ix], imdates_good[ix+1]), file=f)
print("{} {}".format(imdates_good[ix], imdates_good[ix+1]), flush=True)
print("\nConnected network (year, n_image):", file=f)
print("\nConnected network (year, n_image):", flush=True)
for list1 in connected_list:
print("{0}-{1} ({2:.2f}, {3})".format(list1[0], list1[1], list1[2], list1[3]), file=f)
print("{0}-{1} ({2:.2f}, {3})".format(list1[0], list1[1], list1[2], list1[3]), flush=True)
print('\nIf you want to change the bad ifgs to be discarded, re-run with different thresholds before next step.', flush=True)
#%% Finish
elapsed_time = time.time()-start
hour = int(elapsed_time/3600)
minite = int(np.mod((elapsed_time/60),60))
sec = int(np.mod(elapsed_time,60))
print("\nElapsed time: {0:02}h {1:02}m {2:02}s".format(hour,minite,sec))
print('\n{} Successfully finished!!\n'.format(os.path.basename(argv[0])))
print('Output directory: {}\n'.format(os.path.relpath(tsadir)))
#transects = SDS_tools.transects_from_geojson(geojson_file)
# option 3: create the transects by manually providing the coordinates of two points
#transects = dict([])
#transects['Transect 1'] = np.array([[342836, 6269215], [343315, 6269071]])
#transects['Transect 2'] = np.array([[342482, 6268466], [342958, 6268310]])
#transects['Transect 3'] = np.array([[342185, 6267650], [342685, 6267641]])
# intersect the transects with the 2D shorelines to obtain time-series of cross-shore distance
# (also saved a .csv file with the time-series, dates are in UTC time)
settings['along_dist'] = 25
cross_distance = SDS_transects.compute_intersection(output, transects, settings)
# plot the time-series
from matplotlib import gridspec
fig = plt.figure()
gs = gridspec.GridSpec(len(cross_distance),1)
gs.update(left=0.05, right=0.95, bottom=0.05, top=0.95, hspace=0.05)
for i,key in enumerate(cross_distance.keys()):
if np.all(np.isnan(cross_distance[key])):
continue
ax = fig.add_subplot(gs[i,0])
ax.grid(linestyle=':', color='0.5')
ax.set_ylim([-50,50])
ax.plot(output['dates'], cross_distance[key]- np.nanmedian(cross_distance[key]), '-^', markersize=6)
ax.set_ylabel('distance [m]', fontsize=12)
ax.text(0.5,0.95,'Transect ' + key, bbox=dict(boxstyle="square", ec='k',fc='w'), ha='center',
va='top', transform=ax.transAxes, fontsize=14)
mng = plt.get_current_fig_manager()
mng.window.showMaximized()
fig.set_size_inches([15.76, 8.52])
from astropy.table import Table
data_path = '/Users/krogager/Data/NOT/MALS/test_pynot/raw/'
raw_frames = [
'ALzh010097.fits',
'ALzh010098.fits',
'ALzh010099.fits',
'ALzh010100.fits',
]
red_fname = '/Users/krogager/Data/NOT/MALS/test_pynot/imaging/GRB160801A/GRB160801A_r_SDSS.fits'
seg_fname = '/Users/krogager/Data/NOT/MALS/test_pynot/imaging/GRB160801A/GRB160801A_r_SDSS_seg.fits'
red = fits.getdata(red_fname)
hdr = fits.getheader(red_fname)
med_r = np.nanmedian(red)
std_r = np.median(np.fabs(red - med_r))
vmin_r = med_r - 5 * std_r
vmax_r = med_r + 10 * std_r
wcs = WCS(hdr)
cmap = plt.cm.afmhot_r
plt.close('all')
dpi = 100
for num, fname in enumerate(raw_frames):
raw = fits.getdata(data_path + fname)
med = np.nanmedian(raw)
std = np.median(np.fabs(raw - med))
