def test():
dummy_data = {
'PRES': ma.masked_array([1.0, 100, 200, 300, 500, 5000]),
'TEMP': ma.masked_array([27.44, 14.55, 11.96, 11.02, 7.65, 2.12]),
'PSAL': ma.masked_array([35.71, 35.50, 35.13, 35.02, 34.72, 35.03])
}
features = {
'tukey53H': ma.masked_array([0, 0, 0.3525000000000009,
0.35249999999999915, 0, 0],
mask=[True, True, False, False, True, True]),
'tukey53H_norm': ma.masked_array([0, 0, 0.07388721803621254,
0.07388721803621218, 0, 0],
mask = [True, True, False, False, True, True])
}
flags = {'tukey53H_norm': np.array([0, 0, 1, 1, 0, 0], dtype='i1')}
cfg = {
'l': 5,
'threshold': 6,
'flag_good': 1,
'flag_bad': 4
}
y = Tukey53H(dummy_data, 'TEMP', cfg)
y.test()
assert type(y.features) is dict
for f in y.features:
assert ma.allclose(y.features[f], features[f])
for f in y.flags:
assert ma.allclose(y.flags[f], flags[f])
def recip_ex(detector_size, pixel_size, calibrated_center, dist_sample,
ub_mat, wavelength, motors, i_stack, H_range, K_range, L_range):
# convert to Q space
q_values = diffraction.process_to_q(motors, detector_size, pixel_size,
calibrated_center, dist_sample,
wavelength, ub_mat)
# minimum and maximum values of the voxel
q_min = np.array([H_range[0], K_range[0], L_range[0]])
q_max = np.array([H_range[1], K_range[1], L_range[1]])
# no. of bins
dqn = np.array([40, 40, 1])
# process the grid values
(grid_data, grid_occu, std_err,
grid_out, bounds) = diffraction.grid3d(q_values, i_stack, dqn[0], dqn[1],
dqn[2])
grid = np.mgrid[0:dqn[0], 0:dqn[1], 0:dqn[2]]
r = (q_max - q_min) / dqn
X = grid[0] * r[0] + q_min[0]
Y = grid[1] * r[1] + q_min[1]
Z = grid[2] * r[2] + q_min[2]
# creating a mask
_mask = grid_occu <= 10
grid_mask_data = ma.masked_array(grid_data, _mask)
grid_mask_std_err = ma.masked_array(std_err, _mask)
grid_mask_occu = ma.masked_array(grid_occu, _mask)
return X, Y, Z, grid_mask_data
def _generate_with_coordinates(frame, outliers=None):
"""Generate a numpy array to be used for coordinate plotting.
Parameters
----------
frame : (list-like (x, y, z))
The (x, y, z) values to be used in the array
outliers : (number)
Default : None
The value to be used when calculating outliers.
If the value is None then outliers will not be calculated.
Returns
-------
arr : (ndarray)
The generated numpy array
"""
arr = np.vstack(frame)
xs, ys, zs = arr.transpose()
zeros = np.zeros((256, 256))
zeros[(xs, ys)] = zs
zmask = ma.masked_array(zeros, mask=zeros == 0)
# Use Chauvenet's criterion to find the outliers
if outliers is not None:
d_max = np.abs(zmask - zmask.mean()) / zmask.std()
return ma.masked_array(zmask, mask=d_max >= outliers)
return zmask
def __init__(self, matrix, regions='', overlap = False):
# Extract bin names
if matrix.endswith('.gz'):
with gzip.open(matrix) as inFile:
self.binNames = inFile.readline().strip.split('\t')
else:
with open(matrix) as inFile:
self.binNames = inFile.readline().strip().split('\t')
# Create bin dataframe
binDF = pd.DataFrame()
binDF['chr'], binDF['start'], binDF['end'] = zip(
*[re.split(':|-', x) for x in self.binNames])
binDF[['start', 'end']] = binDF[['start', 'end']].astype(int)
binDF['chr'] = binDF['chr'].astype(str)
binDF['centre'] = np.mean([binDF['start'], binDF['end']], axis=0)
self.binDF = binDF
# Open probability matrix and check it is square
probMatrix = np.loadtxt(matrix, skiprows = 1)
if probMatrix.shape[0] != probMatrix.shape[1]:
raise IOError('Matrix must be square')
# Create mask matrix and remove low values
chrArray = np.array(self.binDF['chr'])
maskMatrix = chrArray != chrArray[:,None]
lowValues = probMatrix.sum(axis=0) < 0.5
maskMatrix[lowValues,:] = True
maskMatrix[:,lowValues] = True
# Add group data to dataframe
groups = self.binDF['chr'].copy()
groups[lowValues] = np.nan
self.binDF['group'] = groups
# Create masked probability and distance matrices
self.probMatrix = ma.masked_array(probMatrix, mask = maskMatrix)
centreArray = np.array(self.binDF['centre'])
distMatrix = np.abs(centreArray - centreArray[:,None])
self.distMatrix = ma.masked_array(distMatrix, mask = maskMatrix)
def vectorField(self,plt,X,Y,U,V,title):
if self.plotFields:
# Create mask corresponding to 0 fluid velocity values inside obstructions
M = zeros([X.quiverLength,Y.quiverLength],dtype='bool')
M = (U.quiver == 0)
# Mask the obstructions in the fluid velocity vector field
U.quiver = ma.masked_array(U.quiver,mask=M)
V.quiver = ma.masked_array(V.quiver,mask=M)
# Build and scale the plot
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_ylim(Y.minValue, Y.maxValue)
ax.set_xlim(X.minValue, X.maxValue)
pos1 = ax.get_position()
pos2 = [pos1.x0+((pos1.width - (pos1.width*self.scaleX))*.5), pos1.y0+((pos1.height - (pos1.height*self.scaleY))*.5), pos1.width*self.scaleX, pos1.height*self.scaleY]
ax.set_position(pos2)
title = title + ' Vector Field'
plt.title(title)
plt.xlabel("X [m]")
plt.ylabel("Y [m]")
plt.grid()
quiver(X.quiver,Y.quiver,U.quiver,V.quiver)
def test_posterior_predictive_statistic():
N, D = 10, 4 # D needs to be even
defn = model_definition(N, [bb] * D)
Y = toy_dataset(defn)
prng = rng()
view = numpy_dataview(Y)
latents = [model.initialize(defn, view, prng) for _ in xrange(10)]
q = ma.masked_array(
np.array([(False,) * D], dtype=[('', bool)] * D),
mask=[(False,) * (D / 2) + (True,) * (D / 2)])
statistic = query.posterior_predictive_statistic(q, latents, prng)
assert_equals(statistic.shape, (1,))
assert_equals(len(statistic.dtype), D)
statistic = query.posterior_predictive_statistic(
q, latents, prng, merge='mode')
assert_equals(statistic.shape, (1,))
assert_equals(len(statistic.dtype), D)
statistic = query.posterior_predictive_statistic(
q, latents, prng, merge=['mode', 'mode', 'avg', 'avg'])
assert_equals(statistic.shape, (1,))
assert_equals(len(statistic.dtype), D)
q = ma.masked_array(
np.array([(False,) * D] * 3, dtype=[('', bool)] * D),
mask=[(False,) * (D / 2) + (True,) * (D / 2)] * 3)
statistic = query.posterior_predictive_statistic(q, latents, prng)
assert_equals(statistic.shape, (3,))
assert_equals(len(statistic.dtype), D)
def alias(t,fm,p):
"""
Evaluate the Bayes Ratio between signal with P and 0.5*P
Parameters
----------
t : Time series
fm : Flux series
p : Parameter dictionary.
"""
pA = copy.deepcopy(p)
pA['P'] = 0.5 * pA['P']
resL = LDTwrap(t,fm,pA)
res = np.hstack(resL)
# Masked array corresponding to P = 2 P
tfold = getT(res['tdt'],pA['P'],pA['epoch'],pA['tdur'])
tT = ma.masked_array(res['tdt'],copy=True,mask=tfold.mask)
fT = ma.masked_array(res['fdt'],copy=True,mask=tfold.mask)
X2 = lambda par : ma.sum( (fT - keptoy.P05(pd2a(par),tT))**2 )
return X2(p),X2(pA)
def trgvsref(troot, rdir, rroot, mdir, mroot, wls, srcs, phasetag='median'):
if phasetag is 'mean':
print 'Phasetag:', phasetag
phasetag = np.mean
else:
print 'Phasetag:', phasetag
phasetag = np.median
for s in srcs:
mname = os.path.join(mdir, '{0}_s{1}.npy'.format(mroot, s))
m = np.load(mname)
for w in wls:
tname = '{0}_wl{1}_s{2}_phi.npy'.format(troot, w, s)
t = np.load(tname)
tm = ma.masked_array(t, mask=m)
rname = os.path.join(rdir, '{0}_wl{1}_s{2}_phi.npy'.format(rroot, w, s))
r = np.load(rname)
rm = ma.masked_array(r, mask=m)
d = phasetag(tm-rm)
if d > np.pi:
print 'w:', w, ' s:', s, '-2PI'
t = t - 2.0*np.pi
os.rename(tname, '{0}_wrapped'.format(tname))
np.save(tname, t)
elif d < -np.pi:
print 'w:', w, ' s:', s, '+2PI'
t = t + 2.0*np.pi
os.rename(tname, '{0}_wrapped'.format(tname))
np.save(tname, t)
def plot_result2D(data, theory, view='linear'):
import matplotlib.pyplot as plt
from numpy.ma import masked_array, masked
#print "not a number",sum(np.isnan(data.data))
#data.data[data.data<0.05] = 0.5
mdata = masked_array(data.data, data.mask)
mdata[np.isnan(mdata)] = masked
if view is 'log':
mdata[mdata <= 0] = masked
mdata = np.log10(mdata)
mtheory = masked_array(np.log10(theory), mdata.mask)
else:
mtheory = masked_array(theory, mdata.mask)
mresid = masked_array((theory-data.data)/data.err_data, data.mask)
vmin = min(mdata.min(), mtheory.min())
vmax = max(mdata.max(), mtheory.max())
print np.exp(np.mean(mtheory)), np.std(mtheory),np.max(mtheory),np.min(mtheory)
plt.subplot(1, 3, 1)
plot_data(data, mdata, vmin=vmin, vmax=vmax)
plt.colorbar()
plt.subplot(1, 3, 2)
plot_data(data, mtheory, vmin=vmin, vmax=vmax)
plt.colorbar()
plt.subplot(1, 3, 3)
print abs(mresid).max()
plot_data(data, mresid)
plt.colorbar()
def getCountyVar(self, var):
nyears, ncounties, nper = len(self.year), len(self.county), len(self.per)
nyears1, nyears2 = len(self.cp1.year), len(self.cp2.year)
v1 = self.cp1.getCountyVar(var)
v2 = self.cp2.getCountyVar(var)
# harmonize along county
varr = masked_array(zeros((nyears1 + nyears2, ncounties, nper)), mask = ones((nyears1 + nyears2, ncounties, nper)))
for i in range(ncounties):
c = self.county[i]
if c in self.cp1.county:
idx = where(self.cp1.county == c)[0][0]
varr[: nyears1, i] = v1[:, idx]
if c in self.cp2.county:
idx = where(self.cp2.county == c)[0][0]
varr[nyears1 :, i] = v2[:, idx]
if self.crop == 'wheat.winter':
newvarr = masked_array(zeros((nyears, ncounties, nper)), mask = ones((nyears, ncounties, nper)))
newvarr[: nyears1 + nyears2] = varr
varr = newvarr
return varr
def _detect_outliers(self, xs, ys, outs, degree=2):
xs = array(xs)
ys = array(ys)
mxs = masked_array(xs, mask=outs)
# print 's', sum(mxs), outs
mys = masked_array(ys, mask=outs)
o = OLS(mxs, mys, fitdegree=degree)
coeffs = o.get_coefficients()
n = len(xs) - sum(outs)
# coeff_errs = o.get_coefficient_standard_errors()
# ymean = ys.mean()
yeval = polyval(coeffs, xs)
# calculate detection_tol. use error of fit
devs = abs(ys - yeval)
ssr = sum(devs ** 2)
detection_tol = 2.5 * (ssr / ((n) - (degree))) ** 0.5
for i, xi, ys, di, mi in zip(xrange(len(xs)), xs, ys, devs, outs):
if di > detection_tol:
outs[i] = 1
omit = 'OK' if di <= detection_tol and not mi else 'User omitted'
# print xi, ys, di, detection_tol, omit, mi
return outs
def setUp(self):
# Sets up some useful arrays for use with the land/sea mask
# decompression.
self.land = np.array([[0, 1, 0, 0],
[1, 0, 0, 0],
[0, 0, 0, 1]], dtype=np.float64)
sea = ~self.land.astype(np.bool)
self.land_masked_data = np.array([1, 3, 4.5])
self.sea_masked_data = np.array([1, 3, 4.5, -4, 5, 0, 1, 2, 3])
# Compute the decompressed land mask data.
self.decomp_land_data = ma.masked_array([[0, 1, 0, 0],
[3, 0, 0, 0],
[0, 0, 0, 4.5]],
mask=sea,
dtype=np.float64)
# Compute the decompressed sea mask data.
self.decomp_sea_data = ma.masked_array([[1, -10, 3, 4.5],
[-10, -4, 5, 0],
[1, 2, 3, -10]],
mask=self.land,
dtype=np.float64)
self.land_mask = mock.Mock(data=self.land,
lbrow=self.land.shape[0],
lbnpt=self.land.shape[1])
def set_gen3data(self, Apaths, Ppaths, Xdet, Zdet, omega, mask=None):
self.omega = omega
self.wbyv = omega/self.v
self.amshape = mask.shape
self.mask = mask
for Apath, Ppath in zip(Apaths, Ppaths):
# read the data
Amp = bp.io.read_frame(Apath)
Pha = bp.io.read_frame(Ppath)
# mask all data arrays
Amp = ma.compressed(ma.masked_array(Amp, self.mask))
Pha = ma.compressed(ma.masked_array(Pha, self.mask))
try:
Amps = np.vstack((Amps, Amp))
Phas = np.vstack((Phas, Pha))
except:
Amps = Amp
Phas = Pha
self.Amps = Amps
self.Phas = Phas
# masked detector positions
self.dIdx = np.arange(len(Xdet))
self.dIdx = ma.compressed(ma.masked_array(self.dIdx.reshape(self.amshape), self.mask))
self.Xdet = ma.compressed(ma.masked_array(Xdet.reshape(self.amshape), self.mask))
self.Zdet = ma.compressed(ma.masked_array(Zdet.reshape(self.amshape), self.mask))
def make_full(flt):
"""
Adds top and bottom layers to array fld. This is intended for 3D ROMS data
fields that are on the vertical rho grid, and where we want (typically for
plotting purposes) to extend this in a smart way to the sea floor and the
sea surface.
Input:
flt is a tuple with either 1 ndarray (fld_mid),
or 3 ndarrays (fld_bot, fld_mid, fld_top)
Output:
fld is the "full" field
"""
if len(flt)==3:
fld = np.concatenate(flt, axis=0)
elif len(flt)==1:
fld_mid = flt[0]
N, M, L = fld_mid.shape
fld_bot = fld_mid[0].copy()
fld_bot = ma.masked_array(fld_bot.reshape(1, M, L).copy(), fld_mid[0]._mask)
fld_top = fld_mid[-1].copy()
fld_top = ma.masked_array(fld_top.reshape(1, M, L).copy(),fld_mid[-1]._mask)
fld = np.concatenate((fld_bot, fld_mid, fld_top), axis=0)
return fld
def set_gen3data(self, Apath, Ppath, Xdet, Zdet,
omega, mask_cutoff, toprowsmask=0, polygonmaskfile=None):
self.omega = omega
self.wbyv = omega/self.v
# read the data
self.Amp = bp.io.read_frame(Apath)
self.Pha = bp.io.read_frame(Ppath)
self.amshape = self.Amp.shape
# set the mask
self.mask = np.ones_like(self.Amp, dtype='bool')
self.mask[self.Amp > mask_cutoff*np.max(self.Amp)] = False
# mask top rows
if toprowsmask > 0:
toprowsmask = toprowsmask -1
self.mask[0:toprowsmask,:] = True
# mask all data arrays
self.Amp = ma.compressed((ma.masked_array(self.Amp, self.mask)))
self.Pha = ma.compressed(ma.masked_array(self.Pha, self.mask))
# masked detector positions
self.dIdx = np.arange(len(Xdet))
self.dIdx = ma.compressed(ma.masked_array(self.dIdx.reshape(self.amshape), self.mask))
self.Xdet = ma.compressed(ma.masked_array(Xdet.reshape(self.amshape), self.mask))
self.Zdet = ma.compressed(ma.masked_array(Zdet.reshape(self.amshape), self.mask))
def getZmv(a,mv):
"""
Helper for stdDat
Arguments:
* a: array of strings with the input data
* mv: string for missing values (e.g. 'x')
Returns:
* z: standardized masked array
"""
mascara=N.zeros(a.shape)
for i in range(a.shape[0]):
for j in range(a.shape[1]):
if a[i,j]==mv:
mascara[i,j]=1
a[i,j]=0
am=ma.masked_array(a,mask=mascara)
am=N.array(am,dtype=float)
z=N.copy(am)
z=(z-z.mean(axis=0))/z.std(axis=0)
z=ma.masked_array(z,dtype=str)
for i in range(mascara.shape[0]):
for j in range(mascara.shape[1]):
if mascara[i,j]==1:
z[i,j]='x'
return z
def segfitm(t,fm,bv):
"""
Segment fit masked
Parameters
----------
t : time
fm : flux for a particular segment
bv : vstack of basis vectors
"""
ncbv = bv.shape[0]
tm = ma.masked_array(t,copy=True,mask=fm.mask)
mask = fm.mask
bv = ma.masked_array(bv)
bv.mask = np.tile(mask, (ncbv,1) )
# Eliminate masked elements
tseg = tm.compressed()
fseg = fm.compressed()
bvseg = bv.compressed()
bvseg = bvseg.reshape(ncbv,bvseg.size/ncbv)
fdtseg,ffitseg,p1seg = segfit(tseg,fseg,bvseg)
return fdtseg,ffitseg
def pi(n = 1000, internal = False, draw = False):
'''Stochastic estimation of number "pi" based on Monte-Carlo method. '''
n = int(n)
radius = 1.0
try:
r = uniform(low = -radius, high = radius, size= (n,2))
inside = power(radius,2) >= power(r,2).sum(1)
nk = inside.sum() # number of hits inside the circle
if draw:
x = masked_array(r[:,0:1], mask=inside)
y = masked_array(r[:,1:2], mask=inside)
hold(True)
plot(x.compressed(), y.compressed(),'.b')
x = masked_array(r[:,0:1], mask=~inside)
y = masked_array(r[:,1:2], mask=~inside)
plot(x.compressed(), y.compressed(),'.r')
except MemoryError: # divide and conquer
print '** reducing sample count to %d' % int(n/100)
(n, nk) = array([ pi(n = n / 1E2, internal = True ) for i in range(int(1E2)) ]).sum(0)
if internal:
return (n, nk)
if draw: show()
p = 4.0 * nk / n
print 'nk = %d, n = %d, pi = 4 nk / n = %f' % (nk, n, p)
return p
def getVar(self, years):
# interpolate to common times and counties
frac = self.census.getFrac(years)
frac = self.interpolateCounty(frac, self.census.counties, self.counties)
# interpolate to common times
yldsum = self.interpolateTime(self.yldsum, self.ysum.years, years)
yldirr = self.interpolateTime(self.yldirr, self.yirr.years, years, fillgaps = False) # no gap filling!
hvtsum = self.interpolateTime(self.hvtsum, self.hsum.years, years)
hvtirr = self.interpolateTime(self.hvtirr, self.hirr.years, years, fillgaps = False)
# extrapolate irrigated yield, sum area, and irrigated area fraction
yldirr, delta = self.extrapolateYield(yldirr, yldsum, years)
hvtsum = self.extrapolateTime(hvtsum, years)
frac = self.extrapolateFrac(frac, hvtsum, hvtirr, years)
ny, nc, ni = len(years), len(self.counties), len(self.irr)
yld = masked_array(zeros((ny, nc, ni)), mask = ones((ny, nc, ni)))
hvt = masked_array(zeros((ny, nc, ni)), mask = ones((ny, nc, ni)))
for i in range(nc):
for j in range(ny):
hvt[j, i] = self.computeArea(hvtsum[j, i], hvtirr[j, i], frac[j, i])
yld[j, i] = self.computeYield(yldsum[j, i], yldirr[j, i], hvt[j, i], delta[j, i])
# extrapolate area in time
for j in range(ni):
area = hvt[:, i, j]
if isMaskedArray(area) and area.mask.any() and not area.mask.all():
hvt[:, i, j] = interp(years, years[~area.mask], area[~area.mask])
# convert
yld *= self.yldconv
hvt *= self.hvtconv
return yld, hvt
def cloud_statistics(file_name):
"""
Return core duration, minimum core base, maximum core height, mean core
mass, formation time, dissipation time, maximum depth, depth evolution and
corresponding times for tracked clouds.
Parameters
----------
file_name : netCDF file name
id_profile file for a tracked core with dimensions double t(t),
double z(z).
Return
------
tuple : cloud_id, lifetime, base, top, mass, l_min, l_max, depths,
max_depth, times
"""
# Read netCDF dataset
data = Dataset(file_name)
# Core ID
cloud_id = int(file_name[-11:-3])
# Core duration (seconds)
times = data.variables['t'][...]
lifetime = len(times)*mc.dt
# Formation time, dissipation time (seconds)
l_min = times.min()*mc.dt
l_max = times.max()*mc.dt
# Minimum core base, maximum core height, maximum depth, depth evolution
# (metres)
area = ma.masked_invalid(data.variables['AREA'][...])
z = data.variables['z'][...]
z = z*np.ones(np.shape(area))
z = ma.masked_array(z, ma.getmask(area))
bases = z.min(axis=1)
tops = z.max(axis=1)
depths = tops - bases + mc.dz
max_depth = depths.max()
base = bases.min()
top = tops.max()
# Mean core mass mass (kilograms)
qn = ma.masked_invalid(data.variables['QN'][...])
rho = ma.masked_invalid(data.variables['RHO'][...])
mass = np.mean(np.sum(area*rho*mc.dz, axis=1))
# Remove missing values
times = ma.masked_array(times, ma.getmask(depths))
depths = depths[~depths.mask]
times = times[~times.mask]
data.close()
return cloud_id, lifetime, base, top, mass, l_min, l_max, depths, \
max_depth, times
def test_delete_masked_points():
"""Test deleting masked points."""
a = ma.masked_array(np.arange(5), mask=[False, True, False, False, False])
b = ma.masked_array(np.arange(5), mask=[False, False, False, True, False])
expected = np.array([0, 2, 4])
a, b = _delete_masked_points(a, b)
assert_array_equal(a, expected)
assert_array_equal(b, expected)
def fromtextfile(fname, delimitor=None, commentchar='#', missingchar='',
varnames=None, vartypes=None):
"""Creates a mrecarray from data stored in the file `filename`.
Parameters
----------
filename : {file name/handle}
Handle of an opened file.
delimitor : {None, string}, optional
Alphanumeric character used to separate columns in the file.
If None, any (group of) white spacestring(s) will be used.
commentchar : {'#', string}, optional
Alphanumeric character used to mark the start of a comment.
missingchar : {'', string}, optional
String indicating missing data, and used to create the masks.
varnames : {None, sequence}, optional
Sequence of the variable names. If None, a list will be created from
the first non empty line of the file.
vartypes : {None, sequence}, optional
Sequence of the variables dtypes. If None, it will be estimated from
the first non-commented line.
Ultra simple: the varnames are in the header, one line"""
# Try to open the file ......................
f = openfile(fname)
# Get the first non-empty line as the varnames
while True:
line = f.readline()
firstline = line[:line.find(commentchar)].strip()
_varnames = firstline.split(delimitor)
if len(_varnames) > 1:
break
if varnames is None:
varnames = _varnames
# Get the data ..............................
_variables = masked_array([line.strip().split(delimitor) for line in f
if line[0] != commentchar and len(line) > 1])
(_, nfields) = _variables.shape
# Try to guess the dtype ....................
if vartypes is None:
vartypes = _guessvartypes(_variables[0])
else:
vartypes = [np.dtype(v) for v in vartypes]
if len(vartypes) != nfields:
msg = "Attempting to %i dtypes for %i fields!"
msg += " Reverting to default."
warnings.warn(msg % (len(vartypes), nfields))
vartypes = _guessvartypes(_variables[0])
# Construct the descriptor ..................
mdescr = [(n, f) for (n, f) in zip(varnames, vartypes)]
mfillv = [ma.default_fill_value(f) for f in vartypes]
# Get the data and the mask .................
# We just need a list of masked_arrays. It's easier to create it like that:
_mask = (_variables.T == missingchar)
_datalist = [masked_array(a, mask=m, dtype=t, fill_value=f)
for (a, m, t, f) in zip(_variables.T, _mask, vartypes, mfillv)]
return fromarrays(_datalist, dtype=mdescr)
def test_masked_constant_not_in_place(self):
# Cube in_place arithmetic operation.
dtype = np.int64
dat = ma.masked_array(0, 1, dtype)
cube = Cube(dat)
res = self.cube_func(cube, 5, in_place=False)
self.assertMaskedArrayEqual(ma.masked_array(0, 1), res.data)
self.assertEqual(dtype, res.dtype)
self.assertIsNot(res, cube)
def difference(param):
diff = []
param_ldr = ma.masked_array(param, mask=LU_ldr)
param_hdr = ma.masked_array(param, mask=LU_hdr)
param_com = ma.masked_array(param, mask=LU_com)
for i in range(len(param)):
diff.append(x[i] - y[i])
return diff
def principal_directions(img, sigma, H=None):
"""
will ignore calculation of principal directions of masked areas
despite the name, this function actually returns the theta corresponding to
leading and trailing principal directions, i.e. angle w / x axis
"""
if H is None:
H = hessian_matrix(img, sigma)
Hxx, Hxy, Hyy = H
# check if input image is masked
try:
mask = img.mask
except AttributeError:
masked = False
else:
masked = True
dims = img.shape
# where to store
trailing_thetas = np.zeros_like(img)
leading_thetas = np.zeros_like(img)
# maybe implement a small angle correction
for i, (xx, xy, yy) in enumerate(np.nditer([Hxx, Hxy, Hyy])):
# grab the (x,y) coordinate of the hxx, hxy, hyy you're using
subs = np.unravel_index(i, dims)
# ignore masked areas (if masked array)
if masked and img.mask[subs]:
continue
h = np.array([[xx, xy], [xy, yy]]) # per-pixel hessian
l, v = eig(h) # eigenvectors as columns
# reorder eigenvectors by (increasing) magnitude of eigenvalues
v = v[:,np.argsort(np.abs(l))]
# angle between each eigenvector and positive x-axis
# arccos of first element (dot product with (1,0) and eigvec is already
# normalized)
trailing_thetas[subs] = np.arccos(v[0,0]) # first component of each
leading_thetas[subs] = np.arccos(v[0,1]) # first component of each
if masked:
leading_thetas = ma.masked_array(leading_thetas, mask)
trailing_thetas = ma.masked_array(trailing_thetas, mask)
return trailing_thetas, leading_thetas
def get_error_over_prec(len_var,var_names,signs,dirs,error):
for idx in np.arange(0,len_var):
data = get_data(dirs[idx])
mask = ma.getmaskarray(data)
if var_names[idx] == 'PREC':
Prec = signs[idx]*ma.masked_array(data,mask)
error+=data*signs[idx]
error_m = ma.masked_array(error,mask)
error_over_Prec = error_m/Prec
return (error_m[0],error_over_Prec[0])
def __init__(self, x, y):
x = masked_array(x, copy=False, subok=True, dtype=float_, order="F").ravel()
y = masked_array(y, copy=False, subok=True, dtype=float_, order="F").ravel()
if x.size != y.size:
msg = "Incompatible size between observations (%s) and response (%s)!"
raise ValueError(msg % (x.size, y.size))
idx = x.argsort()
self._x = x[idx]
self._y = y[idx]
self._mask = mask_or(self._x._mask, self._y._mask, copy=False)
def _numpy2ma(self, inarray, reason_n=None):
"""Convert input numpy array to masked array
"""
if reason_n==None:
outarray=ma.masked_array(inarray, mask=np.zeros(self.reason_n.shape) )
outarray.mask[self.reason_n!=0]=1
else:
outarray=ma.masked_array(inarray, mask=np.zeros(reason_n.shape) )
outarray.mask[reason_n!=0]=1
return outarray
def get_error_extreme(len_var,var_names,signs,dirs,error):
for idx in np.arange(0,len_var):
data = get_data(dirs[idx])
mask = ma.getmaskarray(data)
if var_names[idx] == 'PREC':
Prec = signs[idx]*ma.masked_array(data,mask)
error+=data*signs[idx]
error_m = ma.masked_array(error,mask)
error_over_Prec = error_m/Prec
return np.unravel_index(error_m[0].argmax(),error_m[0].shape)
def test_GIVEN_masked_entries_for_lats_and_lons_THEN_summary_does_not_contain_elements(self):
latitudes = ma.masked_array([3, 5.1, 1.1, 6.5], [False, True, False, False])
longitudes = ma.masked_array([0, 120, -120, 100], [False, False, True, False])
gen = GeoJSONGenerator(latitudes=latitudes, longitudes=longitudes)
geojson = gen.get_elasticsearch_geojson()
for result, expected in zip(geojson["geometries"]["display"]["coordinates"], [(0, 3.0), (100, 6.5)]):
assert_that(result[0], close_to(expected[0], 0.01), "lat" )
assert_that(result[1], close_to(expected[1], 0.01), "lon")
def __init__(self, n):
self._fval = masked_array(empty((n,), dtype=float_, order='F'))
self._rw = masked_array(empty((n,), dtype=float_, order='F'))
self._fres = masked_array(empty((n,), dtype=float_, order='F'))
def __init__(self, y):
self.y = masked_array(y, subok=True, copy=False).ravel()
self._mask = self.y._mask
if self._mask.any():
raise ValueError("Masked arrays should be filled first!")
self.y_eff = self.y.compressed()
def __init__(self, n):
self._seasonal = masked_array(empty((n,), float_))
self._trend = masked_array(empty((n,), float_))
self._weights = masked_array(empty((n,), float_))
self._residuals = masked_array(empty((n,), float_))
def main():
"""Run the main routine for the script."""
# Define the limits to plot in the various stellar parameters.
temp_lims = (5400, 6300) * u.K
mtl_lims = (-0.75, 0.45)
logg_lims = (4.1, 4.6)
tqdm.write('Unpickling transitions list..')
with open(vcl.final_selection_file, 'r+b') as f:
transitions_list = pickle.load(f)
vprint(f'Found {len(transitions_list)} transitions.')
# Define the model to use.
if args.linear:
model_func = fit.linear_model
elif args.quadratic:
model_func = fit.quadratic_model
elif args.cross_term:
model_func = fit.cross_term_model
elif args.quadratic_magnitude:
model_func = fit.quadratic_mag_model
# model_func = fit.quadratic_model
model_name = '_'.join(model_func.__name__.split('_')[:-1])
tqdm.write(f'Using {model_name} model.')
db_file = vcl.databases_dir / f'stellar_db_{model_name}_params.hdf5'
# Load data from HDF5 database file.
tqdm.write('Reading data from stellar database file...')
star_transition_offsets = u.unyt_array.from_hdf5(
db_file, dataset_name='star_transition_offsets')
star_transition_offsets_EotWM = u.unyt_array.from_hdf5(
db_file, dataset_name='star_transition_offsets_EotWM')
star_transition_offsets_EotM = u.unyt_array.from_hdf5(
db_file, dataset_name='star_transition_offsets_EotM')
star_temperatures = u.unyt_array.from_hdf5(
db_file, dataset_name='star_temperatures')
with h5py.File(db_file, mode='r') as f:
star_metallicities = hickle.load(f, path='/star_metallicities')
star_magnitudes = hickle.load(f, path='/star_magnitudes')
star_gravities = hickle.load(f, path='/star_gravities')
column_dict = hickle.load(f, path='/transition_column_index')
star_names = hickle.load(f, path='/star_row_index')
# Handle various fitting and plotting setup:
eras = {'pre': 0, 'post': 1}
param_dict = {'temp': 0, 'mtl': 1, 'logg': 2}
plot_types = ('temp', 'mtl', 'logg')
params_list = []
# Figure out how many parameters the model function takes, so we know how
# many to dynamically give it later.
num_params = len(signature(model_func).parameters)
for i in range(num_params - 1):
params_list.append(0.)
# Set up the figure with subplots.
comp_fig, axes_dict = create_comparison_figure(ylims=None)
if args.label:
labels = (args.label, )
else:
# labels = ['4219.893V1_16', '4490.998Fe1_25', '4492.660Fe2_25',
# '4500.398Fe1_25', '4589.484Cr2_28', '4653.460Cr1_29',]
# '4738.098Fe1_32', '4767.190Mn1_33', '4811.877Zn1_34',
# '4940.192Fe1_37', '5138.510Ni1_42', '5178.000Ni1_43',
# '5200.158Fe1_43', '5571.164Fe1_50', '5577.637Fe1_50',
# '6067.161Fe1_59', '6123.910Ca1_60', '6144.183Si1_60',
# '6155.928Na1_61', '6162.452Na1_61', '6178.520Ni1_61',
# '6192.900Ni1_61']
labels = []
for transition in tqdm(transitions_list):
for order_num in transition.ordersToFitIn:
label = '_'.join([transition.label, str(order_num)])
labels.append(label)
tqdm.write(f'Analyzing {len(labels)} transitions.')
if not args.nbins:
bin_dict = {}
# Set bins manually.
# bin_dict[name] = np.linspace(5457, 6257, 5)
bin_dict['temp'] = [
5377, 5477, 5577, 5677, 5777, 5877, 5977, 6077, 6177, 6277
]
# bin_dict[name] = np.linspace(-0.75, 0.45, 5)
bin_dict['mtl'] = [-0.75, -0.6, -0.45, -0.3, -0.15, 0, 0.15, 0.3, 0.45]
# bin_dict[name] = np.linspace(4.1, 4.6, 5)
bin_dict['logg'] = [4.04, 4.14, 4.24, 4.34, 4.44, 4.54, 4.64]
for time in eras.keys():
for plot_type, lims in zip(plot_types,
(temp_lims, mtl_lims, logg_lims)):
ax = axes_dict[f'{plot_type}_{time}']
for limit in bin_dict[plot_type]:
ax.axvline(x=limit, color='Green', alpha=0.6, zorder=1)
# Create an array to store all the individual sigma_sys values in in order
# to get the means and STDs for each bin.
row_len = len(labels)
temp_col_len = len(bin_dict['temp']) - 1
metal_col_len = len(bin_dict['mtl']) - 1
logg_col_len = len(bin_dict['logg']) - 1
# First axis is for pre- and post- fiber change values: 0 = pre, 1 = post
temp_array = np.full([2, row_len, temp_col_len], np.nan)
metal_array = np.full([2, row_len, metal_col_len], np.nan)
logg_array = np.full([2, row_len, logg_col_len], np.nan)
full_arrays_dict = {
key: value
for key, value in zip(plot_types, (temp_array, metal_array,
logg_array))
}
for label_num, label in tqdm(enumerate(labels), total=len(labels)):
vprint(f'Analyzing {label}...')
# The column number to use for this transition:
try:
col = column_dict[label]
except KeyError:
print(f'Incorrect key given: {label}')
sys.exit(1)
for time in tqdm(eras.keys()):
vprint(20 * '=')
vprint(f'Working on {time}-change era.')
mean = np.nanmean(star_transition_offsets[eras[time], :, col])
# First, create a masked version to catch any missing entries:
m_offsets = ma.masked_invalid(
star_transition_offsets[eras[time], :, col])
m_offsets = m_offsets.reshape([len(m_offsets), 1])
# Then create a new array from the non-masked data:
offsets = u.unyt_array(m_offsets[~m_offsets.mask], units=u.m / u.s)
vprint(f'Median of offsets is {np.nanmedian(offsets)}')
m_eotwms = ma.masked_invalid(
star_transition_offsets_EotWM[eras[time], :, col])
m_eotwms = m_eotwms.reshape([len(m_eotwms), 1])
eotwms = u.unyt_array(m_eotwms[~m_eotwms.mask], units=u.m / u.s)
m_eotms = ma.masked_invalid(
star_transition_offsets_EotM[eras[time], :, col])
m_eotms = m_eotms.reshape([len(m_eotms), 1])
# Use the same mask as for the offsets.
eotms = u.unyt_array(m_eotms[~m_offsets.mask], units=u.m / u.s)
# Create an error array which uses the greater of the error on
# the mean or the error on the weighted mean.
err_array = np.maximum(eotwms, eotms)
vprint(f'Mean is {np.mean(offsets)}')
weighted_mean = np.average(offsets, weights=err_array**-2)
vprint(f'Weighted mean is {weighted_mean}')
# Mask the various stellar parameter arrays with the same mask
# so that everything stays in sync.
temperatures = ma.masked_array(star_temperatures)
temps = temperatures[~m_offsets.mask]
metallicities = ma.masked_array(star_metallicities)
metals = metallicities[~m_offsets.mask]
magnitudes = ma.masked_array(star_magnitudes)
mags = magnitudes[~m_offsets.mask]
gravities = ma.masked_array(star_gravities)
loggs = gravities[~m_offsets.mask]
stars = ma.masked_array([key for key in star_names.keys()
]).reshape(len(star_names.keys()), 1)
names = stars[~m_offsets.mask]
# Stack the stellar parameters into vertical slices
# for passing to model functions.
x_data = np.stack((temps, metals, loggs), axis=0)
# Create the parameter list for this run of fitting.
params_list[0] = float(mean)
beta0 = tuple(params_list)
vprint(beta0)
# Iterate over binned segments of the data to find what additional
# systematic error is needed to get a chi^2 of ~1.
arrays_dict = {
name: array
for name, array in zip(plot_types, (temps, metals, loggs))
}
popt, pcov = curve_fit(model_func,
x_data,
offsets.value,
sigma=err_array.value,
p0=beta0,
absolute_sigma=True,
method='lm',
maxfev=10000)
model_values = model_func(x_data, *popt)
residuals = offsets.value - model_values
if args.nbins:
nbins = int(args.nbins)
# Use quantiles to get bins with the same number of elements
# in them.
vprint(f'Generating {args.nbins} bins.')
bins = np.quantile(arrays_dict[name],
np.linspace(0, 1, nbins + 1),
interpolation='nearest')
bin_dict[name] = bins
min_bin_size = 7
sigma_sys_dict = {}
num_params = 1
for name in tqdm(plot_types):
sigma_sys_list = []
sigma_list = []
bin_mid_list = []
bin_num = -1
for bin_lims in pairwise(bin_dict[name]):
bin_num += 1
lower, upper = bin_lims
bin_mid_list.append((lower + upper) / 2)
mask_array = ma.masked_outside(arrays_dict[name],
*bin_lims)
num_points = mask_array.count()
vprint(f'{num_points} values in bin ({lower},{upper})')
if num_points < min_bin_size:
vprint('Skipping this bin!')
sigma_list.append(np.nan)
sigma_sys_list.append(np.nan)
continue
temps_copy = temps[~mask_array.mask]
metals_copy = metals[~mask_array.mask]
mags_copy = mags[~mask_array.mask]
residuals_copy = residuals[~mask_array.mask]
errs_copy = err_array[~mask_array.mask].value
x_data_copy = np.stack(
(temps_copy, metals_copy, mags_copy), axis=0)
chi_squared_nu = fit.calc_chi_squared_nu(
residuals_copy, errs_copy, num_params)
sigma_sys_delta = 0.01
sigma_sys = -sigma_sys_delta
chi_squared_nu = np.inf
variances = np.square(errs_copy)
while chi_squared_nu > 1.0:
sigma_sys += sigma_sys_delta
variance_sys = np.square(sigma_sys)
variances_iter = variances + variance_sys
# err_iter = np.sqrt(np.square(errs_copy) +
# np.square(sigma_sys))
weights = 1 / variances_iter
wmean, sum_weights = np.average(residuals_copy,
weights=weights,
returned=True)
chi_squared_nu = fit.calc_chi_squared_nu(
residuals_copy - wmean, np.sqrt(variances_iter),
num_params)
sigma_sys_list.append(sigma_sys)
sigma = np.std(residuals_copy)
sigma_list.append(sigma)
# tqdm.write(f'sigma_sys is {sigma_sys:.3f}')
# tqdm.write(f'chi^2_nu is {chi_squared_nu}')
if sigma_sys / sigma > 1.2:
print('---')
print(bin_lims)
print(mask_array)
print(metals)
print(residuals)
print(n_params)
print(num_params)
print(residuals_copy)
print(errs_copy)
print(sigma)
print(sigma_sys)
sys.exit()
# Store the result in the appropriate full array.
full_arrays_dict[name][eras[time], label_num,
bin_num] = sigma_sys
sigma_sys_dict[f'{name}_sigma_sys'] = sigma_sys_list
sigma_sys_dict[f'{name}_sigma'] = sigma_list
sigma_sys_dict[f'{name}_bin_mids'] = bin_mid_list
# sigma = np.nanstd(residuals)
for plot_type, lims in zip(plot_types,
(temp_lims, mtl_lims, logg_lims)):
ax = axes_dict[f'{plot_type}_{time}']
ax.plot(sigma_sys_dict[f'{plot_type}_bin_mids'],
sigma_sys_dict[f'{plot_type}_sigma_sys'],
color='Black',
alpha=0.15,
zorder=2)
# label=r'$\sigma_\mathrm{sys}$')
# ax.plot(sigma_sys_dict[f'{plot_type}_bin_mids'],
# sigma_sys_dict[f'{plot_type}_sigma'],
# color='Blue', alpha=0.3,
# label=r'$\sigma$')
if args.label:
ax.legend()
# ax.annotate(r'$\sigma_\mathrm{sys}$:'
# f' {sys_err:.2f}',
# (0.01, 0.99),
# xycoords='axes fraction',
# verticalalignment='top')
# ax.annotate(fr'$\chi^2_\nu$: {chi_squared_nu.value:.4f}'
# '\n'
# fr'$\sigma$: {sigma:.2f}',
# (0.99, 0.99),
# xycoords='axes fraction',
# horizontalalignment='right',
# verticalalignment='top')
# data = np.array(ma.masked_invalid(residuals).compressed())
for time in eras.keys():
for name in plot_types:
ax = axes_dict[f'{name}_{time}']
means = []
stds = []
arr = full_arrays_dict[name]
for i in range(0, np.size(arr, 2)):
means.append(np.nanmean(arr[eras[time], :, i]))
stds.append(np.nanstd(arr[eras[time], :, i]))
ax.errorbar(sigma_sys_dict[f'{name}_bin_mids'],
means,
yerr=stds,
color='Red',
alpha=1,
marker='o',
markersize=4,
capsize=4,
elinewidth=2,
zorder=3,
label='Mean and stddev')
ax.legend()
plot_path = Path('/Users/dberke/Pictures/'
f'sigma_sys_stellar_parameter_dependance')
if args.label:
file_name = plot_path / f'{args.label}.png'
elif args.nbins:
file_name = plot_path / f'Combined_{model_name}_quantiles.png'
else:
file_name = plot_path / f'Combined_{model_name}_fixed_bins.png'
# plt.show()
comp_fig.savefig(str(file_name))
def main():
import argparse
parser = argparse.ArgumentParser(
description='This script performs the T-test for \
CICE simulations that should be bit-for-bit, but are not.'
)
parser.add_argument('base_dir', \
help='Path to the baseline history (iceh_inst*) files. REQUIRED')
parser.add_argument('test_dir', \
help='Path to the test history (iceh_inst*) files. REQUIRED')
parser.add_argument('-v', '--verbose', dest='verbose', help='Print debug output?', \
action='store_true')
parser.add_argument('-pt',
'--plot_type',
dest='plot_type',
help='Specify type of plot \
to create',
choices=['scatter', 'contour', 'pcolor'])
parser.set_defaults(verbose=False)
parser.set_defaults(plot_type='pcolor')
# If no arguments are provided, print the help message
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
args = parser.parse_args()
# Set up the logger
global logger
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
# Log to log file as well as stdout
fh = logging.FileHandler(r'qc_log.txt', 'w')
logger = logging.getLogger(__name__)
logger.addHandler(fh)
logger.info('Running QC test on the following directories:')
logger.info(' {}'.format(args.base_dir))
logger.info(' {}'.format(args.test_dir))
dir_a, dir_b, files_base, files_test = gen_filenames(
args.base_dir, args.test_dir)
nfiles = len(files_base)
nlon, nlat, t_lat, t_lon = get_geom(dir_a, files_base[0])
data_base, data_test, data_diff = read_data(dir_a, dir_b, files_base,
files_test, nlon, nlat)
if np.ma.all(data_diff.mask):
logger.info("Data is bit-for-bit. No need to run QC test")
sys.exit(0)
# Run the two-stage test
PASSED, H1_array = two_stage_test(data_base, nfiles, data_diff,
files_base[0], dir_a)
# Delete arrays that are no longer necessary
del data_diff
# If test failed, attempt to create a plot of the failure locations
if not PASSED:
plot_two_stage_failures(H1_array, t_lat, t_lon)
# Create plots of mean ice thickness
baseDir = os.path.abspath(args.base_dir).rstrip('history/').rstrip(\
'history').split('/')[-1]
testDir = os.path.abspath(args.test_dir).rstrip('history/').rstrip( \
'history').split('/')[-1]
plot_data(np.mean(data_base, axis=0), t_lat, t_lon, 'm', baseDir,
args.plot_type)
plot_data(np.mean(data_test, axis=0), t_lat, t_lon, 'm', testDir,
args.plot_type)
plot_data(np.mean(data_base-data_test,axis=0), t_lat, t_lon, 'm', '{}\n- {}'.\
format(baseDir,testDir), args.plot_type)
logger.error('Quality Control Test FAILED')
sys.exit(-1)
# Create a northern hemisphere and southern hemisphere mask
mask_tlat = t_lat < 0
mask_nh = np.zeros_like(data_base)
mask_sh = np.zeros_like(data_base)
for (a, b), val in np.ndenumerate(mask_tlat):
mask_nh[:, a, b] = val
mask_sh[:, a, b] = not val
# Run skill test on northern hemisphere
data_nh_a = ma.masked_array(data_base, mask=mask_nh)
data_nh_b = ma.masked_array(data_test, mask=mask_nh)
if np.ma.all(data_nh_a.mask) and np.ma.all(data_nh_b.mask):
logger.info("Northern Hemisphere data is bit-for-bit")
PASSED_NH = True
else:
PASSED_NH = skill_test(dir_a, files_base[0], data_nh_a, data_nh_b,
nfiles, 'Northern')
# Run skill test on southern hemisphere
data_sh_a = ma.masked_array(data_base, mask=mask_sh)
data_sh_b = ma.masked_array(data_test, mask=mask_sh)
if np.ma.all(data_sh_a.mask) and np.ma.all(data_sh_b.mask):
logger.info("Southern Hemisphere data is bit-for-bit")
PASSED_SH = True
else:
PASSED_SH = skill_test(dir_a, files_base[0], data_sh_a, data_sh_b,
nfiles, 'Southern')
PASSED_SKILL = PASSED_NH and PASSED_SH
# Plot the ice thickness data for the base and test cases
baseDir = os.path.abspath(args.base_dir).rstrip('history/').rstrip( \
'history').split('/')[-1]
testDir = os.path.abspath(args.test_dir).rstrip('history/').rstrip( \
'history').split('/')[-1]
plot_data(np.mean(data_base, axis=0), t_lat, t_lon, 'm', baseDir,
args.plot_type)
plot_data(np.mean(data_test, axis=0), t_lat, t_lon, 'm', testDir,
args.plot_type)
plot_data(np.mean(data_base-data_test,axis=0), t_lat, t_lon, 'm', '{}\n- {}'.\
format(baseDir,testDir), args.plot_type)
logger.info('')
if not PASSED_SKILL:
logger.error('Quality Control Test FAILED')
sys.exit(1) # exit with an error return code
else:
logger.info('Quality Control Test PASSED')
sys.exit(0) # exit with successfull return code
def wise_rgz_gurkan_lowsn():
plt.ion()
# WISE All-sky sample
#
filenames = [
'%s/%s.fits' % (rgz_dir, x)
for x in ('wise_allsky_2M', 'gurkan/gurkan_all', 'rgz_75_wise_16jan')
]
labels = ('WISE all-sky sources', 'Gurkan+14 radio galaxies',
'RGZ 75% radio galaxies')
print ''
for fname, label in zip(filenames, labels):
with fits.open(fname) as f:
d = f[1].data
# Restrict the RGZ-WISE matches to 75% consensus
if label == 'RGZ 75% radio galaxies':
rgz75 = d['ratio'] >= 0.75
snr_w1 = d['snr1'] >= wise_snr
snr_w2 = d['snr2'] >= wise_snr
snr_w3 = d['snr3'] >= wise_snr
d = d[np.logical_not(rgz75 & snr_w1 & snr_w2 & snr_w3)]
w1 = d['w1mpro']
w2 = d['w2mpro']
w3 = d['w3mpro']
w4 = d['w4mpro']
x = w2 - w3
y = w1 - w2
# AGN wedge is INCORRECTLY cited in Gurkan+14; check original Mateos+12 for numbers
#
wedge_lims = (y > -3.172 * x + 7.624) & (y > (0.315 * x - 0.222)) & (
y < (0.315 * x + 0.796))
#
# Very rough loci from Wright et al. (2010)
stars_lims = (x > 0) & (x < 1) & (y > 0.1) & (y < 0.4)
el_lims = (x > 0.5) & (x < 1.3) & (y > 0.) & (y < 0.2)
sp_lims = (x > 1.5) & (x < 3.0) & (y > 0.1) & (y < 0.4)
agn_frac = wedge_lims.sum() / float(len(d))
stars_frac = stars_lims.sum() / float(len(d))
el_frac = el_lims.sum() / float(len(d))
sp_frac = sp_lims.sum() / float(len(d))
print 'Fraction of %25s in AGN wedge: %4.1f percent' % (label,
agn_frac * 100)
print 'Fraction of %25s in stars locus: %4.1f percent' % (
label, stars_frac * 100)
print 'Fraction of %25s in elliptical locus: %4.1f percent' % (
label, el_frac * 100)
print 'Fraction of %25s in spiral locus: %4.1f percent' % (
label, sp_frac * 100)
print ''
print ''
# Bin data and look at differences?
#
with fits.open(filenames[0]) as f:
d = f[1].data
maglim_w1 = d['snr1'] > wise_snr
maglim_w2 = d['snr2'] > wise_snr
maglim_w3 = d['snr3'] < wise_snr
wise = d[maglim_w1 & maglim_w2 & maglim_w3]
with fits.open(filenames[2]) as f:
d = f[1].data
rgz75 = d['ratio'] >= 0.75
snr_w1 = d['snr1'] >= wise_snr
snr_w2 = d['snr2'] >= wise_snr
snr_w3 = d['snr3'] <= wise_snr
rgz = d[rgz75 & snr_w1 & snr_w2 & snr_w3]
xmin, xmax = -1, 6
ymin, ymax = -0.5, 3
bins_w2w3 = np.linspace(xmin, xmax, 40)
bins_w1w2 = np.linspace(ymin, ymax, 40)
hw, xedges, yedges = np.histogram2d(wise['w2mpro'] - wise['w3mpro'],
wise['w1mpro'] - wise['w2mpro'],
bins=(bins_w2w3, bins_w1w2))
hr, xedges, yedges = np.histogram2d(rgz['w2mpro'] - rgz['w3mpro'],
rgz['w1mpro'] - rgz['w2mpro'],
bins=(bins_w2w3, bins_w1w2))
fig = plt.figure(1, (9, 8))
fig.clf()
hw_norm = hw / float(np.max(hw))
hr_norm = hr / float(np.max(hr))
hw_norm_masked = ma.masked_array(hw, mask=(hw < 10))
hr_norm_masked = ma.masked_array(hr_norm, mask=(hr <= 10))
extent = [bins_w2w3[0], bins_w2w3[-1], bins_w1w2[0], bins_w1w2[-1]]
ax1 = fig.add_subplot(111, position=(0.10, 0.10, 0.75, 0.85))
# WISE all-sky
cmap = cm.YlOrRd
cmap.set_bad('w')
Z = hw_norm_masked
im1 = ax1.imshow(Z.T,
cmap=cmap,
alpha=1.0,
extent=extent,
interpolation='nearest',
origin='lower')
'''
fi = gaussian_filter(hw.T,0.5)
levels=np.linspace(10,20000,10)
CS = ax1.contour(bins_w2w3[1:],bins_w1w2[1:],fi,levels,colors='r',linewidths=1)
'''
# RGZ 75% catalog
fi = gaussian_filter(hr.T, 0.5)
levels = np.linspace(3, hr.max(), 10)
CS = ax1.contour(bins_w2w3[1:],
bins_w1w2[1:],
fi,
levels,
colors='b',
linewidths=1.5)
CS.collections[0].set_label('RGZ 75%')
# Gurkan
with fits.open(filenames[1]) as f:
gurkan = f[1].data
ax1.scatter(gurkan['w2mpro'] - gurkan['w3mpro'],
gurkan['w1mpro'] - gurkan['w2mpro'],
color='g',
s=10,
label='PRGs (Gurkan+14)')
xb, yb = 2.250, 0.487
xt, yt = 1.958, 1.413
xab = np.linspace(xb, 6, 100)
xat = np.linspace(xt, 6, 100)
xal = np.linspace(xb, xt, 100)
yab = 0.315 * xab - 0.222
yat = 0.315 * xat + 0.796
yal = -3.172 * xal + 7.624
ax1.plot(xab, yab, color='k', linestyle='--', label='AGN "wedge"')
ax1.plot(xat, yat, color='k', linestyle='--')
ax1.plot(xal, yal, color='k', linestyle='--')
ax1.set_xlabel(r'$(W2-W3)$', fontsize=20)
ax1.set_ylabel(r'$(W1-W2)$', fontsize=20)
ax1.set_xlim(xmin, xmax)
ax1.set_ylim(ymin, ymax)
ax1.set_aspect('auto')
cb_position = fig.add_axes([0.88, 0.1, 0.02, 0.85])
cb = plt.colorbar(im1, cax=cb_position, orientation='vertical')
cb.set_label('WISE all-sky sources', fontsize=16)
h, l = ax1.get_legend_handles_labels()
ax1.legend(h, l, loc='upper left', scatterpoints=2)
plt.show()
fig.savefig('%s/figures/wise_colorcolor_lowsn.eps' % paper_dir)
return None
def read_data(varName, latName, lonName, userVariables, filelist=None):
'''
Purpose::
Read gridded data into (t, lat, lon) arrays for processing
Inputs::
varName: a string representing the variable name to use from the file
latName: a string representing the latitude from the file's metadata
lonName: a string representing the longitude from the file's metadata
userVariables:
filelist (optional): a list of strings representing the filenames between the start and end dates provided
Returns:
Outputs::
A 3D masked array (t,lat,lon) with only the variables which meet the minimum temperature
criteria for each frame
Assumptions::
(1) All the files requested to extract data are from the same instrument/model, and thus have the same
metadata properties (varName, latName, lonName) as entered
(2) Assumes rectilinear grids for input datasets i.e. lat, lon will be 1D arrays
'''
global LAT
global LON
timeName = 'time'
filelistInstructions = userVariables.DIRS['CEoriDirName'] + '/*'
if filelist is None and userVariables.filelist is None:
userVariables.filelist = glob.glob(filelistInstructions)
userVariables.filelist.sort()
inputData = []
timelist = []
time2store = None
tempMaskedValueNp = []
nfiles = len(userVariables.filelist)
# Crash nicely if there are no netCDF files
if nfiles == 0:
print 'Error: no files in this directory! Exiting elegantly'
sys.exit()
else:
# Open the first file in the list to read in lats, lons and generate the grid for comparison
tmp = Dataset(userVariables.filelist[0], 'r+', format='NETCDF4')
alllatsraw = tmp.variables[latName][:]
alllonsraw = tmp.variables[lonName][:]
alllonsraw[alllonsraw > 180] = alllonsraw[
alllonsraw > 180] - 360. # convert to -180,180 if necessary
# Get the lat/lon info data (different resolution)
latminNETCDF = utils.find_nearest(alllatsraw,
float(userVariables.LATMIN))
latmaxNETCDF = utils.find_nearest(alllatsraw,
float(userVariables.LATMAX))
lonminNETCDF = utils.find_nearest(alllonsraw,
float(userVariables.LONMIN))
lonmaxNETCDF = utils.find_nearest(alllonsraw,
float(userVariables.LONMAX))
latminIndex = (np.where(alllatsraw == latminNETCDF))[0][0]
latmaxIndex = (np.where(alllatsraw == latmaxNETCDF))[0][0]
lonminIndex = (np.where(alllonsraw == lonminNETCDF))[0][0]
lonmaxIndex = (np.where(alllonsraw == lonmaxNETCDF))[0][0]
# Subsetting the data
latsraw = alllatsraw[latminIndex:latmaxIndex]
lonsraw = alllonsraw[lonminIndex:lonmaxIndex]
LON, LAT = np.meshgrid(lonsraw, latsraw)
latsraw = []
lonsraw = []
tmp.close()
for files in userVariables.filelist:
try:
thisFile = Dataset(files, 'r', format='NETCDF4')
# Clip the dataset according to user lat, lon coordinates
# Mask the data and fill with zeros for later
tempRaw = thisFile.variables[
varName][:, latminIndex:latmaxIndex,
lonminIndex:lonmaxIndex].astype('int16')
tempMask = ma.masked_array(tempRaw,
mask=(tempRaw > userVariables.T_BB_MAX),
fill_value=0)
# Get the actual values that the mask returned
# timeIndex, latIndex, lonIndex = index
tempMaskedValue = tempMask
tempMaskedValue[tempMask.mask] = 0
xtimes = thisFile.variables[timeName]
# Convert this time to a python datastring
time2store, _ = get_model_times(xtimes, timeName)
# Extend instead of append because get_model_times returns a list already and we don't
# want a list of list
timelist.extend(time2store)
inputData.extend(tempMaskedValue)
thisFile.close()
except:
print 'bad file! ', files
inputData = ma.array(inputData)
return inputData, timelist, LAT, LON, userVariables
tp = pd.read_csv('../simulations/id/{:03d}/totp.csv.bz2'.format(id),
header=None).as_matrix()
o2a = pd.read_csv('../simulations/id/{:03d}/O2abs.csv.bz2'.format(id),
header=None).as_matrix()
# o2r = pd.read_csv('../simulations/id/{:03d}/O2rel.csv.bz2'.format(id), header=None).as_matrix()
a[x1 - 1, x2 - 1, x3 - 1, x4 - 1, :, :, 0] = t
a[x1 - 1, x2 - 1, x3 - 1, x4 - 1, :, :, 1] = chl
a[x1 - 1, x2 - 1, x3 - 1, x4 - 1, :, :, 2] = tp
a[x1 - 1, x2 - 1, x3 - 1, x4 - 1, :, :, 3] = o2a
# a[x1-1, x2-1, x3-1, x4-1, :, :, 4] = o2r
for i, x1, x2, x3, x4, id in d.itertuples():
if not os.path.exists('../simulations/id/{:03d}/t.csv.bz2'.format(id)):
m[x1 - 1, x2 - 1, x3 - 1, x4 - 1, :, :, :] = True
a = ma.masked_array(a, mask=m)
## maximum chl in last year
r1 = a[:, :, :, :, (365 * 3 + 180):(365 * 3 + 270), 0, 1].mean(axis=4)
r1max = r1.max()
r1min = r1.min()
## number days surface anoxia in last year
r2 = (a[:, :, :, :, (365 * 3):, 80, 3] < 0.01).sum(axis=4)
r2max = r2.max()
r2min = r2.min()
cmg = plt.get_cmap('Greens')
normg = matplotlib.colors.Normalize(r1min, r1max, True)
cmr = plt.get_cmap('Reds')
def plot(self,
figure=None,
overlays=[],
colorbar=True,
vmin=None,
vmax=None,
linear=True,
showz=True,
yres=DEFAULT_YRES,
max_dist=None,
**matplotlib_args):
"""
Plot spectrogram onto figure.
Parameters
----------
figure : matplotlib.figure.Figure
Figure to plot the spectrogram on. If None, new Figure is created.
overlays : list
List of overlays (functions that receive figure and axes and return
new ones) to be applied after drawing.
colorbar : bool
Flag that determines whether or not to draw a colorbar. If existing
figure is passed, it is attempted to overdraw old colorbar.
vmin : float
Clip intensities lower than vmin before drawing.
vmax : float
Clip intensities higher than vmax before drawing.
linear : bool
If set to True, "stretch" image to make frequency axis linear.
showz : bool
If set to True, the value of the pixel that is hovered with the
mouse is shown in the bottom right corner.
yres : int or None
To be used in combination with linear=True. If None, sample the
image with half the minimum frequency delta. Else, sample the
image to be at most yres pixels in vertical dimension. Defaults
to 1080 because that's a common screen size.
max_dist : float or None
If not None, mask elements that are further than max_dist away
from actual data points (ie, frequencies that actually have data
from the receiver and are not just nearest-neighbour interpolated).
"""
# [] as default argument is okay here because it is only read.
# pylint: disable=W0102,R0914
if linear:
delt = yres
if delt is not None:
delt = max(
(self.freq_axis[0] - self.freq_axis[-1]) / (yres - 1),
_min_delt(self.freq_axis) / 2.)
delt = float(delt)
data = _LinearView(self.clip_values(vmin, vmax), delt)
freqs = np.arange(self.freq_axis[0], self.freq_axis[-1],
-data.delt)
else:
data = np.array(self.clip_values(vmin, vmax))
freqs = self.freq_axis
figure = plt.gcf()
if figure.axes:
axes = figure.axes[0]
else:
axes = figure.add_subplot(111)
params = {
'origin': 'lower',
'aspect': 'auto',
}
params.update(matplotlib_args)
if linear and max_dist is not None:
toplot = ma.masked_array(data, mask=data.make_mask(max_dist))
pass
else:
toplot = data
im = axes.imshow(toplot, **params)
xa = axes.get_xaxis()
ya = axes.get_yaxis()
xa.set_major_formatter(FuncFormatter(self.time_formatter))
if linear:
# Start with a number that is divisible by 5.
init = (self.freq_axis[0] % 5) / data.delt
nticks = 15.
# Calculate MHz difference between major ticks.
dist = (self.freq_axis[0] - self.freq_axis[-1]) / nticks
# Round to next multiple of 10, at least ten.
dist = max(round(dist, -1), 10)
# One pixel in image space is data.delt MHz, thus we can convert
# our distance between the major ticks into image space by dividing
# it by data.delt.
ya.set_major_locator(IndexLocator(dist / data.delt, init))
ya.set_minor_locator(IndexLocator(dist / data.delt / 10, init))
def freq_fmt(x, pos):
# This is necessary because matplotlib somehow tries to get
# the mid-point of the row, which we do not need here.
x = x + 0.5
return self.format_freq(self.freq_axis[0] - x * data.delt)
else:
freq_fmt = _list_formatter(freqs, self.format_freq)
ya.set_major_locator(MaxNLocator(integer=True, steps=[1, 5, 10]))
ya.set_major_formatter(FuncFormatter(freq_fmt))
axes.set_xlabel(self.t_label)
axes.set_ylabel(self.f_label)
# figure.suptitle(self.content)
figure.suptitle(' '.join([
get_day(self.start).strftime("%d %b %Y"),
'Radio flux density',
'(' + ', '.join(self.instruments) + ')',
]))
for tl in xa.get_ticklabels():
tl.set_fontsize(10)
tl.set_rotation(30)
figure.add_axes(axes)
figure.subplots_adjust(bottom=0.2)
figure.subplots_adjust(left=0.2)
if showz:
axes.format_coord = self._mk_format_coord(
data,
figure.gca().format_coord)
if colorbar:
if len(figure.axes) > 1:
Colorbar(figure.axes[1], im).set_label("Intensity")
else:
figure.colorbar(im).set_label("Intensity")
for overlay in overlays:
figure, axes = overlay(figure, axes)
for ax in figure.axes:
ax.autoscale()
if isinstance(figure, SpectroFigure):
figure._init(self, freqs)
return axes
def test_with_masked_constant(self):
masked_data = ma.masked_array([8], mask=True)
masked_constant = masked_data[0]
result = as_lazy_data(masked_constant)
self.assertIsInstance(result, da.core.Array)
# fig = matplotlib.pyplot.gcf()
# cax = matplotlib.pyplot.gca()
# fig.set_size_inches(20, 20)
# fig.subplots_adjust(right=0.93)
# cbar_ax = fig.add_axes([0.95, 0.15, 0.01, 0.7])
# _ = fig.colorbar(ax2, cax=cbar_ax)
# paf vectors for right elbow and right wrist
from numpy import ma
U = paf_avg[:, :, 16] * -1
V = paf_avg[:, :, 17]
X, Y = np.meshgrid(np.arange(U.shape[1]), np.arange(U.shape[0]))
M = np.zeros(U.shape, dtype='bool')
M[U**2 + V**2 < 0.5 * 0.5] = True
U = ma.masked_array(U, mask=M)
V = ma.masked_array(V, mask=M)
# 1
plt.figure()
# plt.imshow(oriImg[:,:,[2,1,0]], alpha = .5)
s = 5
Q = plt.quiver(X[::s, ::s],
Y[::s, ::s],
U[::s, ::s],
V[::s, ::s],
scale=50,
headaxislength=4,
alpha=.5,
width=0.001,
color='r')
add_var(outdata, 'rain', dims, values=climdata['rain'].filled(fill_value), atts=atts, fill_value=fill_value)
# 2m mean Temperature
atts = dict(long_name='Temperature at 2m', units='K')
add_var(outdata, 'T2', dims, values=climdata['T2'].filled(fill_value), atts=atts, fill_value=fill_value)
# 2m maximum Temperature
atts = dict(long_name='Maximum 2m Temperature', units='K')
add_var(outdata, 'Tmax', dims, values=climdata['Tmax'].filled(fill_value), atts=atts, fill_value=fill_value)
# 2m minimum Temperature
atts = dict(long_name='Minimum 2m Temperature', units='K')
add_var(outdata, 'Tmin', dims, values=climdata['Tmin'].filled(fill_value), atts=atts, fill_value=fill_value)
# 2m water vapor
atts = dict(long_name='Water Vapor Pressure at 2m', units='hPa')
add_var(outdata, 'Q2', dims, values=climdata['Q2'].filled(fill_value), atts=atts, fill_value=fill_value)
# land mask
atts = dict(long_name='Land Mask', units='')
tmp = ma.masked_array(ma.ones((datashape[1],datashape[2])), mask=dataMask)
add_var(outdata, 'landmask', ('lat','lon'), values=tmp.filled(0)) # create climatology variables
# for (key,value) in indata.iteritems():
# copy_vars(outdata, value, [key], namemap=varlist, copy_data=False, fill_value=) # , incl_=True
# outdata.variables[key][:,:,:] = climdata[key]
# ## dataset feedback and diagnostics
# # print dataset meta data
# print outdata
# # print dimensions meta data
# for dimobj in outdata.dimensions.values():
# print dimobj
# # print variable meta data
# for varobj in outdata.variables.values():
# print varobj
def famed(woa_oxyg_dh,
woa_oxyg_dh_m,
temp_cl,
temp_cl_m,
depth,
woa_rhom=None):
import numpy as np
from numpy import ma
# Main FAME computation ...
# =========================
# INPUTS FOR THE FAME CALCULATIONS
# woa_oxyg_dh == d18sw in seawater <time mean> shape is [102, 180, 360] e.g. depth, lat, lon
# woa_oxyg_dh_m == d18sw in seawater monthly shape is [12,57, 180, 360] e.g. depth, lat, lon
# temp_cl == temperature in seawater <time mean> shape is [1, 102, 180, 360] e.g. time (degenerate), depth, lat, lon
# temp_cl_m == temperature in seawater monthly shape is [12, 57, 180, 360] e.g. time , depth, lat, lon
# depth == depth of the levels in meters assumed positive here
# FAME is coded in Kelvins ...
temp_kl = temp_cl + 273.15
temp_kl_m = temp_cl_m + 273.15
# Computation of equilibrium calcite from WOA fields ...
delt_dh_init = delta_c_eq(temp_cl[...], woa_oxyg_dh)
delt_dh_init_m = delta_c_eq(temp_cl_m, woa_oxyg_dh_m)
#�Added the computation of the equation of Marchitto everywhere, no weighting.
#� Probably useless, only written for compatibility
#~ d18Oca_mar = ma.mean(delta_c_mar(temp_cl ,woa_oxyg_dh),axis=0)
# Actual calculation here
d18Oca_mar_m = ma.mean(delta_c_mar(temp_cl_m, woa_oxyg_dh_m), axis=0)
if not (woa_rhom is None):
# [DENSITY] -- addition of density from WOA
rho_m = woa_rhom
#endif
# i.e. auld method, d18Oca averaged over 50 meters ... == "Lukas Jonkers" methodology
depth_50m = find_closest(depth, 50.0)
depth_00m = find_closest(depth, 0.0)
indx_less_50m = depth <= 50.0
#~ if depth_50m == depth_00m : depth_50m += 1
# NOTA: all *_lj variables have a dimension without time and depth
# e.g. [180,360], lat, lon
d18Osw_lj = ma.mean(woa_oxyg_dh[indx_less_50m, ...], axis=0)
tempcl_lj = ma.mean(temp_cl[indx_less_50m, ...], axis=0)
d18Oca_lj = delta_c_eq(tempcl_lj, d18Osw_lj)
d18Osw_ol = woa_oxyg_dh[depth_00m, ...]
tempcl_ol = temp_cl[depth_00m, ...]
d18Oca_ol = delta_c_eq(tempcl_ol, d18Osw_ol)
if not (woa_rhom is None):
# [DENSITY] -- addition of density from WOA
rhom_ol = ma.mean(woa_rhom[:, depth_00m, ...], axis=0)
#endif
import forams_prod_l09 as fpl
# Maximum shape of result: nb_forams, lat, lon
max_shape_final = (len(fpl.l09_cnsts_dic), ) + d18Osw_ol.shape
# Create a placeholder for the foram result
delt_forams = ma.zeros(max_shape_final, np.float32)
if not (woa_rhom is None):
# [DENSITY] -- addition of density from WOA
rhom_forams = ma.zeros(max_shape_final, np.float32)
#endif
for foram_specie in fpl.l09_cnsts_dic:
# Rate of growth from the Lombard et al., 2009 methodology
foram_growth = fpl.growth_rate_l09_array(foram_specie, temp_kl[...])
foram_growth_m = fpl.growth_rate_l09_array(foram_specie, temp_kl_m)
# Get the depth of the STD living foram in FAME
f_dept = fpl.get_living_depth(foram_specie)
# Find this depth as an index in the array water column
indx_dfm = find_closest(depth, abs(float(f_dept[0])))
#~ if indx_dfm == depth_00m : indx_dfm += 1
indx_less_dfm = depth <= np.abs(float(f_dept[0]))
# Shrink the FAME arrays to the foram living depth
foram_growth = foram_growth[indx_less_dfm,
...] # shape is depth, lat, lon
foram_growth_m = foram_growth_m[:, indx_less_dfm,
...] # shape is time, depth, lat, lon
# Do the same for the equilibrium calcite from WOA
delt_dh = delt_dh_init[indx_less_dfm, ...] # idem
delt_dh_m = delt_dh_init_m[:, indx_less_dfm, ...] # idem
# [DENSITY] -- addition of density from WOA
if not (woa_rhom is None):
rho_m_specie = rho_m[:, indx_less_dfm, ...] # idem
#endif
# Get the location where there is SOME growth, based on a certain epsilon
epsilon_growth = 0.1 * fpl.l09_maxgrowth_dic[foram_specie][
0] # or 0.032
# Mask out the regions where the foram_growth is less than the epsilon
masked_f_growth = ma.masked_less_equal(foram_growth, epsilon_growth)
#~ nb_points_growth = (masked_f_growth * 0.0 + 1.0).filled(0.0)
#~ if monthly is True:
#~ nb_points_growth_m = (masked_f_growth_m * 0.0 + 1.0).filled(0.0)
#~ nb_points_growth = ma.where(foram_growth > epsilon_growth,1,0) # 0.000001
#~ if monthly is True:
#~ nb_points_growth_m = ma.where(foram_growth_m > epsilon_growth,1,0) # 0.000001
# Now sum the growth over the depth ...
f_growth = ma.sum(masked_f_growth, axis=0)
#~ n_growth = ma.where(ma.sum(nb_points_growth,axis=0)>0,1,0) # axis 0 = depth
masked_f_growth_m = ma.masked_less_equal(foram_growth_m,
epsilon_growth)
f_growth_m = ma.sum(ma.sum(masked_f_growth_m, axis=1), axis=0)
location_max_foramprod = ma.argmax(masked_f_growth_m, axis=1)
location_max_foramprod = ma.masked_array(
location_max_foramprod,
mask=masked_f_growth_m[:, :, ...].mask.all(axis=1))
# location_max_foramprod = ma.masked_array(location_max_foramprod,mask=masked_f_growth_m[:,0,...].mask)
# Computing the weighted sum for d18Ocalcite using growth over depth
delt_fp = ma.sum(delt_dh * masked_f_growth, axis=0)
delt_fp_m = ma.sum(ma.sum(delt_dh_m * masked_f_growth_m, axis=1),
axis=0)
# [DENSITY] -- addition of density from WOA
if not (woa_rhom is None):
rho_fp_m = ma.sum(ma.sum(rho_m_specie * masked_f_growth_m, axis=1),
axis=0)
#endif
# Mask out the points where no growth occur at all, in order to avoid NaNs ...
delt_fp = delt_fp / ma.masked_less_equal(f_growth, 0.0)
delt_fp_m = delt_fp_m / ma.masked_less_equal(f_growth_m, 0.0)
if not (woa_rhom is None):
# [DENSITY] -- addition of density from WOA
rho_fp_m = rho_fp_m / ma.masked_less_equal(f_growth_m, 0.0)
#endif
# Result of FAME
Z_om_fm = delt_fp
Z_om_fm_m = ma.masked_array(delt_fp_m,
mask=ma.max(location_max_foramprod[:, ...],
axis=0).mask)
# [DENSITY] -- addition of density from WOA
if not (woa_rhom is None):
Z_om_rho_m = ma.masked_array(rho_fp_m,
mask=ma.max(
location_max_foramprod[:, ...],
axis=0).mask)
#endif
if foram_specie == "pachy_s":
Z_om_fm = Z_om_fm + 0.1 # in per mil
Z_om_fm_m = Z_om_fm_m + 0.1 # in per mil
index_for = list(fpl.l09_cnsts_dic.keys()).index(foram_specie)
delt_forams[index_for, ...] = Z_om_fm_m
# [DENSITY] -- addition of density from WOA
if not (woa_rhom is None):
rhom_forams[index_for, ...] = Z_om_rho_m
#endif
#endfor on foram_specie
# For comparison with Lukas Jonkers: old method on first 50 meters
Z_om_lj = d18Oca_lj
# For comparison with previous figures: old method on first 00 meters
Z_om_ol = d18Oca_ol
# [DENSITY] -- addition of density from WOA
if not (woa_rhom is None):
print("Fame is used with density ...")
return delt_forams, Z_om_ol, d18Oca_mar_m, rhom_forams, rhom_ol
else:
print("Fame is used without density ...")
return delt_forams, Z_om_ol, d18Oca_mar_m
def main():
args = {
'doc': 'chain.pkl',
'allWalkerGraph': False,
'firstWalkerHist': False,
'triangle': False,
'version': 2,
'filter' : False,
'firstTrial' : 0
}
#args['doc'] = sys.argv[1]
options = []
i = 1
while i < len(sys.argv):
if sys.argv[i][0] == '-':
if sys.argv[i] == '-awg':
args['allWalkerGraph'] = True
elif sys.argv[i] == '-fwh':
args['firstWalkerHist'] = True
elif sys.argv[i] == '-tri':
args['triangle'] = True
elif sys.argv[i] == '-fil':
args['filter'] = True
else:
if sys.argv[i-1][0] == '-':
if sys.argv[i-1] == '-v':
args['version'] = int(sys.argv[i])
if sys.argv[i-1] == '-tr':
args['firstTrial'] = int(sys.argv[i])
elif sys.argv[i-1] == 'chainInterpreter.py':
args['doc'] = sys.argv[i]
i = i + 1
print args
with open(args['doc'], 'rb') as doc:
chain = pickle.load(doc)
if args['version'] == 2:
chain = np.exp(chain)
nwalkers, nruns, ndim = chain.shape
# print chain.shape
chain = chain[:,args['firstTrial']:nruns,:]
# print chain.shape
if args['filter']:
chain = ma.masked_array(chain, mask=False)
for i in range(ndim):
for j in range(nwalkers):
stDev = np.std(chain[j,:,i])
if stDev > .001:
chain[j,:,i] = ma.masked_array([None] * (nruns - args['firstTrial']), mask=True)
if args['allWalkerGraph']:
for i in range(ndim):
plt.figure(i)
data = chain[:,:,i].transpose()
plt.plot(range(args['firstTrial'], nruns), data)
average = np.average(data)
plt.plot([0, nruns], [testParams[i], testParams[i]], 'k')
plt.plot([0, nruns], [average, average], 'b')
plt.xlabel('runs')
plt.ylabel('value')
plt.title(testParamNames[i] + ' = ' + str(testParams[i]) + ' | avg = %.2E' % average)
plt.show()
if args['firstWalkerHist']:
plt.figure()
plt.plot(range(nruns), chain[0,:,:])
plt.show()
if args['triangle']:
dynRange = [(0, 1)] * 5;
samples = chain[:, :, :].reshape((-1, ndim))
fig = corner.corner(samples, labels=testParamNames, truths=testParams)
fig.savefig("triangle.png")
def zoom_mollview(sig, cmin=None, cmax=None, nest=True):
from numpy.ma import masked_array
from matplotlib.patches import Rectangle
if cmin is None:
cmin = np.min(sig)
if cmax is None:
cmax = np.max(sig)
projected = hp.mollview(sig, return_projected_map=True, nest=nest)
plt.clf()
nmesh = 400
loleft = -35
loright = -30
grid = hp.cartview(sig, latra=[-2.5,2.5], lonra=[loleft,loright], fig=1, xsize=nmesh, return_projected_map=True, nest=nest)
plt.clf()
nside = hp.npix2nside(len(sig))
theta, phi = hp.pix2ang(nside, np.arange(hp.nside2npix(nside)))
# Get position for the zoom window
theta_min = 87.5/180*np.pi
theta_max = 92.5/180*np.pi
delta_theta = 0.55/180*np.pi
phi_min = (180 - loleft)/180.0*np.pi
phi_max = (180 - loright)/180.0*np.pi
delta_phi = 0.55/180*np.pi
angles = np.array([theta, phi]).T
m0 = np.argmin(np.sum((angles - np.array([theta_max, phi_max]))**2, axis=1))
m1 = np.argmin(np.sum((angles - np.array([theta_max, phi_min]))**2, axis=1))
m2 = np.argmin(np.sum((angles - np.array([theta_min, phi_max]))**2, axis=1))
m3 = np.argmin(np.sum((angles - np.array([theta_min, phi_min]))**2, axis=1))
proj = hp.projector.MollweideProj(xsize=800)
m0 = proj.xy2ij(proj.vec2xy(hp.pix2vec(ipix=m0, nside=nside)))
m1 = proj.xy2ij(proj.vec2xy(hp.pix2vec(ipix=m1, nside=nside)))
m2 = proj.xy2ij(proj.vec2xy(hp.pix2vec(ipix=m2, nside=nside)))
m3 = proj.xy2ij(proj.vec2xy(hp.pix2vec(ipix=m3, nside=nside)))
width = m0[1] - m1[1]
height = m2[0] - m1[0]
test_pro = np.full(shape=(400, 1400), fill_value=-np.inf)
test_pro_1 = np.full(shape=(400, 1400), fill_value=-np.inf)
test_pro[:,:800] = projected
test_pro_1[:,1000:1400] = grid.data
tt_0 = masked_array(test_pro, test_pro<-1000)
tt_1 = masked_array(test_pro_1, test_pro_1<-1000)
fig = plt.figure(frameon=False, figsize=(12,8))
ax = fig.add_axes([0, 0, 1, 1])
ax.axis('off')
ax = fig.gca()
plt.plot(np.linspace(m1[1]+width, 1000), np.linspace(m1[0], 0), 'k-')
plt.plot(np.linspace(m1[1]+width, 1000), np.linspace(m2[0], 400), 'k-')
plt.vlines(x=[1000, 1399], ymin=0, ymax=400)
plt.hlines(y=[0,399], xmin=1000, xmax=1400)
c = Rectangle((m1[1], m1[0]), width, height, color='k', fill=False, linewidth=3, zorder=100)
ax.add_artist(c)
cm = plt.cm.Blues
cm = plt.cm.RdBu_r
# cm = plt.cm.coolwarm # Not working, I do not know why it is not working
# cm.set_bad("white")
im1 = ax.imshow(tt_0, cmap=cm, vmin=cmin, vmax=cmax)
cbaxes1 = fig.add_axes([0.08,0.2,0.4,0.04])
cbar1 = plt.colorbar(im1, orientation="horizontal", cax=cbaxes1)
im2 = ax.imshow(tt_1, cmap=cm, vmin=cmin, vmax=cmax)
cbaxes2 = fig.add_axes([1.02,0.285,0.025,0.43])
cbar2 = plt.colorbar(im2, orientation="vertical", cax=cbaxes2)
plt.xticks([])
plt.yticks([])
return fig
def localizeCallback(self, data):
mapToLaser = self.tfListener.lookupTransform('/map', '/laser',
rospy.Time(0))
toLaserTransMatrix = np.asarray(mapToLaser[0][:2])
carOrientQTMap = (data.pose.orientation.x, data.pose.orientation.y,
data.pose.orientation.z, data.pose.orientation.w)
carYawMap = euler_from_quaternion(carOrientQTMap)[2] #yaw
toLaserRotMatrix = np.array([[np.cos(carYawMap), -np.sin(carYawMap)],
[np.sin(carYawMap),
np.cos(carYawMap)]])
toLaser = lambda coords: (coords - toLaserTransMatrix).dot(
toLaserRotMatrix)
#finding goal waypoint and other metadata
carPositionMap = np.array([data.pose.position.x, data.pose.position.y])
carPositionLaser = toLaser(
carPositionMap) #this should be the zero vector
#apply positional extrapolation based on localization delay
extrap = self.extrapolatePosition()
self.LOOKAHEAD_DISTANCE = self.decideLookahead()
print(self.lookSpeedMultiplier())
print(self.lookAngleMultiplier())
pathPointsLaser = toLaser(np.asarray(self.path_points))
distances = np.linalg.norm(pathPointsLaser - carPositionLaser, axis=1)
mask = np.ones(len(distances), dtype=int)
viable = np.where(
np.logical_and((distances >= self.LOOKAHEAD_DISTANCE),
(pathPointsLaser[:, 0] >
(self.FOV_MULT * pathPointsLaser[:, 1]))))
mask[viable] = 0
viableMask = ma.masked_array(distances, mask=mask)
waypointIndex = viableMask.argmin()
waypoint = pathPointsLaser[waypointIndex]
goalX = waypoint[0]
goalY = waypoint[1]
#calculate goal angle and velocity
self.findCurveRadius = lambda distance, offset: distance**2 / (
2 * np.abs(offset))
self.turnRadius = self.findCurveRadius(distances[waypointIndex], goalY)
self.curvature = 1 / self.turnRadius
self.steeringAngle = np.arcsin(
self.WHEELBASE / self.turnRadius) * np.sign(goalY)
self.steeringAngle = np.clip(self.steeringAngle, -self.MAX_TURN_ANGLE,
self.MAX_TURN_ANGLE)
print("===========================")
print("X: " + str(goalX))
print("Y: " + str(goalY))
print("R: " + str(self.turnRadius))
print("angle: " + str(self.steeringAngle))
print("ex: " + str(extrap[0]))
print("ey: " + str(extrap[1]))
print("LOOK: " + str(self.decideLookahead()))
print("SPD: " + str(self.decideVelocity()))
print("SPD-EST: " + str(self.currentSpeed))
print("===========================")
msg = drive_param()
msg.velocity = self.decideVelocity()
msg.angle = self.steeringAngle
self.drivePub.publish(msg)
def test_non_masked_elements(mask, expected_idx, expected_element):
"""Test with a valid element."""
a = ma.masked_array(np.arange(5), mask=mask)
idx, element = _next_non_masked_element(a, 1)
assert idx == expected_idx
assert element == expected_element
def plot_map(self,
dataset,
attribute_data,
min_value=None,
max_value=None,
file=None,
my_title="",
filter=None,
background=None):
""" Plots a 2D image of attribute given by 'name'. matplotlib required.
The dataset must have a method 'get_2d_attribute' defined that returns
a 2D array that is to be plotted. If min_value/max_value are given, all values
that are smaller/larger than these values are set to min_value/max_value.
Argument background is a value to be used for background. If it is not given,
it is considered as a 1/100 under the minimum value of the array.
Filter is a 2D array. Points where filter is > 0 are masked out (put into background).
"""
import matplotlib
matplotlib.use('Qt4Agg')
from matplotlib.pylab import jet, imshow, colorbar, show, axis, savefig, close, figure, title, normalize
from matplotlib.pylab import rot90
attribute_data = attribute_data[filter]
coord_2d_data = dataset.get_2d_attribute(attribute_data=attribute_data)
data_mask = coord_2d_data.mask
# if filter is not None:
# if isinstance(filter, ndarray):
# if not ma.allclose(filter.shape, coord_2d_data.shape):
# raise StandardError, "Argument filter must have the same shape as the 2d attribute."
# filter_data = filter
# else:
# raise TypeError, "The filter type is invalid. A character string or a 2D numpy array allowed."
# filter_data = where(ma.filled(filter_data,1) > 0, 1,0)
# data_mask = ma.mask_or(data_mask, filter_data)
nonmaskedmin = ma.minimum(coord_2d_data) - .2 * (
ma.maximum(coord_2d_data) - ma.minimum(coord_2d_data))
if max_value == None:
max_value = ma.maximum(coord_2d_data)
if min_value == None:
min_value = nonmaskedmin
coord_2d_data = ma.filled(coord_2d_data, min_value)
if background is None:
value_range = max_value - min_value
background = min_value - value_range / 100
coord_2d_data = ma.filled(
ma.masked_array(coord_2d_data, mask=data_mask), background)
# Our data uses NW as 0,0, while matplotlib uses SW for 0,0.
# Rotate the data so the map is oriented correctly.
coord_2d_data = rot90(coord_2d_data, 1)
jet()
figure()
norm = normalize(min_value, max_value)
im = imshow(
coord_2d_data,
origin='lower',
aspect='equal',
interpolation=None,
norm=norm,
)
tickfmt = '%4d'
if isinstance(min_value, float) or isinstance(max_value, float):
tickfmt = '%1.4f'
colorbar(format=tickfmt)
title(my_title)
axis('off')
if file:
savefig(file)
close()
else:
show()
def fromtextfile(fname,
delimitor=None,
commentchar='#',
missingchar='',
varnames=None,
vartypes=None):
"""
Creates a mrecarray from data stored in the file `filename`.
Parameters
----------
fname : {file name/handle}
Handle of an opened file.
delimitor : {None, string}, optional
Alphanumeric character used to separate columns in the file.
If None, any (group of) white spacestring(s) will be used.
commentchar : {'#', string}, optional
Alphanumeric character used to mark the start of a comment.
missingchar : {'', string}, optional
String indicating missing data, and used to create the masks.
varnames : {None, sequence}, optional
Sequence of the variable names. If None, a list will be created from
the first non empty line of the file.
vartypes : {None, sequence}, optional
Sequence of the variables dtypes. If None, it will be estimated from
the first non-commented line.
Ultra simple: the varnames are in the header, one line"""
# Try to open the file.
ftext = openfile(fname)
# Get the first non-empty line as the varnames
while True:
line = ftext.readline()
firstline = line[:line.find(commentchar)].strip()
_varnames = firstline.split(delimitor)
if len(_varnames) > 1:
break
if varnames is None:
varnames = _varnames
# Get the data.
_variables = masked_array([
line.strip().split(delimitor) for line in ftext
if line[0] != commentchar and len(line) > 1
])
(_, nfields) = _variables.shape
ftext.close()
# Try to guess the dtype.
if vartypes is None:
vartypes = _guessvartypes(_variables[0])
else:
vartypes = [np.dtype(v) for v in vartypes]
if len(vartypes) != nfields:
msg = "Attempting to %i dtypes for %i fields!"
msg += " Reverting to default."
warnings.warn(msg % (len(vartypes), nfields), stacklevel=2)
vartypes = _guessvartypes(_variables[0])
# Construct the descriptor.
mdescr = [(n, f) for (n, f) in zip(varnames, vartypes)]
mfillv = [ma.default_fill_value(f) for f in vartypes]
# Get the data and the mask.
# We just need a list of masked_arrays. It's easier to create it like that:
_mask = (_variables.T == missingchar)
_datalist = [
masked_array(a, mask=m, dtype=t, fill_value=f)
for (a, m, t, f) in zip(_variables.T, _mask, vartypes, mfillv)
]
return fromarrays(_datalist, dtype=mdescr)
def compare_density():
# WISE All-sky sample
#
filenames = [
'%s/%s.fits' % (rgz_dir, x)
for x in ('wise_allsky_2M', 'gurkan/gurkan_all', 'rgz_75_wise')
]
labels = ('WISE all-sky sources', 'Gurkan+14 radio galaxies',
'RGZ 75% radio galaxies')
print ''
for fname, label in zip(filenames, labels):
with fits.open(fname) as f:
d = f[1].data
if label == 'RGZ 75% radio galaxies':
d = d[d['ratio'] >= 0.75]
# SNR cut
snr_w1 = d['snr1'] >= wise_snr
snr_w2 = d['snr2'] >= wise_snr
snr_w3 = d['snr3'] >= wise_snr
rgz = d[rgz75 & snr_w1 & snr_w2 & snr_w3]
w1 = d['w1mpro']
w2 = d['w2mpro']
w3 = d['w3mpro']
w4 = d['w4mpro']
x = w2 - w3
y = w1 - w2
# AGN wedge is INCORRECTLY cited in Gurkan+14; check original Mateos+12 for numbers
#
wedge_lims = (y > -3.172 * x + 7.624) & (y > (0.315 * x - 0.222)) & (
y < (0.315 * x + 0.796))
#
# Very rough loci from Wright et al. (2010)
stars_lims = (x > 0) & (x < 1) & (y > 0.1) & (y < 0.4)
el_lims = (x > 0.5) & (x < 1.3) & (y > 0.) & (y < 0.2)
sp_lims = (x > 1.5) & (x < 3.0) & (y > 0.1) & (y < 0.4)
agn_frac = wedge_lims.sum() / float(len(d))
stars_frac = stars_lims.sum() / float(len(d))
el_frac = el_lims.sum() / float(len(d))
sp_frac = sp_lims.sum() / float(len(d))
print 'Fraction of %25s in AGN wedge: %4.1f percent' % (label,
agn_frac * 100)
print 'Fraction of %25s in stars locus: %4.1f percent' % (
label, stars_frac * 100)
print 'Fraction of %25s in elliptical locus: %4.1f percent' % (
label, el_frac * 100)
print 'Fraction of %25s in spiral locus: %4.1f percent' % (
label, sp_frac * 100)
print ''
print ''
'''
# Make empty arrays for TOPCAT
#
xb,yb = 2.250,0.487
xt,yt = 1.958,1.413
xab = np.linspace(xb,6,100)
xat = np.linspace(xt,6,100)
xal = np.linspace(xb,xt,100)
yab = 0.315*xab - 0.222
yat = 0.315*xat + 0.796
yal =-3.172*xal + 7.624
xall = np.append(xab,np.append(xat,xal))
yall = np.append(yab,np.append(yat,yal))
with open('%s/csv/agn_wedge.csv' % rgz_dir,'w') as f:
for x,y in zip(xall,yall):
print >> f,x,y
'''
# Bin data and look at differences?
#
with fits.open(filenames[0]) as f:
wise = f[1].data
with fits.open(filenames[2]) as f:
rgz = f[1].data
bins_w2w3 = np.linspace(-1, 7, 25)
bins_w1w2 = np.linspace(-0.5, 3, 25)
hw, xedges, yedges = np.histogram2d(wise['w2mpro'] - wise['w3mpro'],
wise['w1mpro'] - wise['w2mpro'],
bins=(bins_w2w3, bins_w1w2))
hr, xedges, yedges = np.histogram2d(rgz['w2mpro'] - rgz['w3mpro'],
rgz['w1mpro'] - rgz['w2mpro'],
bins=(bins_w2w3, bins_w1w2))
from matplotlib import pyplot as plt
from matplotlib import cm
fig = plt.figure(1, (10, 5))
fig.clf()
hw_norm = hw / float(np.max(hw))
hr_norm = hr / float(np.max(hr))
from numpy import ma
hw_norm_masked = ma.masked_array(hw_norm, mask=(hw <= 10))
hr_norm_masked = ma.masked_array(hr_norm, mask=(hr <= 10))
extent = [bins_w2w3[0], bins_w2w3[-1], bins_w1w2[0], bins_w1w2[-1]]
ax1 = fig.add_subplot(121)
cmap = cm.jet
cmap.set_bad('w')
im1 = ax1.imshow(hw_norm_masked.T,
alpha=1.0,
extent=extent,
vmin=0.,
vmax=1.,
interpolation='nearest',
origin='lower')
ax1.set_title('WISE All-Sky')
ax1.set_xlabel('(W2-W3)')
ax1.set_ylabel('(W1-W2)')
ax1.set_aspect('auto')
ax2 = fig.add_subplot(122)
cmap = cm.jet
im2 = ax2.imshow(hr_norm_masked.T,
alpha=1.0,
extent=extent,
vmin=0.,
vmax=1.,
interpolation='nearest',
origin='lower')
ax2.set_title('RGZ 75%')
ax2.set_xlabel('(W2-W3)')
ax2.set_aspect('auto')
position = fig.add_axes([0.92, 0.1, 0.02, 0.80])
cb = plt.colorbar(im2, cax=position, orientation='vertical')
cb.set_label('Normalized ratio', fontsize=16)
'''
ax3 = fig.add_subplot(133)
cmap = cm.jet
im3 = ax3.imshow((np.log10(hr_norm/hw_norm)).T, alpha=1.0, extent=extent,interpolation='nearest', origin='lower')
ax3.set_title('RGZ/WISE ratio')
ax3.set_aspect('auto')
position=fig.add_axes([0.92,0.1,0.02,0.80])
cb = plt.colorbar(im3,cax=position,orientation='vertical')
cb.set_label('log(ratio)',fontsize=16)
'''
#plt.show()
fig.savefig('%s/wise_rgz_fractions.png' % rgz_dir)
return None
def loadData(image, Oi, Om, Pm, size):
if len(Pm.shape) > 2:
Pm = Pm[..., 0]
Pm = Pm.astype(np.uint8) #parts mask
Oi = Oi.astype(
np.uint8
) #object instances #more explanation here: http://groups.csail.mit.edu/vision/datasets/ADE20K/
Pm = cv2.resize(Pm, size)
Oi = cv2.resize(Oi, size)
Om = cv2.resize(Om, size)
persons = (Om == 1831).astype(
np.uint8) #find all person instances. person label is 1831
H, W = image.shape[0], image.shape[1]
Oi = ma.masked_array(
Oi, mask=np.logical_not(persons)) # eliminate the non-person instances
Oi = np.ma.filled(Oi, 0)
gt_boxes = [] #will have shape: [N,x1,y1,x2,y2,cls]
masks_instances = [] #shape: [N,H,W,7]
for x in np.unique(Oi): #for each person
if x == 0:
continue
per = (Oi == x).astype(np.uint8).copy() #get the mask for a person
per_before_erosion = per.copy()
per = cv2.erode(
per, np.ones((3, 3), np.uint8), iterations=2
) # the image is badly annotated so I eliminate some artifacts
_, contours, hierarchy = cv2.findContours(per, 1,
2) ######### from here
if len(contours) == 0:
continue
x1 = 100000
y1 = 100000
x2 = -10000
y2 = -10000
for contour in contours:
x, y, w, h = cv2.boundingRect(contour)
xw, yh = x + w, y + h
if x < x1:
x1 = x
if y < y1:
y1 = y
if xw > x2:
x2 = xw
if yh > y2:
y2 = yh
if x2 - x1 < 30 or y2 - y1 < 30: ######### to here i find the bbox of the person as the mask for the person might contain multiple blobs
continue
masks_for_person = np.zeros((H, W, 7),
dtype=np.uint8) # whole body + 6 parts
masks_for_person[...,
0] = per_before_erosion # the first mask is the body
if True: ####### ONLY BODY
Partsmask = ma.masked_array(Pm, mask=np.logical_not(per))
Partsmask = np.ma.filled(Partsmask, 0)
Parts = Partsmask > 0
torso = cv2.erode(
per - Parts, np.ones((5, 5), np.uint8), iterations=2
) # the torso is not annotated. so i remove from the whole body, the parts=> torso
_, contours, hierarchy = cv2.findContours(torso.copy(), 1, 2)
torso2 = np.zeros(shape=torso.shape, dtype=np.uint8)
maxArea, mx, my, mw, mh = 0, 0, 0, 0, 0
for contour in contours:
x, y, w, h = cv2.boundingRect(contour)
if (w * h > maxArea):
maxArea = w * h
mx, my, mw, mh = x, y, w, h
torso2[my:my + mh, mx:mx + mw] = torso[
my:my + mh, mx:mx +
mw] ## again the torso is badly annotated that results in multiple blobs. so I select the biggest blob
for p in np.unique(
Partsmask
): # the parts mask contain labels that are not parts. so i skip over those labels
if p not in body_parts_dict.keys():
continue
part = (Partsmask == p).astype(np.uint8) #select one body part
masks_for_person[..., body_parts_dict[p]] = np.logical_or(
masks_for_person[..., body_parts_dict[p]], part
) #this is where I combine for example right upper leg and right lower leg
masks_for_person[...,
2] = np.logical_or(masks_for_person[..., 2],
torso2) # here is the torso
if float(np.sum(masks_for_person)) / float(
H * W * 7
) < 0.0001: # sometimes the object instance mask exists but there is no body parts annotation. so I skip over it
continue
gt_boxes.append([x1, y1, x2, y2, 1])
masks_instances.append(masks_for_person)
if len(gt_boxes) == 0:
return False, None, None, None, H, W
masks_instances = np.array(masks_instances, dtype=np.uint8)
gt_boxes = np.array(gt_boxes, dtype=np.float32)
mask = masks_instances[
0, :, :, 1] # this mask is used for visualization in tensorboard
return True, gt_boxes, masks_instances, mask, H, W
def skill_test(path_a, fname, data_a, data_b, num_files, hemisphere):
'''Calculate Taylor Skill Score'''
# First calculate the weight attributed to each grid point (based on Area)
nfid = nc.Dataset("{}/{}".format(path_a, fname), 'r')
tarea = nfid.variables['tarea'][:]
nfid.close()
tarea = ma.masked_array(tarea, mask=data_a[0, :, :].mask)
area_weight = tarea / np.sum(tarea)
weighted_mean_a = 0
weighted_mean_b = 0
for i in np.arange(num_files):
weighted_mean_a = weighted_mean_a + np.sum(
area_weight * data_a[i, :, :])
weighted_mean_b = weighted_mean_b + np.sum(
area_weight * data_b[i, :, :])
weighted_mean_a = weighted_mean_a / num_files
weighted_mean_b = weighted_mean_b / num_files
nonzero_weights = np.count_nonzero(area_weight)
area_var_a = 0
area_var_b = 0
for t in np.arange(num_files):
area_var_a = area_var_a + np.sum(
area_weight * np.square(data_a[t, :, :] - weighted_mean_a))
area_var_b = area_var_b + np.sum(
area_weight * np.square(data_b[t, :, :] - weighted_mean_b))
area_var_a = nonzero_weights / (num_files * nonzero_weights -
1.) * area_var_a
area_var_b = nonzero_weights / (num_files * nonzero_weights -
1.) * area_var_b
std_a = np.sqrt(area_var_a)
std_b = np.sqrt(area_var_b)
combined_cov = 0
for i in np.arange(num_files):
combined_cov = combined_cov + np.sum(area_weight*(data_a[i, :, :]-weighted_mean_a)*\
(data_b[i, :, :]-weighted_mean_b))
combined_cov = nonzero_weights / (num_files * nonzero_weights -
1.) * combined_cov
weighted_r = combined_cov / (std_a * std_b)
s = np.square((1+weighted_r)*(std_a*std_b)/\
(area_var_a + area_var_b))
logger.debug('%s Hemisphere skill score = %f', hemisphere, s)
s_crit = 0.99
if s < 0 or s > 1:
logger.error('Skill score out of range for %s Hemisphere', hemisphere)
return False
elif s > s_crit:
logger.info('Quadratic Skill Test Passed for %s Hemisphere',
hemisphere)
return True
else:
logger.info('Quadratic Skill Test Failed for %s Hemisphere',
hemisphere)
return False
def feedForwardWtaReadout(layers,
wtaStrength=1.,
offset=0.,
noiseMagnitude=1.,
inhStrength=None,
noWtaMask=False,
fixedPatternNoiseSigma=0.0):
"""
Create a the connection matrix as a masked matrix of a feedforward network with a winner-take-all network in the last layer.
The network uses He initialization (He et al IEEE Comp Vision 2015).
Keywords:
--- layers: list of number of neurons in the consecutive layers
--- wtaStrength: wieght of the excitatory connections
--- offset: offset in the feedforward connections
--- noiseMagnitued: magnitude of the uniform noise in the feedforward connections
--- inhStrength: the strength of the inhibitory connections if specified
"""
# get the number of neurons
N = np.sum(layers)
W = np.zeros((N, N)) # Placeholder
WMask = np.ones((N, N))
low = 0
mid = layers[0]
upper = mid + layers[1]
for i in range(len(layers) - 1):
WMask[mid:upper, low:mid] = 0
# Initialize the weights
numbBefore = mid - low
numbAfter = upper - mid
norm = np.sqrt(2. / float(numbBefore))
W[mid:upper, low:mid] = np.random.randn(numbAfter, numbBefore) * norm
if not i == len(layers) - 2:
low = mid
mid = upper
upper += layers[i + 2]
# create WTA matrix
if not (inhStrength is None):
inhW = inhStrength
elif layers[-1] != 1:
inhW = wtaStrength / (layers[-1] - 1.)
WX = ma.masked_array(W, mask=WMask.astype(int))
index = np.where(WX.mask == 1)
WX.data[index] = 0
Nlast = layers[-1]
wta = -1. * np.ones((Nlast, Nlast)) * inhW
np.fill_diagonal(wta, wtaStrength)
relNoise = 1. + np.random.normal(
0.0, fixedPatternNoiseSigma, size=(Nlast, Nlast))
relNoise = np.maximum(relNoise, 0.0)
wta = wta * relNoise
if not noWtaMask:
WMask[-Nlast:, -Nlast:] = 0
W[-Nlast:, -Nlast:] = wta
return WX
import numpy as np import numpy.ma as MA mask_arr = MA.masked_array(range(10),fill_value = -999) print mask_arr , mask_arr.fill_value mask_arr[2] = MA.masked print mask_arr print mask_arr.mask narr = MA.masked_where(mask_arr < 6, mask_arr) print narr, narr.fill_value x= MA.filled(narr) print x print np.dtype(x)
def fit(self,
conver=DEFAULT_CONVERGENCE,
minit=DEFAULT_MINIT,
maxit=DEFAULT_MAXIT,
fflag=DEFAULT_FFLAG,
maxgerr=DEFAULT_MAXGERR,
going_inwards=False):
"""
Fit an elliptical isophote.
Parameters
----------
conver : float, optional
The main convergence criterion. Iterations stop when the
largest harmonic amplitude becomes smaller (in absolute
value) than ``conver`` times the harmonic fit rms. The
default is 0.05.
minit : int, optional
The minimum number of iterations to perform. A minimum of 10
(the default) iterations guarantees that, on average, 2
iterations will be available for fitting each independent
parameter (the four harmonic amplitudes and the intensity
level). For the first isophote, the minimum number of
iterations is 2 * ``minit`` to ensure that, even departing
from not-so-good initial values, the algorithm has a better
chance to converge to a sensible solution.
maxit : int, optional
The maximum number of iterations to perform. The default is
50.
fflag : float, optional
The acceptable fraction of flagged data points in the
sample. If the actual fraction of valid data points is
smaller than this, the iterations will stop and the current
`~photutils.isophote.Isophote` will be returned. Flagged
data points are points that either lie outside the image
frame, are masked, or were rejected by sigma-clipping. The
default is 0.7.
maxgerr : float, optional
The maximum acceptable relative error in the local radial
intensity gradient. This is the main control for preventing
ellipses to grow to regions of too low signal-to-noise
ratio. It specifies the maximum acceptable relative error
in the local radial intensity gradient. `Busko (1996; ASPC
101, 139)
<https://ui.adsabs.harvard.edu/abs/1996ASPC..101..139B/abstract>`_
showed that the fitting precision relates to that relative
error. The usual behavior of the gradient relative error is
to increase with semimajor axis, being larger in outer,
fainter regions of a galaxy image. In the current
implementation, the ``maxgerr`` criterion is triggered only
when two consecutive isophotes exceed the value specified by
the parameter. This prevents premature stopping caused by
contamination such as stars and HII regions.
A number of actions may happen when the gradient error
exceeds ``maxgerr`` (or becomes non-significant and is set
to `None`). If the maximum semimajor axis specified by
``maxsma`` is set to `None`, semimajor axis growth is
stopped and the algorithm proceeds inwards to the galaxy
center. If ``maxsma`` is set to some finite value, and this
value is larger than the current semimajor axis length, the
algorithm enters non-iterative mode and proceeds outwards
until reaching ``maxsma``. The default is 0.5.
going_inwards : bool, optional
Parameter to define the sense of SMA growth. When fitting
just one isophote, this parameter is used only by the code
that defines the details of how elliptical arc segments
("sectors") are extracted from the image, when using area
extraction modes (see the ``integrmode`` parameter in the
`~photutils.isophote.EllipseSample` class). The default is
`False`.
Returns
-------
result : `~photutils.isophote.Isophote` instance
The fitted isophote, which also contains fit status
information.
Examples
--------
>>> from photutils.isophote import EllipseSample, EllipseFitter
>>> sample = EllipseSample(data, sma=10.)
>>> fitter = EllipseFitter(sample)
>>> isophote = fitter.fit()
"""
sample = self._sample
# this flag signals that limiting gradient error (`maxgerr`)
# wasn't exceeded yet.
lexceed = False
# here we keep track of the sample that caused the minimum harmonic
# amplitude(in absolute value). This will eventually be used to
# build the resulting Isophote in cases where iterations run to
# the maximum allowed (maxit), or the maximum number of flagged
# data points (fflag) is reached.
minimum_amplitude_value = np.Inf
minimum_amplitude_sample = None
# these must be passed throughout the execution chain.
fixed_parameters = self._sample.geometry.fix
for i in range(maxit):
# Force the sample to compute its gradient and associated values.
sample.update(fixed_parameters)
# The extract() method returns sampled values as a 2-d numpy array
# with the following structure:
# values[0] = 1-d array with angles
# values[1] = 1-d array with radii
# values[2] = 1-d array with intensity
values = sample.extract()
# We have to check for a zero-length condition here, and bail out
# in case it is detected. The scipy fitter won't raise an exception
# for zero-length input arrays, but just prints an "INFO" message.
# This may result in an infinite loop.
if len(values[2]) < 1:
s = str(sample.geometry.sma)
log.warning("Too small sample to warrant a fit. SMA is " + s)
sample.geometry.fix = fixed_parameters
return Isophote(sample, i + 1, False, 3)
# Fit harmonic coefficients. Failure in fitting is
# a fatal error; terminate immediately with sample
# marked as invalid.
try:
coeffs = fit_first_and_second_harmonics(values[0], values[2])
coeffs = coeffs[0]
except Exception as e:
log.warning(e)
sample.geometry.fix = fixed_parameters
return Isophote(sample, i + 1, False, 3)
# Mask out coefficients that control fixed ellipse parameters.
free_coeffs = ma.masked_array(coeffs[1:], mask=fixed_parameters)
# Largest non-masked harmonic in absolute value drives the
# correction.
largest_harmonic_index = np.argmax(np.abs(free_coeffs))
largest_harmonic = free_coeffs[largest_harmonic_index]
# see if the amplitude decreased; if yes, keep the
# corresponding sample for eventual later use.
if abs(largest_harmonic) < minimum_amplitude_value:
minimum_amplitude_value = abs(largest_harmonic)
minimum_amplitude_sample = sample
# check if converged
model = first_and_second_harmonic_function(values[0], coeffs)
residual = values[2] - model
if ((conver * sample.sector_area * np.std(residual)) >
np.abs(largest_harmonic)):
# Got a valid solution. But before returning, ensure
# that a minimum of iterations has run.
if i >= minit - 1:
sample.update(fixed_parameters)
return Isophote(sample, i + 1, True, 0)
# it may not have converged yet, but the sample contains too
# many invalid data points: return.
if sample.actual_points < (sample.total_points * fflag):
# when too many data points were flagged, return the
# best fit sample instead of the current one.
minimum_amplitude_sample.update(fixed_parameters)
return Isophote(minimum_amplitude_sample, i + 1, True, 1)
# pick appropriate corrector code.
corrector = _CORRECTORS[largest_harmonic_index]
# generate *NEW* EllipseSample instance with corrected
# parameter. Note that this instance is still devoid of other
# information besides its geometry. It needs to be explicitly
# updated for computations to proceed. We have to build a new
# EllipseSample instance every time because of the lazy
# extraction process used by EllipseSample code. To minimize
# the number of calls to the area integrators, we pay a
# (hopefully smaller) price here, by having multiple calls to
# the EllipseSample constructor.
sample = corrector.correct(sample, largest_harmonic)
sample.update(fixed_parameters)
# see if any abnormal (or unusual) conditions warrant
# the change to non-iterative mode, or go-inwards mode.
proceed, lexceed = self._check_conditions(sample, maxgerr,
going_inwards, lexceed)
if not proceed:
sample.update(fixed_parameters)
return Isophote(sample, i + 1, True, -1)
# Got to the maximum number of iterations. Return with
# code 2, and handle it as a valid isophote. Use the
# best fit sample instead of the current one.
minimum_amplitude_sample.update(fixed_parameters)
return Isophote(minimum_amplitude_sample, maxit, True, 2)
def __init__(self,
path,
deadPixeltol=2,
aspect="auto",
Linescan=True,
autoclose=False,
save=False):
self.shift_is_held = False
self.path = path
self.deadPixeltol = deadPixeltol
self.aspect = aspect
self.Linescan = Linescan
self.leftpressed = False
if autoclose:
plt.ioff()
else:
plt.ion()
dirname, filename = os.path.split(self.path)
filename, ext = os.path.splitext(filename)
hyppath = self.path
specpath = os.path.join(dirname, filename + '_X_axis.asc')
filepath = os.path.join(dirname,
filename + '_SEM image after carto.tif')
start = time.time()
data = pd.read_csv(hyppath, delimiter='\t', header=None).to_numpy()
end = time.time()
print(end - start)
#data = np.loadtxt(hyppath)
xlen = int(data[0, 0]) #nbr de pts selon x
ylen = int(data[1, 0]) #nbr de pts selon y
wavelenght = np.loadtxt(specpath)
self.wavelenght = wavelenght[:2048] #bins du spectro
xcoord = data[0, 1:]
ycoord = data[1, 1:]
CLdata = data[
2:,
1:] #tableau de xlen * ylen points (espace) et 2048 longueur d'onde CLdata[:,n] n = numero du spectr
self.hypSpectrum = np.transpose(
np.reshape(np.transpose(CLdata),
(ylen, xlen, len(self.wavelenght))), (0, 1, 2))
#correct dead / wrong pixels
self.hypSpectrum, self.hotpixels = correct_dead_pixel(
self.hypSpectrum, tol=self.deadPixeltol)
self.wavelenght = ma.masked_array(self.wavelenght, mask=False)
hypmask = np.resize(self.wavelenght.mask, self.hypSpectrum.shape)
self.hypSpectrum = ma.masked_array(self.hypSpectrum, mask=hypmask)
xscale_CL, yscale_CL, acceleration, image = scaleSEMimage(filepath)
if self.Linescan:
self.fig, (self.ax, self.bx, self.cx) = plt.subplots(
3, 1, sharex=True, gridspec_kw={'height_ratios': [1, 1, 3]})
else:
self.fig, (self.ax, self.bx, self.cx) = plt.subplots(3,
1,
sharex=True,
sharey=True)
self.fig.patch.set_alpha(0) #Transparency style
self.fig.subplots_adjust(top=0.9,
bottom=0.12,
left=0.15,
right=0.82,
hspace=0.1,
wspace=0.05)
newX = np.linspace(xscale_CL[int(xcoord.min())],
xscale_CL[int(xcoord.max())], len(xscale_CL))
newY = np.linspace(yscale_CL[int(ycoord.min())],
yscale_CL[int(ycoord.max())], len(yscale_CL))
self.X = np.linspace(np.min(newX), np.max(newX),
self.hypSpectrum.shape[1])
self.Y = np.linspace(np.min(newY), np.max(newY),
self.hypSpectrum.shape[0])
nImage = np.array(
image.crop(
(xcoord.min(), ycoord.min(), xcoord.max(), ycoord.max())))
self.ax.imshow(
nImage,
cmap='gray',
vmin=0,
vmax=65535,
interpolation="None",
extent=[np.min(newX),
np.max(newX),
np.max(newY),
np.min(newY)])
self.hypimage = np.nansum(self.hypSpectrum, axis=2)
jet = cm.get_cmap("jet")
jet.set_bad(color='k')
self.hyperspectralmap = cm.ScalarMappable(cmap=jet)
self.hyperspectralmap.set_clim(vmin=np.nanmin(self.hypimage),
vmax=np.nanmax(self.hypimage))
self.lumimage = self.bx.imshow(
self.hypimage,
cmap=self.hyperspectralmap.cmap,
norm=self.hyperspectralmap.norm,
interpolation="None",
extent=[np.min(newX),
np.max(newX),
np.max(newY),
np.min(newY)])
if self.Linescan:
self.linescan = np.nansum(self.hypSpectrum, axis=0)
self.linescanmap = cm.ScalarMappable(cmap=jet)
self.linescanmap.set_clim(vmin=np.nanmin(self.linescan),
vmax=np.nanmax(self.linescan))
# ATTENTION pcolormesh =/= imshow
#imshow takes values at pixel
#pcolormesh takes values between
temp = np.linspace(newX.min(), newX.max(), self.X.size + 1)
self.im = self.cx.pcolormesh(temp,
eV_To_nm / self.wavelenght,
self.linescan.T,
cmap=self.linescanmap.cmap,
shading='flat',
rasterized=True)
#dX=np.diff(self.X).mean()
#newX = np.arange(min(self.X),max(self.X)+dX,dX)-dX/2
# dW = np.diff(self.wavelenght).mean()
# newW = np.arange(min(self.wavelenght),max(self.wavelenght)+dW,dW)-dW/2
#self.im=self.cx.pcolormesh(newX,eV_To_nm/self.wavelenght,self.linescan.T,cmap=self.linescanmap.cmap,shading = 'flat',rasterized=True)
def format_coord(x, y):
xarr = self.X
yarr = eV_To_nm / self.wavelenght
if ((x > xarr.min()) and (x <= xarr.max()) and (y > yarr.min())
and (y <= yarr.max())):
col = np.argmin(abs(xarr - x)) #np.searchsorted(xarr, x)-1
row = np.argmin(abs(yarr - y)) #np.searchsorted(yarr, y)-1
z = self.linescan.T[row, col]
return f'x={x:1.4f}, y={y:1.4f}, lambda={eV_To_nm/y:1.2f}, z={z:1.2e} [{row},{col}]'
else:
return f'x={x:1.4f}, y={y:1.4f}, lambda={eV_To_nm/y:1.2f}'
self.cx.format_coord = format_coord
self.cx.set_ylabel("Energy (eV)")
self.cx.set_xlabel("distance (µm)")
self.cx.set_aspect(self.aspect)
else:
self.im = self.cx.imshow(self.wavelenght[np.argmax(
self.hypSpectrum, axis=2)],
cmap='viridis',
extent=[
np.min(newX),
np.max(newX),
np.max(newY),
np.min(newY)
])
self.cx.set_aspect(aspect)
self.bx.set_aspect(self.aspect)
self.ax.set_aspect(self.aspect)
self.ax.get_shared_y_axes().join(self.ax, self.bx)
self.ax.set_ylabel("distance (µm)")
# fig.text(ax.get_position().bounds[0]-0.11, ax.get_position().bounds[1],'distance (µm)',fontsize=16, va='center', rotation='vertical')
pos = self.cx.get_position().bounds
cbar_ax = self.fig.add_axes([0.85, pos[1], 0.05, pos[-1] * 0.9])
self.fig.colorbar(self.linescanmap, cax=cbar_ax)
cbar_ax.ticklabel_format(axis='both', style='sci', scilimits=(0, 0))
pos = self.bx.get_position().bounds
cbar_ax = self.fig.add_axes([0.85, pos[1], 0.05, pos[-1] * 0.9])
self.fig.colorbar(self.hyperspectralmap, cax=cbar_ax)
cbar_ax.ticklabel_format(axis='both', style='sci', scilimits=(0, 0))
if save == True:
self.fig.savefig(os.path.join(dirname, filename + ".png"), dpi=300)
if autoclose == True:
plt.close(self.fig)
return None
else:
self.spec_fig, self.spec_ax = plt.subplots()
self.spec_data, = self.spec_ax.plot(
eV_To_nm / self.wavelenght, np.zeros(self.wavelenght.shape[0]))
self.spec_ax.set_ylim((0, self.hypSpectrum.max()))
self.spec_ax.set_xlabel('Energy (eV)')
self.spec_ax.set_ylabel('Intensity (a.u)')
self.spec_fig.subplots_adjust(top=0.885,
bottom=0.125,
left=0.13,
right=0.94)
self.spec_ax.ticklabel_format(axis='y',
style='sci',
scilimits=(0, 0))
self.spec_minbar = self.spec_ax.axvline(eV_To_nm /
self.wavelenght.max())
self.spec_maxbar = self.spec_ax.axvline(eV_To_nm /
self.wavelenght.min())
#if Linescan:
#self.cursor = MultiCursor(self.fig.canvas, (self.ax, self.bx), color='r', lw=1,horizOn=True, vertOn=True)
def onmotion(event):
if self.leftpressed:
x = event.xdata
y = event.ydata
if ((event.inaxes is not None) and (x > self.X.min())
and (x <= self.X.max()) and (y > self.Y.min())
and (y <= self.Y.max())):
indx = np.argmin(abs(x - self.X))
indy = np.argmin(abs(y - self.Y))
self.spec_data.set_ydata(self.hypSpectrum.data[indy,
indx])
self.spec_fig.canvas.draw_idle()
def onclick(event):
if event.button == 1:
self.leftpressed = True
#self.cursor.active = True
#if event.dblclick:
# if not(self.cursor.visible):
# self.fig.canvas.blit(self.fig.bbox)
#self.cursor.active = not(self.cursor.active)
# self.cursor.visible = not(self.cursor.visible)
# self.fig.canvas.draw_idle()
# self.fig.canvas.blit(self.fig.bbox)
# elif self.cursor.active:
# x = event.xdata
# y = event.ydata
# if ((event.inaxes is not None) and (x > self.X.min()) and (x <= self.X.max()) and
# (y > self.Y.min()) and (y <= self.Y.max())):
# indx = np.argmin(abs(x-self.X))
# indy=np.argmin(abs(y-self.Y))
# self.spec_data.set_ydata(self.hypSpectrum.data[indy,indx])
# self.spec_fig.canvas.draw_idle()
elif event.button == 3:
self.wavelenght = ma.masked_array(self.wavelenght.data,
mask=False)
hypmask = np.resize(self.wavelenght.mask,
self.hypSpectrum.shape)
self.hypSpectrum = ma.masked_array(self.hypSpectrum.data,
mask=hypmask)
self.hypimage = np.nansum(self.hypSpectrum, axis=2)
self.lumimage.set_array(self.hypimage)
self.hyperspectralmap.set_clim(
vmin=np.nanmin(self.hypimage),
vmax=np.nanmax(self.hypimage))
self.cx.set_ylim(eV_To_nm / self.wavelenght.max(),
eV_To_nm / self.wavelenght.min())
# TODO : A recoder
self.linescan = np.nansum(self.hypSpectrum, axis=0)
self.spec_maxbar.set_xdata(
np.repeat(eV_To_nm / self.wavelenght.min(), 2))
self.spec_minbar.set_xdata(
np.repeat(eV_To_nm / self.wavelenght.max(), 2))
self.im.set_array(
(self.linescan.T[:, :]).ravel()) #flat shading
#self.im.set_array((self.linescan.T).ravel())#gouraud shading
self.linescanmap.set_clim(vmin=np.nanmin(self.linescan),
vmax=np.nanmax(self.linescan))
self.im.set_norm(self.linescanmap.norm)
#self.im.set_cmap(self.linescanmap.cmap)
self.fig.canvas.draw_idle()
self.spec_fig.canvas.draw_idle()
self.fig.canvas.blit(self.fig.bbox)
def onrelease(event):
if event.button == 1:
self.leftpressed = False
#self.cursor.active = False
def onselect(ymin, ymax):
indmin = np.argmin(abs(eV_To_nm / self.wavelenght - ymin))
indmax = np.argmin(abs(eV_To_nm / self.wavelenght - ymax))
#indmin resp.max est lindex tel que wavelenght[indmin] minimise
# la distance entre la position du clic et la longueur d'onde
# print(eV_To_nm/self.wavelenght[indmin])
# print(eV_To_nm/self.wavelenght[indmax])
if abs(indmax - indmin) < 1: return
self.wavelenght = ma.masked_outside(self.wavelenght.data,
eV_To_nm / ymax,
eV_To_nm / ymin)
hypmask = np.resize(self.wavelenght.mask,
self.hypSpectrum.shape)
self.hypSpectrum = ma.masked_array(self.hypSpectrum.data,
mask=hypmask)
self.hypimage = np.nansum(self.hypSpectrum, axis=2)
self.lumimage.set_array(self.hypimage)
self.hyperspectralmap.set_clim(vmin=np.nanmin(self.hypimage),
vmax=np.nanmax(self.hypimage))
self.cx.set_ylim(eV_To_nm / self.wavelenght.max(),
eV_To_nm / self.wavelenght.min())
self.linescan = np.nansum(self.hypSpectrum, axis=0)
self.spec_maxbar.set_xdata(
np.repeat(eV_To_nm / self.wavelenght.min(), 2))
self.spec_minbar.set_xdata(
np.repeat(eV_To_nm / self.wavelenght.max(), 2))
self.im.set_array(
(self.linescan.T[:, :]).ravel()) #flat shading
#self.im.set_array((self.linescan.T).ravel())#gouraud shading
self.linescanmap.set_clim(vmin=np.nanmin(self.linescan),
vmax=np.nanmax(self.linescan))
self.im.set_norm(self.linescanmap.norm)
#self.im.set_cmap(self.linescanmap.cmap)
self.fig.canvas.draw_idle()
self.spec_fig.canvas.draw_idle()
self.fig.canvas.blit(self.fig.bbox)
self.span = None
if Linescan:
self.span = SpanSelector(self.cx,
onselect,
'vertical',
useblit=True,
rectprops=dict(alpha=0.5,
facecolor='red'),
button=1)
self.fig.canvas.mpl_connect('button_press_event', onclick)
self.fig.canvas.mpl_connect('motion_notify_event', onmotion)
self.fig.canvas.mpl_connect('button_release_event', onrelease)
plt.show(block=True)
r2_df = pd.DataFrame(data=stat_col_arr.mean(axis=1),
columns=subsp_axis,
index=layers)
r2var_df = pd.DataFrame(data=stat_col_arr.std(axis=1),
columns=subsp_axis,
index=layers)
kappa_df = pd.DataFrame(data=param_col_arr[:, :, :, 3].mean(axis=1),
columns=subsp_axis,
index=layers)
print("Rsquare data frame")
print(r2_df)
print("kappa data frame")
print(kappa_df)
#%% Using masked array to avoid the unsuccessful fittings.
import numpy.ma as ma
makappa = ma.masked_array(data=param_col_arr[:, :, :, 3],
mask=(stat_col_arr < 0.5) | np.isnan(stat_col_arr))
mar2 = ma.masked_array(data=stat_col_arr,
mask=np.isinf(stat_col_arr) | np.isnan(stat_col_arr))
import pandas as pd
r2_df = pd.DataFrame(data=mar2.mean(axis=1), columns=subsp_axis, index=layers)
r2var_df = pd.DataFrame(data=mar2.std(axis=1),
columns=subsp_axis,
index=layers)
kappa_df = pd.DataFrame(data=makappa.mean(axis=1),
columns=subsp_axis,
index=layers)
print("Rsquare data frame")
print(r2_df)
print("kappa data frame")
print(kappa_df)
import numpy.ma as ma # print dir(ma) mrr = ma.masked_array(range(10), fill_value=-999) print mrr mrr[2] = ma.masked print mrr print mrr.mask narr = ma.masked_where(mrr > 6, mrr) print narr x = ma.filled(narr) print x print type(x)
import numpy.ma as MA
mask = MA.masked_array(range(0, 9),
fill_value=-999,
mask=[0, 0, 0, 1, 0, 0, 0, 0, 0])
print(mask)
# Time indexes
btidx0 = where(time == tmin)[0][0]
btidx1 = where(time == tmax)[0][0] + 1
ftidx0 = where(ftime == tmin)[0][0]
ftidx1 = where(ftime == tmax)[0][0] + 1
time = time[btidx0:btidx1]
detrend = masked_where(isnan(detrend), detrend)
# Get dimensions
(nlats, nlons) = aggmap.shape
nt = len(time)
nirr = 3
varr = masked_array(zeros((nt, nlats, nlons, nirr)),
mask=ones((nt, nlats, nlons, nirr)))
with Dataset(irfile) as f:
var = f.variables[vname]
if 'units' in var.ncattrs():
units = var.units
else:
units = ''
if 'long_name' in var.ncattrs():
lname = var.long_name
else:
lname = ''
varr[:, :, :, 0] = var[ftidx0:ftidx1]
if isfile(rffile):
with Dataset(rffile) as f:
