def main():
# rocket = T1_test_vehicle
rocket = T2_test_vehicle
# x, v
Y0 = [0, 0]
flight = Flight(rocket, Y0)
Y = flight.solve()
# Aliases
# print "Y:", Y.shape, Y
x = np.array(Y[:, 0])
v = np.array(Y[:, 1])
D = np.array(Y[:, 2])
W = np.array(Y[:, 3])
t = flight.getTimes()
pyplot.plot(t, x, t, v)
# odeint can skip over many time steps, leaving blanks for intermediate values
# Hence, masked arrays to ignore the blank values.
D = ma.masked_invalid(D)
W = ma.masked_invalid(W)
mask = D.mask
t = ma.masked_array(t, mask=mask)
D = D.compressed()
W = W.compressed()
t = t.compressed()
pyplot.figure()
pyplot.plot(t, D, t, W)
pyplot.show()
return Y
def score(self, dataframe, parallel): global _parallel_score_dataframe n_comparators = len(self.comparators) _parallel_score_dataframe = dataframe if parallel==True: _parallel_score_dataframe = dataframe pool = multiprocessing.Pool() elif parallel==False: pool=None else: pool=parallel #assumed to be instance of multiprocessing.Pool() if pool is not None: scores = pool.map(parallel_score, self.comparators) else: scores = map(serial_score, self.comparators, [dataframe] * len(self.comparators)) s_scores, b_scores = zip(*scores) s_scores = ma.masked_invalid(s_scores) b_scores = ma.masked_invalid(b_scores) return ( scipy.stats.mstats.gmean(s_scores, axis=0).data, scipy.stats.mstats.gmean(b_scores, axis=0).data )
def extract(self,PSF):
#Placeholder
pixsig = 1.
sh = self.sh
npsf = PSF.size
flux = np.zeros(sh)
uflux = np.zeros(sh)
sky1d = np.zeros(sh)
usky1d = np.zeros(sh)
im = self.Order.image_rect
uim = self.Order.uimage_rect
sky = self.Order.sky_rect
usky = self.Order.usky_rect
for i in np.arange(sh):
# flux[i] = (PSF*im[i,:]).sum() / (PSF**2).sum()
flux[i] = (PSF*im[i,:]/uim[i,:]**2).sum() / (PSF**2/uim[i,:]**2).sum()
uflux[i] = np.sqrt(1.0/(PSF**2/uim[i,:]**2.).sum())
sky1d[i] = (np.abs(PSF)*sky[i,:]).sum() / (PSF**2).sum()
usky1d[i] = np.sqrt(1.0/(PSF**2/usky[i,:]**2.).sum())
flux = ma.masked_invalid(flux)
flux = ma.filled(flux,1.)
uflux = ma.masked_invalid(uflux)
uflux = ma.filled(uflux,1000.)
sky1d = ma.masked_invalid(sky1d)
sky1d = ma.filled(sky1d,1.)
usky1d = ma.masked_invalid(usky1d)
usky1d = ma.filled(usky1d,1000.)
sky_cont = self._fitCont(self.wave,sky1d)
return flux,uflux,sky1d-sky_cont,usky1d
def calc_stat_1d(self, trial_array, calc_corr=True):
"""take an 1D array of power spectra and find some basic statistics
on it"""
stat_1d = {}
old_settings = np.seterr(invalid="warn", under="warn")
mtrial_array = ma.masked_invalid(trial_array)
stat_1d['mean'] = np.ma.filled(np.ma.mean(mtrial_array, axis=0),
fill_value=np.nan)
stat_1d['std'] = np.ma.filled(np.ma.std(mtrial_array, axis=0, ddof=1),
fill_value=np.nan)
if calc_corr:
mtrial_array_trans = np.ma.transpose(
ma.masked_invalid(trial_array))
stat_1d['corr'] = np.ma.filled(np.ma.corrcoef(mtrial_array_trans,
ddof=1), fill_value=np.nan)
stat_1d['cov'] = np.ma.filled(np.ma.cov(mtrial_array_trans,
ddof=1), fill_value=np.nan)
np.seterr(**old_settings)
return stat_1d
def probability_bincount(obs, drop=True):
"""
Normalise the observed bincount from `obs` to sum up to unity.
Parameters
----------
obs: iterable
An iterable of integer numerals.
drop: bool (optional)
Determines whether or not bins with zero events are dropped from the
resulting list.
Returns
-------
A frequency distribution that is normalised to unity.
"""
obs = nma.asanyarray(obs, dtype="int32")
nma.masked_invalid(obs, copy=False)
obs = numpy.ravel(obs)
if obs.size == 0:
return list()
total = float(len(obs))
freq = numpy.bincount(obs[~obs.mask])
points = [(k, val / total) for (k, val) in enumerate(freq) if val > 0]
return points
def compute_zscore(obs, random_stats):
"""
Parameters
----------
obs: numeral
original observation
random_stats : iterable
same observable in randomised versions
"""
random_stats = nma.asanyarray(random_stats)
nma.masked_invalid(random_stats, copy=False)
random_stats = numpy.ravel(random_stats)
if random_stats.size == 0:
return numpy.nan
mean = numpy.mean(random_stats.filled(0.0))
std = numpy.std(random_stats.filled(0.0))
nominator = obs- mean
if nominator == 0.0:
return nominator
if std == 0.0:
if nominator < 0.0:
return -numpy.inf
else:
return numpy.inf
else:
return (nominator / std)
def set_profile_data(self, pts=[[],[]]):
try:
pres = ma.masked_invalid(self.data['pres'])
except KeyError:
raise KeyError, "Pres in hPa (PRES) is required!"
try:
tc=ma.masked_invalid(self.data['temp'])
except KeyError:
raise KeyError, "Temperature in C (TEMP) is required!"
try:
dwpt=ma.masked_invalid(self.data['dwpt'])
except KeyError:
warnings.warn("Warning: No DWPT available")
dwpt=ma.masked_array(zeros(pres.shape),mask=True)
try:
sknt=self.data['sknt']
drct=self.data['drct']
rdir = (270.-drct)*(pi/180.)
uu = (sknt*cos(rdir))
vv = (sknt*sin(rdir))
except KeyError:
warnings.warn("Warning: No SKNT/DRCT available")
uu=zeros(pres.shape)
vv=zeros(pres.shape)
self.tcprof_line.set_data(tc, pres)
self.dpprof_line.set_data(dwpt, pres)
self.plt_title.set_text("%s %s"%(self["StationNumber"],self['SoundingDate']))
self.wind_pts.set_data(*zip(*pts))
return self.tcprof_line,self.dpprof_line,self.plt_title,self.wind_pts
def despike(self, n1=2, n2=20, block=100, keep=0):
"""
Wild Edit Seabird-like function. Passes with Standard deviation
`n1` and `n2` with window size `block`.
"""
data = self.values.astype(float).copy()
roll = rolling_window(data, block)
roll = ma.masked_invalid(roll)
std = n1 * roll.std(axis=1)
mean = roll.mean(axis=1)
# Use the last value to fill-up.
std = np.r_[std, np.tile(std[-1], block - 1)]
mean = np.r_[mean, np.tile(mean[-1], block - 1)]
mask = (np.abs(data - mean.filled(fill_value=np.NaN)) >
std.filled(fill_value=np.NaN))
data[mask] = np.NaN
# Pass two recompute the mean and std without the flagged values from pass
# one and removed the flagged data.
roll = rolling_window(data, block)
roll = ma.masked_invalid(roll)
std = n2 * roll.std(axis=1)
mean = roll.mean(axis=1)
# Use the last value to fill-up.
std = np.r_[std, np.tile(std[-1], block - 1)]
mean = np.r_[mean, np.tile(mean[-1], block - 1)]
values = self.values.astype(float)
mask = (np.abs(values - mean.filled(fill_value=np.NaN)) >
std.filled(fill_value=np.NaN))
clean = self.astype(float).copy()
clean[mask] = np.NaN
return clean
def plot_glider(cube, mask_topo=False, **kw):
"""Plot glider cube."""
cmap = kw.pop('cmap', plt.cm.rainbow)
lon = cube.coord(axis='X').points.squeeze()
lat = cube.coord(axis='Y').points.squeeze()
z = cube.coord(axis='Z').points.squeeze()
data = cube.data
data = ma.masked_invalid(data,copy=True)
z = ma.masked_invalid(z,copy=True)
t = cube.coord(axis='T')
t = t.units.num2date(t.points)
dist, pha = sw.dist(lat, lon, units='km')
dist = np.r_[0, np.cumsum(dist)]
dist, z = np.broadcast_arrays(dist[..., None], z)
fig, ax = plt.subplots(figsize=(9, 3.75))
cs = ax.pcolor(dist, z, data, cmap=cmap, snap=True, **kw)
plt.colorbar(cs)
if mask_topo:
h = z.max(axis=1)
x = dist[:, 0]
ax.plot(x, h, color='black', linewidth='0.5', zorder=3)
ax.fill_between(x, h, y2=h.max(), color='0.9', zorder=3)
ax.invert_yaxis()
ax.set_title('Glider track from {} to {}'.format(t[0], t[-1]))
fig.tight_layout()
return fig, ax, cs
def get_lapmat(pos0, pos1, max_disp=1000., dist_function=np.square):
pos0 = np.asarray(pos0)
pos1 = np.asarray(pos1)
num_in, ndim = pos0.shape
num_out, ndim = pos1.shape
if ndim not in (2, 3):
raise ValueError('''Only 2d and 3d data are supported''')
lapmat = np.zeros((num_in + num_out,
num_in + num_out)) * np.nan
costmat, p90 = get_costmat(pos0, pos1, max_disp, dist_function)
m_costmat = ma.masked_invalid(costmat)
lapmat[:num_in, :num_out] = costmat
if np.all(np.isnan(costmat)):
birthcost = deathcost = 1.
else:
birthcost = deathcost = np.percentile(m_costmat.compressed(), 90)
lapmat[num_in:, :num_out] = get_birthmat(pos1, birthcost)
lapmat[:num_in, num_out:] = get_deathmat(pos0, deathcost)
m_lapmat = ma.masked_invalid(lapmat)
fillvalue = m_lapmat.max() * 1.05
lapmat[num_in:, num_out:] = get_lowerright(costmat, fillvalue)
return lapmat
def calc_ratio(fsoil_mary, fsoil_kettle):
lon, lat, topo = sp.parse_STEM_coordinates(
os.path.join(os.environ['SARIKA_INPUT'], 'TOPO-124x124.nc'))
fsoil_mary = maskoceans(lon, lat, fsoil_mary)
fsoil_kettle = maskoceans(lon, lat, fsoil_kettle)
ratio = ma.masked_invalid(fsoil_kettle) / ma.masked_invalid(fsoil_mary)
return(ratio)
def set_UVC(self, U, V, C=None):
self.u = ma.masked_invalid(U, copy=False).ravel()
self.v = ma.masked_invalid(V, copy=False).ravel()
if C is not None:
c = ma.masked_invalid(C, copy=False).ravel()
x, y, u, v, c = delete_masked_points(self.x.ravel(),
self.y.ravel(),
self.u, self.v, c)
else:
x, y, u, v = delete_masked_points(self.x.ravel(), self.y.ravel(),
self.u, self.v)
magnitude = np.hypot(u, v)
flags, barbs, halves, empty = self._find_tails(magnitude,
self.rounding,
**self.barb_increments)
# Get the vertices for each of the barbs
plot_barbs = self._make_barbs(u, v, flags, barbs, halves, empty,
self._length, self._pivot, self.sizes,
self.fill_empty, self.flip)
self.set_verts(plot_barbs)
# Set the color array
if C is not None:
self.set_array(c)
# Update the offsets in case the masked data changed
xy = np.hstack((x[:, np.newaxis], y[:, np.newaxis]))
self._offsets = xy
def get_roms(url, time_slice, n=3):
url = parse_url(url)
with Dataset(url) as nc:
ncv = nc.variables
time = ncv['ocean_time']
tidx = date2index(time_slice, time, select='nearest')
time = num2date(time[tidx], time.units, time.calendar)
mask = ncv['mask_rho'][:]
lon_rho = ncv['lon_rho'][:]
lat_rho = ncv['lat_rho'][:]
anglev = ncv['angle'][:]
u = ncv['u'][tidx, -1, ...]
v = ncv['v'][tidx, -1, ...]
u = shrink(u, mask[1:-1, 1:-1].shape)
v = shrink(v, mask[1:-1, 1:-1].shape)
u, v = rot2d(u, v, anglev[1:-1, 1:-1])
lon = lon_rho[1:-1, 1:-1]
lat = lat_rho[1:-1, 1:-1]
u, v = u[::n, ::n], v[::n, ::n]
lon, lat = lon[::n, ::n], lat[::n, ::n]
u = ma.masked_invalid(u)
v = ma.masked_invalid(v)
return dict(lon=lon, lat=lat, u=u, v=v, time=time)
def agg_stat_2d_pwrspec(self, use_masked=False, debug=None):
r"""TODO: add corr and cov"""
comb_arr = self.combination_array_2d(counts=False, debug=debug)
counts_arr = self.combination_array_2d(counts=True, debug=debug)
stat_summary = {}
for treatment in self.treatment_cases:
entry = {}
if use_masked:
counts_ma = ma.masked_invalid(counts_arr[treatment])
comb_ma = ma.masked_invalid(comb_arr[treatment])
entry['counts'] = np.ma.mean(counts_ma, axis=2)
entry['mean'] = np.ma.mean(comb_ma, axis=2)
if self.num_comb > 1:
entry['std'] = np.ma.std(comb_ma, axis=2, ddof=1)
else:
entry['std'] = 0.
else:
entry['counts'] = np.mean(counts_arr[treatment], axis=2)
entry['mean'] = np.mean(comb_arr[treatment], axis=2)
if self.num_comb > 1:
try:
entry['std'] = np.std(comb_arr[treatment], axis=2, ddof=1)
except FloatingPointError:
print "ERROR: stdev of spectral pairs failed"
entry['std'] = 0.
else:
entry['std'] = 0.
stat_summary[treatment] = entry
return stat_summary
def __init__(self, lats, lons, latsCenter, lonsCenter, elev, alti, img, cameraPosGCRS, photoTime,
station, minBrightness=None, maxBrightness=None):
"""
:param lats: (h+1,w+1) in degrees
:param lons: (h+1,w+1) in degrees
:param latsCenter: (h,w) in degrees
:param lonsCenter: (h,w) in degrees
:param elev: elevation in degrees for each pixel center (h,w), can be None
:param alti: the altitude in km onto which the image was mapped (e.g. 110)
:param img: masked array, (h,w) or (h,w,3) in [0,255] as int or float, can have NaN's
:param cameraPosGCRS: [x,y,z] in km
:param photoTime: datetime object
:param string station:
:param minBrightness:
:param maxBrightness:
"""
assert img.ndim == 2
h, w = img.shape[0], img.shape[1]
assert lats.shape == lons.shape == (h+1, w+1)
assert elev is None or elev.shape == (img.shape[0], img.shape[1])
# adapted from web filenames: RANK.2013.09.26.05.03.gif
identifier = station + '.' + photoTime.strftime('%Y.%m.%d.%H.%M.%S')
BaseMapping.__init__(self, alti, cameraPosGCRS, photoTime, identifier)
self._img = img[:,:,None]
self._lats = ma.masked_invalid(lats, copy=False)
self._lons = ma.masked_invalid(lons, copy=False)
self._latsCenter = ma.masked_invalid(latsCenter, copy=False)
self._lonsCenter = ma.masked_invalid(lonsCenter, copy=False)
self._elevation = ma.masked_invalid(elev, copy=False)
# allow to be changed from outside:
self.minBrightness = minBrightness
self.maxBrightness = maxBrightness
def agg_stat_1d_pwrspec(self, from_2d=False, use_masked=False):
r"""TODO: use masked array here"""
(counts_arr, gerror_arr, comb_arr) = \
self.combination_array_1d(from_2d=from_2d)
stat_summary = {}
for treatment in self.treatment_cases:
entry = {}
if use_masked:
gerror_ma = ma.masked_invalid(gerror_arr[treatment])
counts_ma = ma.masked_invalid(counts_arr[treatment])
comb_ma = ma.masked_invalid(comb_arr[treatment])
if self.num_comb > 1:
entry['counts'] = np.ma.mean(counts_ma, axis=1)
entry['mean'] = np.ma.mean(comb_ma, axis=1)
entry['gauss_std'] = np.ma.mean(gerror_ma, axis=1)
entry['std'] = np.ma.std(comb_ma, axis=1, ddof=1)
entry['corr'] = np.ma.corrcoef(comb_ma)
entry['cov'] = np.ma.cov(comb_ma)
else:
entry['counts'] = counts_ma[:, 0]
entry['mean'] = comb_ma[:, 0]
entry['gauss_std'] = gerror_ma[:, 0]
entry['std'] = 0.
entry['corr'] = 0.
entry['cov'] = 0.
else:
comb_treat = comb_arr[treatment]
if self.num_comb > 1:
entry['counts'] = np.mean(counts_arr[treatment], axis=1)
entry['mean'] = np.mean(comb_treat, axis=1)
entry['gauss_std'] = np.mean(gerror_arr[treatment], axis=1)
entry['std'] = np.std(comb_treat, axis=1, ddof=1)
try:
entry['corr'] = np.corrcoef(comb_treat)
except FloatingPointError:
print "error in calculating corr function"
nside = entry['mean'].shape[0]
entry['corr'] = np.zeros((nside, nside))
try:
entry['cov'] = np.cov(comb_treat)
except FloatingPointError:
print "error in calculating cov function"
nside = entry['mean'].shape[0]
entry['cov'] = np.zeros((nside, nside))
else:
entry['counts'] = counts_arr[treatment][:, 0]
entry['mean'] = comb_treat[:, 0]
entry['gauss_std'] = gerror_arr[treatment][:, 0]
entry['std'] = 0.
entry['corr'] = 0.
entry['cov'] = 0.
stat_summary[treatment] = entry
return stat_summary
def cloud_statistics(file_name):
"""
Return core duration, minimum core base, maximum core height, mean core
mass, formation time, dissipation time, maximum depth, depth evolution and
corresponding times for tracked clouds.
Parameters
----------
file_name : netCDF file name
id_profile file for a tracked core with dimensions double t(t),
double z(z).
Return
------
tuple : cloud_id, lifetime, base, top, mass, l_min, l_max, depths,
max_depth, times
"""
# Read netCDF dataset
data = Dataset(file_name)
# Core ID
cloud_id = int(file_name[-11:-3])
# Core duration (seconds)
times = data.variables['t'][...]
lifetime = len(times)*mc.dt
# Formation time, dissipation time (seconds)
l_min = times.min()*mc.dt
l_max = times.max()*mc.dt
# Minimum core base, maximum core height, maximum depth, depth evolution
# (metres)
area = ma.masked_invalid(data.variables['AREA'][...])
z = data.variables['z'][...]
z = z*np.ones(np.shape(area))
z = ma.masked_array(z, ma.getmask(area))
bases = z.min(axis=1)
tops = z.max(axis=1)
depths = tops - bases + mc.dz
max_depth = depths.max()
base = bases.min()
top = tops.max()
# Mean core mass mass (kilograms)
qn = ma.masked_invalid(data.variables['QN'][...])
rho = ma.masked_invalid(data.variables['RHO'][...])
mass = np.mean(np.sum(area*rho*mc.dz, axis=1))
# Remove missing values
times = ma.masked_array(times, ma.getmask(depths))
depths = depths[~depths.mask]
times = times[~times.mask]
data.close()
return cloud_id, lifetime, base, top, mass, l_min, l_max, depths, \
max_depth, times
def azElCenter(self):
"""
Azimuth and elevation for each pixel center.
"""
az, el = self.calculateAzEl(center=True)
az = ma.masked_invalid(az, copy=False)
el = ma.masked_invalid(el, copy=False)
return az, el
def gaussians(x,x0,A,sig):
x0m=ma.masked_invalid(np.atleast_1d(x0))
Am=ma.masked_invalid(np.atleast_1d(A))
sigm=ma.masked_invalid(np.atleast_1d(sig))
amp=Am*np.sqrt(0.5/np.pi)/sigm
[X,X0]=np.meshgrid(x,x0m)
gg=None
gg=np.einsum('...i,...ij->...j',amp,np.exp(-0.5*(X-X0)**2/np.tile(sigm**2,(np.shape(x)[0],1)).T))
return gg
def water_mass_mixing(T, S, indices):
"""
Computes the water mass mixing percentage based on water mass core
indices.
The calculations are based on the mixing triagle (Mamayev 1975).
Parameters
----------
T : array like
Temperature.
S : array like
Salinity.
indices : array like
A list/array with the core thermohaline indices for each water
mass.
Returns
-------
m1, m2, m3 : array like
Relative composition for water masses 1, 2 and 3.
Examples
--------
Reference
---------
[1] Mamayev, O. I. (Ed.). Temperature -- Salinity Analysis fo World
Ocean Waters Elsevier, 1975, 11.
Notes
-----
This function is based upon code developed by Filipe Fernandes and
available at https://ocefpaf.github.io/python4oceanographers/blog/
2014/03/24/watermass/.
"""
# Makes sure input parameters are numpy arrays
T, S, indices = asarray(T), asarray(S), asarray(indices)
# Creates linear system of three equations based on input parameters.
a = r_[indices, ones((1, 3))]
b = c_[T.ravel(), S.ravel(), ones(T.shape).ravel()].T
m = linalg.solve(a, b)
# The mixing indices
m1 = m[0].reshape(T.shape)
m2 = m[1].reshape(T.shape)
m3 = m[2].reshape(T.shape)
# Mask values outside the mising triangle.
m1 = ma.masked_outside(ma.masked_invalid(m1), 0, 1)
m2 = ma.masked_outside(ma.masked_invalid(m2), 0, 1)
m3 = ma.masked_outside(ma.masked_invalid(m3), 0, 1)
return m1, m2, m3
def __setitem__(self, key, value):
# Makes sure that value is a Variable object
if not isinstance(value, atlantis.data.Variable):
raise ValueError("`{}` is not a Variable!".format(key))
if (~isinstance(value.data, ma.MaskedArray)) & (value.data is not None):
value.data = ma.masked_invalid(value.data)
if (~isinstance(value.standard_deviation, ma.MaskedArray)) & (value.standard_deviation is not None):
value.standard_deviation = ma.masked_invalid(value.standard_deviation)
# Assigns value to class' list of items (fields)
self.fields[key] = value
def predict(self, X):
# Handle missing values
X = ma.masked_invalid(X)
mask = X.mask
dX = ma.compressed(X).reshape(-1, 1)
dZ = self.model.predict(dX)
Z = np.array([np.nan for i in xrange(X.shape[0])])
Z[~mask] = dZ
Z = ma.masked_invalid(Z)
return Z * self.step
def readrasterband(dataset, aband, NoDataVal=None, masked=True):
"""Accepts GDAL raster dataset and band number, returns Numpy 2D-array."""
if dataset.RasterCount >= aband:
# Get one band
band = dataset.GetRasterBand(aband)
# test for user specified input NoDataValue
if NoDataVal is None:
# test for band specified NoDataValue
if band.GetNoDataValue() != None:
NoDataVal = band.GetNoDataValue()
# print NoData
else:
# else set NoDataValue to be 9999.
NoDataVal = 9999
# set NoDataVal for the band (not strictly needed, but good practice if we call the band later).
band.SetNoDataValue(NoDataVal)
# create blank array (full of 0's) to hold extracted data [note Y,X format], get data type from dictionary.
datarray = np.zeros((band.YSize, band.XSize), gdt2npy[band.DataType])
# create loop based on YAxis (i.e. num rows)
for i in range(band.YSize):
# read lines of band
scanline = band.ReadRaster(0, i, band.XSize, 1, band.XSize, 1, band.DataType)
# unpack from binary representation
tuple_of_vals = struct.unpack(gdt2struct[band.DataType] * band.XSize, scanline)
# tuple_of_floats = struct.unpack('f' * band.XSize, scanline)
# add tuple to image array line by line
datarray[i, :] = tuple_of_vals
# check if masked=True
if masked is True:
# check if data type is int or float using dictionary for numeric test.
if npy2gdt[datarray.dtype.name] <= 5:
# data is integer use masked_equal
# apply NoDataValue masking.
dataraster = ma.masked_equal(datarray, NoDataVal, copy=False)
# apply invalid data masking
dataraster = ma.masked_invalid(dataraster, copy=False)
return dataraster
else:
# data is float use masked_values
dataraster = ma.masked_values(datarray, NoDataVal, copy=False)
# finaly apply mask for NaN values
dataraster = ma.masked_invalid(dataraster, copy=False)
# return array (raster)
return dataraster
else:
# user wants numpy array, no masking.
return datarray
else:
raise TypeError
def adaptive_distribution(obs, binwidth, limits=None, factor=2.0, k_max=21, right_most=3):
"""
An adaptive binning technique that overlays multiple histograms with
increasing bin widths.
References
----------
[1] Liebovitch, L. S., A. T. Todorov, M. Zochowski, D. Scheurle, L. Colgin,
M. A. Wood, K. A. Ellenbogen, J. M. Herre, and R. C. Bernstein. 1999.
``Nonlinear Properties of Cardiac Rhythm Abnormalities.''
Physical Review E 59 (3): 3312–3319.
"""
obs = nma.asanyarray(obs)
nma.masked_invalid(obs, copy=False)
obs = numpy.ravel(obs)
if obs.size == 0:
return list()
if limits is None:
mn = obs.min()
mx = obs.max()
else:
(mn, mx) = limits
offset = binwidth * 0.5
if limits is None:
limits = (mn - offset, mx + offset)
num_bins = numpy.floor((mx - mn) / binwidth)
freq = numpy.histogram(obs[~obs.mask], num_bins, range=limits)[0]
total = float(freq.sum())
print 0
print binwidth
print freq[0:right_most + 1]
results = list()
iteration = 1
while len(freq) > right_most and not (freq[1:right_most] == 0).any():
for (k, count) in enumerate(freq[1:]):
if count == 0 or k == k_max:
break
results.append(((k + 0.5) * binwidth + limits[0], count / (binwidth * total)))
binwidth *= factor
iteration += 1
# new loop
num_bins = numpy.floor((mx - mn) / binwidth)
freq = numpy.histogram(obs, num_bins, range=limits)[0]
total = float(freq.sum())
print iteration
print binwidth
print freq[0:right_most + 1]
return sorted(results)
def _accum_to_avg(accumulation):
ret = {}
for k,v in accumulation.items():
var = ma.masked_invalid(_wvar(v["weighted_flux_sum"],
v["weighted_flux_squared_sum"], v["weight_sum"],
v["weight_squared_sum"]))
ret[k] = {
"wlen": ma.masked_invalid(v["weighted_wlen_sum"] / v["weight_sum"]),
"flux": ma.masked_invalid(v["weighted_flux_sum"] / v["weight_sum"]),
"var": var,
"dflux": np.ma.sqrt(var),
"old_dflux": np.ma.sqrt(ma.masked_invalid(v["var_sum"] / v["count"])),
"count": v["count"]
}
return ret
def doResample(mapping, pxPerDeg, arcsecPerPx, containsPole):
# trigger calculation of properties so that they are not included in the timing measurements
mapping.lats
mapping.latsCenter
mapping.elevation
mapping.img
t0 = time.time()
if containsPole is None:
containsPole = mapping.containsPole
if arcsecPerPx:
pxPerDeg = plateCarreeResolution(mapping.boundingBox, arcsecPerPx)
else:
try:
_, _ = pxPerDeg
except TypeError:
assert pxPerDeg is not None
pxPerDeg = (pxPerDeg, pxPerDeg)
print('pxPerDeg: ' + str(pxPerDeg))
imgIsInt = mapping.img.dtype in [np.uint8, np.uint16, np.uint32, np.uint64, np.int8, np.int16, np.int32, np.int64]
# merge elevation with rgb array and extract channels afterwards
merged = np.dstack((mapping.img.astype(np.float64).filled(np.nan),
mapping.elevation.filled(np.nan)))
lats, lons, latsCenter, lonsCenter, merged = \
_resample(mapping.latsCenter.filled(np.nan), mapping.lonsCenter.filled(np.nan), mapping.altitude,
merged,
lambda: mapping.outline, mapping.boundingBox,
pxPerDeg, mapping.containsDiscontinuity, containsPole,
method=method)
img, elevation = np.dsplit(merged, [-1])
if imgIsInt:
with np.errstate(invalid='ignore'):
img = np.round(img)
img = np.require(ma.masked_invalid(img, copy=False), mapping.img.dtype)
if mapping.img.ndim == 2:
img = img.reshape(img.shape[0], img.shape[1])
elevation = elevation.reshape(elevation.shape[0],elevation.shape[1])
elevation = ma.masked_invalid(elevation, copy=False)
resampledMapping = mapping.createResampled(lats, lons, latsCenter, lonsCenter, elevation, img)
print('resampling:', time.time()-t0, 's')
return resampledMapping
def _make_verts(self, U, V):
uv = (U + V * 1j)
if self.angles == 'xy' and self.scale_units == 'xy':
# Here eps is 1 so that if we get U, V by diffing
# the X, Y arrays, the vectors will connect the
# points, regardless of the axis scaling (including log).
angles, lengths = self._angles_lengths(U, V, eps=1)
elif self.angles == 'xy' or self.scale_units == 'xy':
# Calculate eps based on the extents of the plot
# so that we don't end up with roundoff error from
# adding a small number to a large.
eps = np.abs(self.ax.dataLim.extents).max() * 0.001
angles, lengths = self._angles_lengths(U, V, eps=eps)
if self.scale_units == 'xy':
a = lengths
else:
a = np.absolute(uv)
if self.scale is None:
sn = max(10, math.sqrt(self.N))
if self.Umask is not ma.nomask:
amean = a[~self.Umask].mean()
else:
amean = a.mean()
scale = 1.8 * amean * sn / self.span # crude auto-scaling
# scale is typical arrow length as a multiple
# of the arrow width
if self.scale_units is None:
if self.scale is None:
self.scale = scale
widthu_per_lenu = 1.0
else:
if self.scale_units == 'xy':
dx = 1
else:
dx = self._dots_per_unit(self.scale_units)
widthu_per_lenu = dx / self._trans_scale
if self.scale is None:
self.scale = scale * widthu_per_lenu
length = a * (widthu_per_lenu / (self.scale * self.width))
X, Y = self._h_arrows(length)
if self.angles == 'xy':
theta = angles
elif self.angles == 'uv':
theta = np.angle(uv)
else:
# Make a copy to avoid changing the input array.
theta = ma.masked_invalid(self.angles, copy=True).filled(0)
theta = theta.ravel()
theta *= (np.pi / 180.0)
theta.shape = (theta.shape[0], 1) # for broadcasting
xy = (X + Y * 1j) * np.exp(1j * theta) * self.width
xy = xy[:, :, np.newaxis]
XY = np.concatenate((xy.real, xy.imag), axis=2)
if self.Umask is not ma.nomask:
XY = ma.array(XY)
XY[self.Umask] = ma.masked
# This might be handled more efficiently with nans, given
# that nans will end up in the paths anyway.
return XY
def _make_verts(self, U, V):
uv = U + V * 1j
a = np.absolute(uv)
if self.scale is None:
sn = max(10, math.sqrt(self.N))
if self.Umask is not ma.nomask:
amean = a[~self.Umask].mean()
else:
amean = a.mean()
scale = 1.8 * amean * sn / self.span # crude auto-scaling
self.scale = scale
length = a / (self.scale * self.width)
X, Y = self._h_arrows(length)
if self.angles == "xy":
theta = self._angles(U, V)
elif self.angles == "uv":
theta = np.angle(uv)
else:
theta = ma.masked_invalid(self.angles, copy=False).filled(0)
theta = theta.ravel()
theta *= np.pi / 180.0
theta.shape = (theta.shape[0], 1) # for broadcasting
xy = (X + Y * 1j) * np.exp(1j * theta) * self.width
xy = xy[:, :, np.newaxis]
XY = np.concatenate((xy.real, xy.imag), axis=2)
if self.Umask is not ma.nomask:
XY = ma.array(XY)
XY[self.Umask] = ma.masked
return XY
def quantify_effect_smoothing_freq():
"""Test how much total dissipation changes by smoothing density to
2, 3, 4, and 6 Hz"""
casts = np.r_[45:1150:3]
eps_all = np.full((4, casts.size), np.nan)
for i, cast in enumerate(casts):
try:
print(cast, end=' ')
_, data = loadMVP_m1(cast, bin_data=False)
for j, n_smooth_rho in enumerate([4, 6, 8, 12]):
eps, Lt = calc_eps(
data['p_raw'], data['prho'], data['z'],
plot_overturns=False, n_smooth_rho=n_smooth_rho)
eps_all[j, i] = eps.sum()
except IndexError:
pass
print('\n\nCompared to low-passing at 3Hz, filtering at a different freq\n'
'produces dissipation values of the following relative magnitude:')
relative_mags = [ma.mean(ma.masked_invalid(eps_all[i]/eps_all[2]))
for i in [0, 1, 3]]
print("""
6 Hz: {0:2.2f}
4 Hz: {1:2.2f}
2 Hz: {2:2.2f}
""".format(*relative_mags))
return eps_all
def main(tcu_fpath, col_name):
data = tcu_io.load_untreated_csv_to_numpy(tcu_fpath)
column = data[col_name]
dtype = column.dtype
masked = None
if dtype == 'i' or dtype == 'f':
masked = ma.masked_invalid(column)
x, y = ecdf(masked)
plt.plot(x, y, 'bo')
plt.show()
else:
#Simple hack for the string case.
#Creates a copy with masked values deleted.
masked = column[column != 'N/A']
cat, y = categorical_hist(masked)
x = range(1, len(cat) + 1)
plt.bar(x, y, width = 0.5)
plt.xticks(x, cat)
plt.show()
def plot_radials(dataset, *,
plot_type='velocity',
output_file=None,
extent=None, lon_ticks=None, lat_ticks=None,
scale=True,
sub=2,
cbar_step=10,
velocity_min=None, velocity_max=None,
markers=None,
prim_filter=False,
title='HF Radar'):
"""
param dataset: a file-path to an xarray compatible object or an xarray Dataset object
"""
try:
ds = xr.open_dataset(dataset)
closing = ds.close
except AttributeError:
if isinstance(dataset, xr.Dataset):
ds = dataset
closing = int # dummy func to close nothing
else:
raise
tds = ds.squeeze()
if prim_filter:
if 'PRIM' in list(tds.keys()):
tds = tds.where(tds.PRIM == 1).squeeze()
title = title + ' - QC'
else:
logging.warning('PRIM flag not found. Bypassing quality control filters')
time = ds.time.values[0]
lon = tds.coords['lon'].data
lat = tds.coords['lat'].data
u = tds['u'].data
v = tds['v'].data
u = ma.masked_invalid(u)
v = ma.masked_invalid(v)
if scale:
angle, speed = uv2spdir(u, v) # convert u/v to angle and speed
u, v = spdir2uv( # convert angle and speed back to u/v, normalizing the arrow sizes
np.ones_like(speed),
angle,
deg=True
)
velocity_min = velocity_min or -40
velocity_max = velocity_max or 40
kwargs = dict(extent=extent, lon_ticks=lon_ticks, lat_ticks=lat_ticks,
output_file=output_file,
sub=sub,
markers=markers,
title=title,
meshgrid=False,
scale=120, headwidth=2.5, headlength=4, headaxislength=4)
if 'velocity' in plot_type:
"""
Velocity displays the direction and magnitude of the radials
"""
kwargs['colorbar'] = True
kwargs['cmap'] = cmocean.cm.balance
# Define arrow colors. Limited by velocity_min and velocity_max
kwargs['color_clipped'] = np.clip(
tds.velocity.data[::sub],
velocity_min,
velocity_max
).squeeze()
kwargs['offset'] = Normalize(vmin=velocity_min, vmax=velocity_max, clip=True)
kwargs['ticks'] = np.append(np.arange(velocity_min, velocity_max, cbar_step), velocity_max)
elif 'motion' in plot_type:
"""
Motion displays the direction (towards or away) from radar
"""
kwargs['colorbar'] = False
kwargs['cmap'] = 'bwr'
velocity = tds.velocity
velocity_temp = velocity.where(velocity > 0, other=-1) # Going away from radar
kwargs['color_clipped'] = velocity_temp.where(velocity < 0, other=1).data # Going towards radar
kwargs['offset'] = TwoSlopeNorm(vmin=-1, vcenter=0, vmax=1)
elif 'qc_pass_fail' in plot_type:
kwargs['colorbar'] = False
kwargs['cmap'] = colors.ListedColormap(['limegreen', 'red'])
if prim_filter:
tds = ds.squeeze()
kwargs['color_clipped'] = tds.PRIM.where(tds.PRIM != 1, other=-1).data # PRIM == 1 where vectors pass qc
kwargs['offset'] = TwoSlopeNorm(vmin=-1, vcenter=0, vmax=1)
closing()
plot_common(time, lon, lat, u, v, **kwargs)
def pcolor_1D(values, y=None, **kwargs):
"""Create a pseudocolour plot of an array, Actually uses pcolormesh for speed.
Create a colour plot of an array.
If the arrays x, y, and values are geobipy.StatArray classes, the axes can be automatically labelled.
Can take any other matplotlib arguments and keyword arguments e.g. cmap etc.
Parameters
----------
values : array_like or StatArray
An array of colour values.
x : 1D array_like or StatArray
Horizontal coordinates of the values edges.
y : 1D array_like or StatArray, optional
Vertical coordinates of the values edges.
Other Parameters
----------------
log : 'e' or float, optional
Take the log of the colour to a base. 'e' if log = 'e', and a number e.g. log = 10.
Values in c that are <= 0 are masked.
equalize : bool, optional
Equalize the histogram of the colourmap so that all colours have an equal amount.
nbins : int, optional
Number of bins to use for histogram equalization.
xscale : str, optional
Scale the x axis? e.g. xscale = 'linear' or 'log'
yscale : str, optional
Scale the y axis? e.g. yscale = 'linear' or 'log'.
flipY : bool, optional
Flip the Y axis
clabel : str, optional
colourbar label
grid : bool, optional
Show the grid lines
noColorBar : bool, optional
Turn off the colour bar, useful if multiple customPlots plotting routines are used on the same figure.
Returns
-------
ax
matplotlib .Axes
See Also
--------
matplotlib.pyplot.pcolormesh : For additional keyword arguments you may use.
"""
# assert np.ndim(values) == 2, ValueError('Number of dimensions must be 2')
equalize = kwargs.pop('equalize', False)
log = kwargs.pop('log', False)
xscale = kwargs.pop('xscale', 'linear')
yscale = kwargs.pop('yscale', 'linear')
cl = kwargs.pop('clabel', None)
grid = kwargs.pop('grid', False)
flipY = kwargs.pop('flipY', False)
noColorBar = kwargs.pop('noColorBar', False)
# Set the grid colour if specified
c = None
if grid:
c = kwargs.pop('color', 'k')
# Get the figure axis
ax = plt.gca()
plt.cla()
pretty(ax)
# Set the x and y axes before meshgridding them
if (y is None):
my = np.arange(np.size(values) + 1)
mx = np.asarray([0.0, 0.1 * (np.nanmax(my) - np.nanmin(my))])
else:
assert y.size == values.size + 1, ValueError('y must be size ' +
str(values.size + 1))
#my = np.hstack([np.asarray(y), y[-1]])
my = y
mx = np.asarray([0.0, 0.1 * (np.nanmax(y) - np.nanmin(y))])
v = ma.masked_invalid(values)
if (log):
v, logLabel = _logSomething(v, log)
# Append with null values to correctly use pcolormesh
v = np.concatenate([
np.atleast_2d(np.hstack([np.asarray(v), 0])),
np.atleast_2d(np.zeros(v.size + 1))
],
axis=0)
if equalize:
nBins = kwargs.pop('nbins', 256)
assert nBins > 0, ValueError('nBins must be greater than zero')
v, dummy = histogramEqualize(v, nBins=nBins)
# Zm = ma.masked_invalid(v, copy=False)
Y, X = np.meshgrid(my, mx)
pm = plt.pcolormesh(X, Y, v, color=c, **kwargs)
ax.set_aspect('equal')
if (not flipY):
ax.invert_yaxis()
plt.yscale(yscale)
ylabel(getNameUnits(y))
ax.get_xaxis().set_ticks([])
if (not noColorBar):
if (equalize):
cbar = plt.colorbar(pm, extend='both')
else:
cbar = plt.colorbar(pm)
if cl is None:
if (log):
clabel(cbar, logLabel + getNameUnits(values))
else:
clabel(cbar, getNameUnits(values))
else:
clabel(cbar, cl)
return ax
def compute_plot_layer_corr_mat(eva_col,
evc_col,
H_col,
layernames,
titstr="BigGAN",
savestr="BigGAN",
figdir="",
use_cuda=False):
Lnum = len(evc_col)
corr_mat_lin = np.zeros((Lnum, Lnum))
corr_mat_log = np.zeros((Lnum, Lnum))
log_reg_slope = np.zeros((Lnum, Lnum))
log_reg_intcp = np.zeros((Lnum, Lnum))
for Li in range(Lnum):
eva, evc = eva_col[Li], evc_col[Li]
eva_i, evc_i = (eva, evc) if not use_cuda else \
(torch.from_numpy(eva).cuda(), torch.from_numpy(evc).cuda())
for Lj in range(Lnum): # hessian target
if H_col is not None:
H = H_col[Lj]
if use_cuda:
H = torch.from_numpy(H).cuda()
alphavec = torch.diag(evc_i.T @ H @ evc_i).cpu().numpy()
else:
alphavec = np.diag(evc_i.T @ H @ evc_i)
else:
if use_cuda:
eva_j, evc_j = torch.from_numpy(
eva_col[Lj]).cuda(), torch.from_numpy(
evc_col[Lj]).cuda()
inpr = evc_i.T @ evc_j
alphavec = torch.diag((inpr * eva_j.view(1, -1)) @ inpr.T)
alphavec = alphavec.cpu().numpy()
else:
inpr = evc_i.T @ evc_col[Lj]
# H = evc_col[Lj] @ np.diag(eva_col[Lj]) @ evc_col[Lj].T
alphavec = np.diag(
(inpr * eva_col[Lj].reshape(1, -1)) @ inpr.T)
# for Li in range(Lnum):
# evc = evc_col[Li]
# eva = eva_col[Li]
# for Lj in range(Lnum):
# H = H_col[Lj]
# alphavec = np.diag(evc.T @ H @ evc)
log10alphavec = np.log10(alphavec)
log10eva = np.log10(eva)
corr_mat_lin[Li, Lj] = np.corrcoef(alphavec, eva)[0, 1]
corr_mat_log[Li, Lj] = ma.corrcoef(
ma.masked_invalid(log10alphavec), ma.masked_invalid(log10eva))[
0, 1] #np.corrcoef(log10alphavec, log10eva)[0,1]
nanmask = (~np.isnan(log10alphavec)) * (~np.isnan(log10eva))
slope, intercept = np.polyfit(log10eva[nanmask],
log10alphavec[nanmask], 1)
log_reg_slope[Li, Lj] = slope
log_reg_intcp[Li, Lj] = intercept
fig1 = plot_layer_mat(corr_mat_lin,
layernames=layernames,
titstr="Linear Correlation of Amplification in %s" %
titstr)
fig1.savefig(join(figdir, "%s_Layer_corr_lin_mat.pdf" % savestr))
fig2 = plot_layer_mat(
corr_mat_log,
layernames=layernames,
titstr="Log scale Correlation of Amplification in %s" % titstr)
fig2.savefig(join(figdir, "%s_Layer_corr_log_mat.pdf" % savestr))
fig3 = plot_layer_mat(log_reg_slope,
layernames=layernames,
titstr="Log scale Slope of Amplification in %s" %
titstr)
fig3.savefig(join(figdir, "%s_Layer_log_reg_slope.pdf" % savestr))
fig4 = plot_layer_mat(log_reg_intcp,
layernames=layernames,
titstr="Log scale intercept of Amplification in %s" %
titstr)
fig4.savefig(join(figdir, "%s_Layer_log_reg_intercept.pdf" % savestr))
return corr_mat_lin, corr_mat_log, log_reg_slope, log_reg_intcp, fig1, fig2, fig3, fig4,
def select_behavioural(
self,
output,
method='treshold',
threshold=0.0,
percBest=0.2,
norm=True,
mask=True
): #ofwel gewoon beide op None en wat niet None, dat doorrekenen
'''
Select bevahvioural parameter sets, based on output evaluation
criterion used.
w
The algorithm makes only a 'mask' for further operation, in order to
avoid memory overload by copying matrices
Parameters
-----------
threshold :
output : ndarray
Nx1
Method can be 'treshold' based or 'percentage' based
All arrays must have same dimension in 0-direction (vertical); output-function only 1 column
InputPar is nxnpar; output is nx1; Output nxnTimesteps
this most be done for each structure independently
Output of OFfunctions/Likelihoods normalised or not (do it if different likelihoods have to be plotted together)
'''
#by using copy, reference is used and no copy (memory-use)
#keep_mask is false, so every value is used
#first mask the outputs with OF=nan or OF= inf/-inf
if mask == True:
#self.ma_output = ma.masked_invalid(output, copy=False, keep_mask = False)
#self.ma_pars = ma.masked_invalid(self.parset2run, copy=False, keep_mask = False)
self.ma_output = ma.masked_invalid(output, copy=False)
self.ma_pars = ma.masked_invalid(self.parset2run, copy=False)
else:
self.ma_output = output
self.ma_pars = self.parset2run
if method == 'treshold':
if mask == True:
self.ma_output = ma.masked_where(self.ma_output < threshold,
self.ma_output,
copy=False)
self.ma_pars = ma.masked_where(self.ma_output < threshold,
self.ma_pars,
copy=False)
else:
self.ma_output = self.ma_output < threshold, self.ma_output
self.ma_pars = self.ma_output < threshold, self.ma_pars
InputPar_Behav = self.ma_pars
output_Behav = self.ma_output
elif method == 'percentage':
nr = np.size(InputPar, 0)
ind = argsort(output) #Geeft indices van de slechtste eerst weer
NbIndic = int(round(
(1 - percBest) * ind.size)) #Percentage Indices selecteren
indBad = ind[:NbIndic] #Slechtste bepalen om weg te doen
output_Behav = np.delete(output, indBad, 0)
InputPar_Behav = np.delete(InputPar, indBad, 0)
else:
print ' Choose appropriate method: treshold or percentage'
#Normaliseren van de Objectieffunctie 1!
if norm == True:
Tempor = output_Behav.sum() / output_Behav
output_Behav = Tempor / Tempor.sum()
#print 'controle voor de som: %f' %output_Behav.sum()
return [InputPar_Behav, output_Behav]
Rad = np.load(
'/home/nacorreasa/Maestria/Datos_Tesis/Arrays/Array_Rad_2018_2019CH2.npy')
fechas_horas = np.load(
'/home/nacorreasa/Maestria/Datos_Tesis/Arrays/Array_FechasHoras_CH2__2018_2019.npy'
)
fechas_horas_GOES = pd.to_datetime(fechas_horas,
format="%Y-%m-%d %H:%M",
errors='coerce')
lat_index_o = np.load(
'/home/nacorreasa/Maestria/Datos_Tesis/Arrays/Array_Lat_index_Malla.npy')
lon_index_o = np.load(
'/home/nacorreasa/Maestria/Datos_Tesis/Arrays/Array_Lon_index_Malla.npy')
mx_lat_index = ma.masked_invalid(lat_index_o)
mx_lon_index = ma.masked_invalid(lon_index_o)
lat_index = mx_lat_index.astype('int')
lon_index = mx_lon_index.astype('int')
##############################################################################################
## ----------------DEFINICION DE LAS ACCIONES PARA LA SELECCION DE DATOS------------------- ##
##############################################################################################
R = np.zeros(Rad.shape, dtype=float) * np.nan
fechas_horas_new = []
raros = []
blist = []
for g in range(len(fechas_horas_GOES)):
try:
n1deflec = np.empty((nconds, nses))
n1sub = np.empty((nconds, nses))
condmat = np.empty((nconds, nses))
submat = np.empty((nconds, nses))
n1sub[:] = np.NAN
n1deflec[:] = np.NAN
condmat[:] = np.NAN
submat[:] = np.NAN
for n in range(0, n1lat.shape[0]):
n1sub[conds[n] - 1, sescount[n] - 1] = n1lat[n]
n1deflec[conds[n] - 1, sescount[n] - 1] = n1onset[n]
condmat[conds[n] - 1, sescount[n] - 1] = conds[n]
submat[conds[n] - 1, sescount[n] - 1] = sescount[n]
n1deflec /= 1000 #Convert from ms to seconds
n1deflec = ma.masked_invalid(n1deflec) # Remove NANs for pyjags
n1sub /= 1000 #Convert from ms to seconds
n1sub = ma.masked_invalid(n1sub) # Remove NANs for pyjags
# Initialize non-decision time with mininum RT for each subject and condition
minrt = np.empty((nconds, nses))
for k in range(0, nconds):
for j in range(0, nses):
where = (sessioncount == (j + 1)) & (condition == (k + 1))
minrt[k, j] = np.min(np.abs(y[where]))
# pyjags code
# Make sure $LD_LIBRARY_PATH sees /usr/local/lib
pyjags.modules.load_module('wiener')
pyjags.modules.load_module('dic')
def evaluate_target_regression_epoch(regressor,
dataloader,
device,
history,
output_folder=None,
seed=None,
cv_flag=False):
y_truths = None
y_preds = None
regressor.eval()
for x_batch, y_batch in dataloader:
x_batch = x_batch.to(device)
y_batch = y_batch.to(device)
with torch.no_grad():
y_truths = np.vstack([
y_truths, y_batch.cpu().detach().numpy()
]) if y_truths is not None else y_batch.cpu().detach().numpy()
y_pred = regressor(x_batch).detach()
y_preds = np.vstack([
y_preds, y_pred.cpu().detach().numpy()
]) if y_preds is not None else y_pred.cpu().detach().numpy()
assert (y_truths.shape == y_preds.shape)
if output_folder is not None:
# output prediction
if cv_flag:
pd.DataFrame(y_truths).to_csv(
f'{output_folder}/cv_truths_{seed}.csv')
pd.DataFrame(y_preds).to_csv(
f'{output_folder}/cv_preds_{seed}.csv')
else:
pd.DataFrame(y_truths).to_csv(f'{output_folder}/truths_{seed}.csv')
pd.DataFrame(y_preds).to_csv(f'{output_folder}/preds_{seed}.csv')
else:
history['dpearsonr'].append(
np.mean(
np.abs([
pearsonr(
y_truths[:,
i][~ma.masked_invalid(y_truths[:, i]).mask],
y_preds[:,
i][~ma.masked_invalid(y_truths[:, i]).mask])[0]
for i in range(y_truths.shape[1])
])).item())
history['cpearsonr'].append(
np.mean(
np.abs([
pearsonr(
y_truths[i, :]
[~ma.masked_invalid(y_truths[i, :]).mask], y_preds[
i, :][~ma.masked_invalid(y_truths[i, :]).mask])[0]
for i in range(y_truths.shape[0])
])).item())
history['dspearmanr'].append(
np.mean(
np.abs([
spearmanr(
y_truths[:,
i][~ma.masked_invalid(y_truths[:, i]).mask],
y_preds[:,
i][~ma.masked_invalid(y_truths[:, i]).mask])[0]
for i in range(y_truths.shape[1])
])).item())
history['cspearmanr'].append(
np.mean(
np.abs([
spearmanr(
y_truths[i, :]
[~ma.masked_invalid(y_truths[i, :]).mask], y_preds[
i, :][~ma.masked_invalid(y_truths[i, :]).mask])[0]
for i in range(y_truths.shape[0])
])).item())
# history['cpearsonr'].append(pd.DataFrame(y_truths).corrwith(pd.DataFrame(y_preds), axis=1).mean())
# history['dpearsonr'].append(pd.DataFrame(y_truths).corrwith(pd.DataFrame(y_preds), axis=0).mean())
history['drmse'].append(
np.mean(np.nanmean(np.square((y_truths - y_preds)),
axis=0)).item())
history['crmse'].append(
np.mean(np.nanmean(np.square((y_truths - y_preds)),
axis=1)).item())
# history['pearsonr'].append(pearsonr(y_truths, y_preds)[0])
# history['spearmanr'].append(spearmanr(y_truths, y_preds)[0])
# history['r2'].append(r2_score(y_true=y_truths, y_pred=y_preds))
# history['rmse'].append(mean_squared_error(y_true=y_truths, y_pred=y_preds, squared=False))
return history
nc_b = Dataset('Greenland_bedrock_topography_V3.nc', 'r+')
nc_o = Dataset('Racmo2MeanSMB_1961-1990.nc', 'r')
#Read the target grid
x_b = nc_b.variables['projection_x_coordinate'][:]
y_b = nc_b.variables['projection_y_coordinate'][:]
nx_b = x_b.shape[0]
ny_b = y_b.shape[0]
#List of variables to merge in
varlist_o = ['smb']
varlist_b = ['acab']
vdata_o = ndarray((ny_b, nx_b))
for v in range(len(varlist_o)):
vdata_o[:, :] = 0.
ncvar_o = nc_o.variables[varlist_o[v]]
vdata_o[:, :] = ncvar_o[:, ::-1].transpose() / 910.
vdata_o = ma.masked_invalid(vdata_o)
vdata_o = vdata_o.filled(0.)
ncvar_b = nc_b.createVariable(varlist_b[v], 'f4', (
'y',
'x',
))
ncvar_b[:, :] = vdata_o[:, :]
copy_atts(ncvar_o, ncvar_b)
nc_b.close()
nc_o.close()
def plot_selection(alldomain=False):
plt.figure(figsize=(6.0,4.0))
ax1 = plt.subplot(1,1,1)
nc = Dataset(simul.ncfile, 'r')
if alldomain:
if adv3d or advdepth==0 or advsurf:
if 'surf' in simul.simul:
sst = np.squeeze(simul.Forder(nc.variables['temp'][simul.infiletime,:,:]))
else:
sst = np.squeeze(simul.Forder(nc.variables['temp'][simul.infiletime,-1,:,:]))
else:
[z_r,z_w] = part.get_depths(simul)
temp = simul.Forder(np.squeeze(nc.variables['temp'][simul.infiletime,:,:,:]))
sst = part.vinterp(temp, [advdepth], z_r,z_w)[:,:,0]
if not light: topo = simul.topo
else:
[ny1,ny2,nx1,nx2] = np.array(coord) #-ng
if adv3d or advdepth==0 or advsurf:
if 'surf' in simul.simul:
sst = np.squeeze(simul.Forder(nc.variables['temp'][simul.infiletime,ny1:ny2,nx1:nx2]))
else:
sst = np.squeeze(simul.Forder(nc.variables['temp'][simul.infiletime,-1,ny1:ny2,nx1:nx2]))
else:
[z_r,z_w] = part.get_depths(simul,coord=[ny1,ny2,nx1,nx2])
temp = simul.Forder(np.squeeze(nc.variables['temp'][simul.infiletime,:,ny1:ny2,nx1:nx2]))
sst = part.vinterp(temp, [advdepth], z_r,z_w)[:,:,0]
if not light: topo = simul.topo[nx1:nx2,ny1:ny2]
nc.close()
#sst[sst<=0] = np.nan
#plt.imshow(sst[:,::1].T); plt.colorbar(shrink=0.25);
plt.pcolormesh(ma.masked_invalid(sst[:,:].T),cmap='jet',rasterized=True);
plt.colorbar(shrink=0.25);
if not advsurf and not adv3d:
plt.contourf(topo.T,[0,-advdepth],
cmap = col.LinearSegmentedColormap.from_list('my_colormap',
['white','lightgray'],256))
plt.contourf(topo.T,[0,topo.min()],
cmap = col.LinearSegmentedColormap.from_list('my_colormap',
['Gainsboro','gray'],256))
plt.contour(topo.T,[topo.min()],colors=('k',),linewidths=(0.5,));
plt.contour(topo.T,[-advdepth],colors=('k',),linewidths=(0.2,));
if alldomain:
plt.plot(px[::1]+0.5,(py[::1]+0.5),'o', markersize=2,
markeredgecolor='k', markerfacecolor='k');
plt.axis([0,sst.shape[0]-1,0,sst.shape[1]-1]);
else:
plt.plot(px[::1]-coord[2]+0.5,(py[::1]-coord[0]+0.5),'o', markersize=1,
markerfacecolor='white');
#plt.axis('scaled');
#plt.axis([nx1-coord[2]+0.5,nx2-coord[2]+0.5,ny1-coord[0]+0.5,ny2-coord[0]+0.5]);
'''
for isub,jsub in product(range(nsub_x),range(nsub_y)):
try:
plot_rect(subtightcoord_saves[jsub+isub*nsub_y],'r')
plot_rect(subcoord_saves[jsub+isub*nsub_y],'k--')
except:
print 'no subdomains to plot'
'''
color = 'w'; box = 'round,pad=0.1'; props = dict(boxstyle=box, fc=color, ec='k', lw=1, alpha=1.)
plt.title(format(np.sum(px>0)) + ' pyticles ' )
plt.savefig(folderout + simulname + '_' + format(nproc) + '_' + '{0:04}'.format(time+dfile) +'.png', dpi=150,bbox_inches='tight'); plt.clf()
# split data where longitude increment is larger than median increment + 2 standard deviations
lon_diff = np.diff(lon)
# files are not always split, so check whether discontinuity in lon exists
try:
lon_split = int(
np.where(lon_diff > (np.median(lon_diff) +
2 * np.std(lon_diff)))[0])
split = True
# if not, don't split
except:
split = False
split = False
soil_moisture = nc["soil_moisture"][:]
soil_moisture[soil_moisture > 100.] = np.nan
soil_moisture = ma.masked_invalid(soil_moisture)
# remap soil moisture on regular lat/lon grid
data_in = pd.DataFrame({
'lat': lat_m.flatten(),
'lon': lon_m.flatten(),
'soil_moisture': soil_moisture.flatten()
})
target_grid = lat_lon_grid(globals.min_lat,
globals.min_lon,
globals.max_lat,
globals.max_lon,
delLat=globals.del_lat,
delLon=globals.del_lon)
target_grid.build_grid()
max_distance = get_max_distance()
def fill_gradient(self, canvas, X, percentiles, color=Tango.colorsHex['mediumBlue'], label=None, **kwargs):
ax = canvas
plots = []
if 'edgecolors' not in kwargs:
kwargs['edgecolors'] = 'none'
if 'facecolors' in kwargs:
color = kwargs.pop('facecolors')
if 'array' in kwargs:
array = kwargs.pop('array')
else:
array = 1.-np.abs(np.linspace(-.97, .97, len(percentiles)-1))
if 'alpha' in kwargs:
alpha = kwargs.pop('alpha')
else:
alpha = .8
if 'cmap' in kwargs:
cmap = kwargs.pop('cmap')
else:
cmap = LinearSegmentedColormap.from_list('WhToColor', (color, color), N=array.size)
cmap._init()
cmap._lut[:-3, -1] = alpha*array
kwargs['facecolors'] = [cmap(i) for i in np.linspace(0,1,cmap.N)]
# pop where from kwargs
where = kwargs.pop('where') if 'where' in kwargs else None
# pop interpolate, which we actually do not do here!
if 'interpolate' in kwargs: kwargs.pop('interpolate')
def pairwise(iterable):
"s -> (s0,s1), (s1,s2), (s2, s3), ..."
from itertools import tee
#try:
# from itertools import izip as zip
#except ImportError:
# pass
a, b = tee(iterable)
next(b, None)
return zip(a, b)
polycol = []
for y1, y2 in pairwise(percentiles):
import matplotlib.mlab as mlab
# Handle united data, such as dates
ax._process_unit_info(xdata=X, ydata=y1)
ax._process_unit_info(ydata=y2)
# Convert the arrays so we can work with them
from numpy import ma
x = ma.masked_invalid(ax.convert_xunits(X))
y1 = ma.masked_invalid(ax.convert_yunits(y1))
y2 = ma.masked_invalid(ax.convert_yunits(y2))
if y1.ndim == 0:
y1 = np.ones_like(x) * y1
if y2.ndim == 0:
y2 = np.ones_like(x) * y2
if where is None:
where = np.ones(len(x), np.bool)
else:
where = np.asarray(where, np.bool)
if not (x.shape == y1.shape == y2.shape == where.shape):
raise ValueError("Argument dimensions are incompatible")
from functools import reduce
mask = reduce(ma.mask_or, [ma.getmask(a) for a in (x, y1, y2)])
if mask is not ma.nomask:
where &= ~mask
polys = []
for ind0, ind1 in mlab.contiguous_regions(where):
xslice = x[ind0:ind1]
y1slice = y1[ind0:ind1]
y2slice = y2[ind0:ind1]
if not len(xslice):
continue
N = len(xslice)
p = np.zeros((2 * N + 2, 2), np.float)
# the purpose of the next two lines is for when y2 is a
# scalar like 0 and we want the fill to go all the way
# down to 0 even if none of the y1 sample points do
start = xslice[0], y2slice[0]
end = xslice[-1], y2slice[-1]
p[0] = start
p[N + 1] = end
p[1:N + 1, 0] = xslice
p[1:N + 1, 1] = y1slice
p[N + 2:, 0] = xslice[::-1]
p[N + 2:, 1] = y2slice[::-1]
polys.append(p)
polycol.extend(polys)
from matplotlib.collections import PolyCollection
if 'zorder' not in kwargs:
kwargs['zorder'] = 0
plots.append(PolyCollection(polycol, **kwargs))
ax.add_collection(plots[-1], autolim=True)
ax.autoscale_view()
return plots
lat_aux = iris.coords.AuxCoord(lat.data,
standard_name=lat.name(),
units=lat.units)
lon_aux = iris.coords.AuxCoord(lon.data,
standard_name=lon.name(),
units=lon.units)
# Add latitude and longitude as coordinates
mean_cube.add_aux_coord(lat_aux, data_dims=(0))
mean_cube.add_aux_coord(lon_aux, data_dims=(1))
iris.util.promote_aux_coord_to_dim_coord(mean_cube, 'latitude')
iris.util.promote_aux_coord_to_dim_coord(mean_cube, 'longitude')
# regrid the cube
rh_cube = regrid_model(mean_cube, regrid_modelcube)
rh_data_regridded = rh_cube.data
rh_data_regridded = rh_data_regridded * 1e-3 * 365
rh_data_regridded = ma.masked_invalid(rh_data_regridded)
rh_data_regridded = np.ma.masked_where(rh_data_regridded <= 0,
rh_data_regridded)
historical_observational_rh = rh_data_regridded.copy()
#%% tau calculation
tau_s = merged_hwsd_ncscd_masked / rh_data_regridded
tau_s_masked = ma.masked_where(np.logical_or(tau_s < 1, tau_s > 1e4), tau_s)
obs_temp = ma.masked_where(np.logical_or(tau_s < 1, tau_s > 1e4), obs_temp)
#%%
# FIGURE 2a
ax = fig_figure1.add_subplot(gs[row, column], projection=ccrs.PlateCarree())
# Add lat/lon grid lines to the figure
def make_measurement(datafile,
error,
outfile,
rms=None,
masknan=True,
overwrite=False,
unit="adu"):
"""Create a FITS files with 2 HDUS, the first being the datavalue and the 2nd being
the data uncertainty. This format makes allows the resulting file to be read into the underlying :class:'~astropy.nddata.CCDData` class.
:param datafile: The FITS file containing the data as a function of spatial coordinates
:type datafile: str
:param error: The errors on the data Possible values for error are:
- a filename with the same shape as datafile containing the error values per pixel
- a percentage value 'XX%' must have the "%" symbol in it
- 'rms' meaning use the rms parameter if given, otherwise look for the RMS keyword in the FITS header of the datafile
:type error: str
:param outfile: The output file to write the result in (FITS format)
:type outfile: str
:param rms: If error == 'rms', this value may give the rms in same units as data (e.g 'erg s-1 cm-2 sr-1').
:type rms: float or :class:`astropy.units.Unit`
:param masknan: Whether to mask any pixel where the data or the error is NaN. Default:true
:type masknan: bool
:param overwrite: If `True`, overwrite the output file if it exists. Default: `False`.
:type overwrite: bool
:param unit: Intensity unit to use for the data, this will override BUNIT in header if present.
:type unit: :class:`astropy.units.Unit` or str
:raises Exception: on various FITS header issues
:raises OSError: if `overwrite` is `False` and the output file exists.
Example usage:
.. code-block:: python
# example with percentage error
Measurement.make_measurement("my_infile.fits",error='10%',outfile="my_outfile.fits")
# example with measurement in units of K km/s and error
# indicated by RMS keyword in input file.
Measurement.make_measurement("my_infile.fits",error='rms',outfile="my_outfile.fits",unit="K km/s",overwrite=True)
"""
_data = fits.open(datafile)
needsclose = False
if error == 'rms':
_error = deepcopy(_data)
if rms is None:
rms = _data[0].header.get("RMS", None)
if rms is None:
raise Exception(
"rms not given as parameter and RMS keyword not present in data header"
)
else:
print("Found RMS in header: %.2E %s" %
(rms, _error[0].data.shape))
tmp = np.full(_error[0].data.shape, rms)
_error[0].data[:] = rms
elif "%" in error:
percent = float(error.strip('%')) / 100.0
_error = deepcopy(_data)
_error[0].data = _data[0].data * percent
else:
_error = fits.open(error)
needsclose = True
fb = _data[0].header.get('bunit',
str(unit)) #use str in case Unit was given
eb = _error[0].header.get('bunit', str(unit))
if fb != eb:
raise Exception(
"BUNIT must be the same in both data (%s) and error (%s) maps"
% (fb, eb))
# Sigh, this is necessary since there is no mode available in
# fits.open that will truncate an existing file for writing
if overwrite and exists(outfile):
remove(outfile)
_out = fits.open(name=outfile, mode="ostream")
_out.append(_data[0])
_out[0].header['bunit'] = fb
_out.append(_error[0])
_out[1].header['extname'] = 'UNCERT'
_out[1].header['bunit'] = eb
_out[1].header['utype'] = 'StdDevUncertainty'
if masknan:
fmasked = ma.masked_invalid(_data[0].data)
emasked = ma.masked_invalid(_error[0].data)
final_mask = utils.mask_union([fmasked, emasked])
# Convert boolean mask to uint since io.fits cannot handle bool.
hduMask = fits.ImageHDU(final_mask.astype(np.uint8), name='MASK')
_out.append(hduMask)
_out.writeto(outfile, overwrite=overwrite)
_data.close()
_out.close()
if needsclose: _error.close()
def generate_param_colour_plot(F, ax, scale):
def fmt(x):
a, b = '{:.2e}'.format(x).split('e')
b = int(b)
return r'${} \times 10^{{{}}}$'.format(a, b)
text = "15 km, 0 rptrs * NP, T=1e-2-1e1, P = 0.9-0.95, march 6 *.pkl"
x = 20
y = 20
files = glob.glob(text)
files.sort(key=lambda x: [int(s) for s in x.split() if s.isdigit()][2])
rates = [[0 for i in range(x)] for j in range(y)]
counter1 = 0
for file in files:
with open(file, 'rb') as f:
stored_path_dict = pickle.load(f)
if stored_path_dict:
fidelities, opt_times, opt_schemes = collect_optimal_protocols(G=None, Alice=None, Bob=None,
stored_path_dict=stored_path_dict)
try:
# index of the smallest fidelity greater than F
index = next(x[0] for x in enumerate(fidelities) if x[1] >= F)
max_rate = 1/opt_times[index]
except StopIteration: # exception occurs when there is no scheme that achieves F, return infinite time
max_rate = 0
counter = [int(s) for s in file.split() if s.isdigit()][2] - 1
print(counter)
print(file)
rates[counter % y][counter // y] = max_rate
counter1 += 1
plt.set_cmap('RdPu')
current_cmap = matplotlib.cm.get_cmap()
current_cmap.set_bad(color='black')
try:
min_time = np.nanmin(np.nanmin(rates))
max_time = np.nanmax(np.nanmax(rates))
except:
min_time, max_time = 0, 0
times = masked_invalid(rates)
current_cmap = matplotlib.cm.get_cmap()
current_cmap.set_bad(color='black', alpha=1.)
skr_grad = np.gradient(ndimage.gaussian_filter(times, sigma=0.35, order=0))
lengths = [[np.power(0.00001+skr_grad[0][j][i]**2 + skr_grad[1][j][i]**2, 1/2)
if np.isfinite(skr_grad[0][j][i]) or np.isfinite(skr_grad[1][j][i])
else 1 for i in range(x)] for j in range(y)]
skr_grad[0] = skr_grad[0]/lengths
skr_grad[1] = skr_grad[1]/lengths
ax.quiver(skr_grad[1], -skr_grad[0], scale=scale, alpha=0.8)
im = ax.matshow(rates, cmap=current_cmap, interpolation=None)
t = np.linspace(min_time, max_time, 5)
cb = plt.colorbar(im, ax=ax, ticks=t, format=ticker.FuncFormatter(fmt), fraction=0.046, pad=0.04)
cb.ax.tick_params(labelsize=11)
cb.ax.set_title('Generation rate', fontsize=10)
corr_all=[]
for j in range(np.size(drug_data,1)):
corr_drug=[] ####### removing cell lines with response NaN
y1 = drug_data[:,j]
a=[i for i, x in enumerate(y1) if not str(x).replace('.','').isdigit()]
y=np.delete(y1,a)
X=np.delete(X1,a,axis=0)
for i in range(iteration):
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=i)
if FS==1:
cc = np.zeros((1,np.size(X,1)))
for j in range(np.size(X,1)):
D= pearsonr(ma.masked_invalid(X_train[:,j]), ma.masked_invalid(y_train))[0]
cc[0,j] = D
cc = np.nan_to_num(cc)
F = normalnp(X, alpha, cc)
sort=np.argsort(F,0)
sort=sort[::-1][:,0]
index=np.concatenate([sort[0:cutoff//2],sort[-cutoff//2:]])
elif FS==0:
CC=[pearsonr(X_train[:, i], y_train)[0] for i in range(np.size(X_train,1))]
sort=np.argsort(np.abs(CC))
sort=sort[::-1]
index=sort[0:cutoff]
else:
print('Invalid feature selection')
def generate_param_colour_plot_throughput(ax):
def fmt(x):
a, b = '{:.2e}'.format(x).split('e')
b = int(b)
return r'${} \times 10^{{{}}}$'.format(a, b)
def binary_entropy(x):
if x not in [0, 1]:
return -x*np.log2(x)-(1-x)*np.log2(1-x)
else:
return 0
def calc_throughput(scheme):
state = scheme.state
time = scheme.time
simulation_type = "IP"
if simulation_type == "IP":
proj1 = qt.Qobj([[1, 0, 0, 0],
[0, 0, 0, 0],
[0, 0, 0, 0],
[0, 0, 0, 1]])
proj2 = (1/2)*qt.Qobj([[1, 0, 0, 1],
[0, 1, 1, 0],
[0, 1, 1, 0],
[1, 0, 0, 1]])
proj3 = (1/2)*qt.Qobj([[1, 0, 0, -1],
[0, 1, 1, 0],
[0, 1, 1, 0],
[-1, 0, 0, 1]])
state00 = qt.Qobj([[1/2, 0, 0, 1/2],
[0, 0, 0, 0],
[0, 0, 0, 0],
[1/2, 0, 0, 1/2]])
state01 = qt.Qobj([[1/2, 0, 0, -1/2],
[0, 0, 0, 0],
[0, 0, 0, 0],
[-1/2, 0, 0, 1/2]])
state10 = (1/2)*qt.Qobj([[0, 0, 0, 0],
[0, 1, 1, 0],
[0, 1, 1, 0],
[0, 0, 0, 0]])
state11 = (1/2)*qt.Qobj([[0, 0, 0, 0],
[0, 1, -1, 0],
[0, -1, 1, 0],
[0, 0, 0, 0]])
proj1.dims = [[2, 2], [2, 2]]
proj2.dims = [[2, 2], [2, 2]]
proj3.dims = [[2, 2], [2, 2]]
state00.dims = [[2, 2], [2, 2]]
state01.dims = [[2, 2], [2, 2]]
state10.dims = [[2, 2], [2, 2]]
state11.dims = [[2, 2], [2, 2]]
elif simulation_type == "MP":
proj1 = qt.ket2dm(qt.basis(81, 28)) + qt.ket2dm(qt.basis(81, 12))
proj2 = qt.ket2dm(qt.basis(81, 28) + qt.basis(81, 12)) / 2 + \
qt.ket2dm(qt.basis(81, 28) - qt.basis(81, 12)) / 2
proj3 = qt.ket2dm(qt.basis(81, 28) + 1j*qt.basis(81, 12)) / 2 + \
qt.ket2dm(qt.basis(81, 28) - 1j*qt.basis(81, 12)) / 2
vec01 = qt.basis(9, 1)
vec10 = qt.basis(9, 3)
state00 = qt.ket2dm(qt.tensor(vec01, vec10) + qt.tensor(vec10, vec01))/2
state01 = qt.ket2dm(qt.tensor(vec01, vec10) - qt.tensor(vec10, vec01))/2
state10 = qt.ket2dm(qt.tensor(vec01, vec01) + qt.tensor(vec10, vec10))/2
state11 = qt.ket2dm(qt.tensor(vec01, vec01) - qt.tensor(vec10, vec10))/2
proj1.dims = [[9, 9], [9, 9]]
proj2.dims = [[9, 9], [9, 9]]
proj3.dims = [[9, 9], [9, 9]]
state00.dims = [[9, 9], [9, 9]]
state01.dims = [[9, 9], [9, 9]]
state10.dims = [[9, 9], [9, 9]]
state11.dims = [[9, 9], [9, 9]]
bb84 = False
if bb84:
qber1 = (state * proj1).tr()
qber2 = (state * proj2).tr()
skf = max(1 - binary_entropy(qber1) - binary_entropy(qber2), 0) # calc secret_key fraction
else:
p00 = (state * state00).tr()
p01 = (state * state01).tr()
p10 = (state * state10).tr()
p11 = (state * state11).tr()
normalisation = p00+p01+p10+p11
# define P' distribution as in https://iopscience.iop.org/article/10.1088/2058-9565/aab31b, eq D1 to D8
Px0 = (p00+p01)**2 + (p10+p11)**2
Px1 = 1-Px0
p00n = (p00**2+p01**2)/Px0
p01n = (2*p00*p01)/Px0
p10n = (p10**2+p11**2)/Px0
p11n = (2*p10*p11)/Px0
if ((p00 + p01) * (p10 + p11)) == 0:
Pbinary_entropy = 1
else:
Pbinary_entropy = (p00 * p10 + p01 * p11) / ((p00 + p01) * (p10 + p11))
skf = max(1-entropy([p00, p01, p10, p11], base=2)+(Px1/2)*binary_entropy(Pbinary_entropy),
(Px0/2)*(1-entropy([p00n, p01n, p10n, p11n], base=2)), 0)
return skf/time
text = "50 km, 1 rptrs * G=0.98-1 in 20, P=* in 20, eps = 0.01, IP, 1 node, 50km, 8 feb, *.pkl"
x = 20
y = 20
files = glob.glob(text)
files = [file for file in files if "pessimistic" not in file]
files.sort(key=lambda x: [int(s) for s in x.split() if s.isdigit()][2])
files.sort(key=lambda x: [int(s) for s in x.split() if s.isdigit()][1])
skr = [[0 for i in range(x)] for j in range(y)]
for file in files:
print(file)
counter = 0
for file in files:
with open(file, 'rb') as f:
stored_path_dict = pickle.load(f)
if stored_path_dict:
fidelities, opt_times, opt_schemes = collect_optimal_protocols(G=None, Alice=None, Bob=None,
stored_path_dict=stored_path_dict)
throughputs = [calc_throughput(scheme) for scheme in opt_schemes]
max_throughput = max(throughputs)
counter = [int(s) for s in file.split() if s.isdigit()][2] - 1
skr[counter % y][counter // y] = max_throughput
counter += 1
print(counter)
plt.set_cmap('RdPu')
current_cmap = matplotlib.cm.get_cmap()
current_cmap.set_bad(color='black')
try:
min_time = np.nanmin(np.nanmin(skr))
max_time = np.nanmax(np.nanmax(skr))
except:
min_time, max_time = 0, 0
skr = masked_invalid(skr)
current_cmap = matplotlib.cm.get_cmap()
current_cmap.set_bad(color='black', alpha=1.)
skr_grad = np.gradient(skr)
lengths = [[np.power(0.001+skr_grad[0][j][i]**2 + skr_grad[1][j][i]**2, 1/2)
if np.isfinite(skr_grad[0][j][i]) or np.isfinite(skr_grad[1][j][i])
else 1 for i in range(x)] for j in range(y)]
maxlength = max(max(lengths))*1.5
skr_grad[0] = skr_grad[0]/maxlength
skr_grad[1] = skr_grad[1]/maxlength
plt.quiver(skr_grad[1], -skr_grad[0])
im = ax.matshow(skr, cmap=current_cmap, interpolation=None)
plt.rc('text', usetex=True)
t = np.linspace(min_time, max_time, 5)
cb = plt.colorbar(im, ax=ax, ticks=t, format=ticker.FuncFormatter(fmt), fraction=0.046, pad=0.04)
cb.ax.tick_params(labelsize=16)
cb.ax.set_title('Secret-key rate', fontsize=17, pad=12)
#trend_io_3m = xr.full_like(trend_io,3*np.mean(trend_io))
#trend_io_3m = mask_oceans(args.imask, trend_io_3m, lon, lat)
# Damp the edges
trend_io_dmp = damp(trend_io, lon, lat)
#trend_io_dmp = trend_io_dmp.values
#Calculate global climatology
clim = clim_anom_xr(sst_t, time_t, start=None)[0]
clim = clim.values
#clip T
clim[clim < -1.77] = -1.77
#Interpolate sst
clim_int = np.zeros(clim.shape)
for i in range(clim.shape[0]):
clim_int[i], converged = fill(ma.masked_invalid(clim[i]), 1, 0, **kw)
#Repeat the climatology 68 times (1950-2017)
nyears = (end_time - start_time) + 1
clim_rpt = np.repeat(clim_int[np.newaxis,:,:,:],nyears,axis=0)\
.reshape(clim_int.shape[0]*nyears,clim_int.shape[1],clim_int.shape[2])
# Repeat IO trend 68 times (1950-1975)
trend_io_rpt = np.repeat(trend_io_int[np.newaxis, :, :],
nyears * 12,
axis=0)
# Create an array to multiply rates with
multiplier = np.arange(1, (nyears * 12) + 1, 1)[:, np.newaxis, np.newaxis]
# Multiply rate with multiplier to get transient trends
trend_io_trans = np.multiply(trend_io_rpt, multiplier)
def scatter(x, y, z, ax=None, robust=False, **kwargs):
"""
Plot a scatter section plot of a small dataset (< 10 000 obs)
Parameters
----------
x : array, dtype=float, shape=[n, ]
continuous horizontal variable (e.g. time, lat, lon)
y : array, dtype=float, shape=[n, ]
continous vertical variable (e.g. depth, density)
z : array, dtype=float, shape=[n, ]
ungridded data variable
ax : matplotlib.axes
a predefined set of axes to draw on
robust : bool=False
if True, uses the 0.5 and 99.5 percentile to set color limits
kwargs : any key:values pair that gets passed to plt.pcolormesh.
Returns
-------
axes
Raises
------
will ask if you want to continue if more than 10000 points
"""
from matplotlib.pyplot import colorbar, subplots
from numpy import (
ma,
nanpercentile,
datetime64,
array,
nanmin,
nanmax,
isnan,
)
from datetime import datetime
z = ma.masked_invalid(z)
m = ~(z.mask | isnan(y))
z = z[m]
x = array(x)[m]
y = array(y)[m]
if y.size >= 1e5:
carry_on = input(
'There are a large number of points to plot ({}). '
'This will take a while to plot.\n'
'Type "y" to continue or "n" to cancel.\n'.format(y.size))
if carry_on != 'y':
print('You have aborted the scatter plot')
return None
x_time = isinstance(x[0], (datetime, datetime64))
if robust:
kwargs['vmin'] = nanpercentile(z, 0.5)
kwargs['vmax'] = nanpercentile(z, 99.5)
if ax is None:
fig, ax = subplots(1, 1, figsize=[9, 3.5], dpi=90)
else:
fig = ax.get_figure()
im = ax.scatter(x, y, c=z, rasterized=True, **kwargs)
ax.cb = colorbar(mappable=im, pad=0.02, ax=ax, fraction=0.05)
ax.set_xlim(x.min(), x.max())
ax.set_ylim(nanmax(y), nanmin(y))
ax.set_ylabel('Depth (m)')
ax.set_xlabel('Date' if x_time else 'Dives')
[tick.set_rotation(45) for tick in ax.get_xticklabels()]
fig.tight_layout()
return ax
def compute_hess_corr(eigval_col,
eigvec_col,
savelabel="",
figdir="",
use_cuda=False):
"""cuda should be used for large mat mul like 512 1024 4096.
small matmul should stay with cpu numpy computation. cuda will add the IO overhead."""
posN = len(eigval_col)
T0 = time()
if use_cuda:
corr_mat_log = torch.zeros((posN, posN)).cuda()
corr_mat_lin = torch.zeros((posN, posN)).cuda()
for eigi in tqdm(range(posN)):
evc_i, eva_i = torch.from_numpy(
eigvec_col[eigi]).cuda(), torch.from_numpy(
eigval_col[eigi]).cuda()
for eigj in range(posN):
evc_j, eva_j = torch.from_numpy(
eigvec_col[eigj]).cuda(), torch.from_numpy(
eigval_col[eigj]).cuda()
inpr = evc_i.T @ evc_j
vHv_ij = torch.diag((inpr * eva_j.unsqueeze(0)) @ inpr.T)
corr_mat_log[eigi,
eigj] = corr_nan_torch(vHv_ij.log10(),
eva_j.log10())
corr_mat_lin[eigi, eigj] = corr_nan_torch(vHv_ij, eva_j)
corr_mat_log = corr_mat_log.cpu().numpy()
corr_mat_lin = corr_mat_lin.cpu().numpy()
else:
corr_mat_log = np.zeros((posN, posN))
corr_mat_lin = np.zeros((posN, posN))
for eigi in tqdm(range(posN)):
eva_i, evc_i = eigval_col[eigi], eigvec_col[eigi]
for eigj in range(posN):
eva_j, evc_j = eigval_col[eigj], eigvec_col[eigj]
inpr = evc_i.T @ evc_j
vHv_ij = np.diag((inpr * eva_j[np.newaxis, :]) @ inpr.T)
corr_mat_log[eigi, eigj] = ma.corrcoef(
ma.masked_invalid(np.log10(vHv_ij)),
ma.masked_invalid(np.log10(eva_j)))[0, 1]
corr_mat_lin[eigi, eigj] = np.corrcoef(vHv_ij, eva_j)[0, 1]
print("%.1f sec" %
(time() - T0)) # 582.2 secs for the 1000 by 1000 mat. not bad!
np.savez(join(figdir, "Hess_%s_corr_mat.npz" % savelabel),
corr_mat_log=corr_mat_log,
corr_mat_lin=corr_mat_lin)
print("Compute results saved to %s" %
join(figdir, "Hess_%s_corr_mat.npz" % savelabel))
corr_mat_log_nodiag = corr_mat_log.copy()
corr_mat_lin_nodiag = corr_mat_lin.copy()
np.fill_diagonal(corr_mat_log_nodiag, np.nan) # corr_mat_log_nodiag =
np.fill_diagonal(corr_mat_lin_nodiag, np.nan) # corr_mat_log_nodiag =
log_nodiag_mean_cc = np.nanmean(corr_mat_log_nodiag)
lin_nodiag_mean_cc = np.nanmean(corr_mat_lin_nodiag)
log_nodiag_med_cc = np.nanmedian(corr_mat_log_nodiag)
lin_nodiag_med_cc = np.nanmedian(corr_mat_lin_nodiag)
print("Log scale non-diag mean corr value %.3f med %.3f" %
(log_nodiag_mean_cc, log_nodiag_med_cc))
print("Lin scale non-diag mean corr value %.3f med %.3f" %
(lin_nodiag_mean_cc, lin_nodiag_med_cc))
return corr_mat_log, corr_mat_lin
#This file contains code for extracting AB compartments from all time points in a structure import time import numpy as np import numpy.ma as ma import matplotlib.pyplot as plt import pdb from sklearn.decomposition import PCA import matplotlib.pyplot as plt import sys sys.path.insert(0, "../") from Utils import util as ut import sys hic_file = sys.argv[1] out_file = sys.argv[2] start_time = time.time() mat = ut.loadConstraintAsMat(hic_file) mat = np.clip(mat, 0, 30) pear = ma.corrcoef(ma.masked_invalid(mat)) pca = PCA(n_components=1) AB = pca.fit_transform(pear) AB_VEC = np.squeeze(AB) np.save(out_file, AB_VEC)
def pcolor(values, x=None, y=None, **kwargs):
"""Create a pseudocolour plot of a 2D array, Actually uses pcolormesh for speed.
Create a colour plot of a 2D array.
If the arrays x, y, and values are geobipy.StatArray classes, the axes can be automatically labelled.
Can take any other matplotlib arguments and keyword arguments e.g. cmap etc.
Parameters
----------
values : array_like or StatArray
A 2D array of colour values.
x : 1D array_like or StatArray, optional
Horizontal coordinates of the values edges.
y : 1D array_like or StatArray, optional
Vertical coordinates of the values edges.
Other Parameters
----------------
log : 'e' or float, optional
Take the log of the colour to a base. 'e' if log = 'e', and a number e.g. log = 10.
Values in c that are <= 0 are masked.
equalize : bool, optional
Equalize the histogram of the colourmap so that all colours have an equal amount.
nbins : int, optional
Number of bins to use for histogram equalization.
xscale : str, optional
Scale the x axis? e.g. xscale = 'linear' or 'log'
yscale : str, optional
Scale the y axis? e.g. yscale = 'linear' or 'log'.
flipX : bool, optional
Flip the X axis
flipY : bool, optional
Flip the Y axis
grid : bool, optional
Plot the grid
noColorbar : bool, optional
Turn off the colour bar, useful if multiple customPlots plotting routines are used on the same figure.
Returns
-------
ax
matplotlib .Axes
See Also
--------
matplotlib.pyplot.pcolormesh : For additional keyword arguments you may use.
"""
assert np.ndim(values) == 2, ValueError('Number of dimensions must be 2')
equalize = kwargs.pop('equalize', False)
log = kwargs.pop('log', False)
xscale = kwargs.pop('xscale', 'linear')
yscale = kwargs.pop('yscale', 'linear')
flipX = kwargs.pop('flipX', False)
flipY = kwargs.pop('flipY', False)
cl = kwargs.pop('clabel', None)
grid = kwargs.pop('grid', False)
noColorBar = kwargs.pop('noColorbar', False)
# Set the grid colour if specified
c = None
if grid:
c = kwargs.pop('color', 'k')
ax = plt.gca()
pretty(ax)
if (x is None):
mx = np.arange(np.size(values, 1) + 1)
else:
mx = mx = np.asarray(x)
if (x.size == values.shape[1]):
mx = x.edges()
else:
assert x.size == values.shape[1] + 1, ValueError(
'x must be size ' + str(values.shape[1] + 1) + '. Not ' +
str(x.size))
if (y is None):
my = np.arange(np.size(values, 0) + 1)
else:
my = np.asarray(y)
if (y.size == values.shape[0]):
my = y.edges()
else:
assert y.size == values.shape[0] + 1, ValueError(
'y must be size ' + str(values.shape[0] + 1) + '. Not ' +
str(y.size))
v = ma.masked_invalid(np.atleast_2d(np.asarray(values)))
if (log):
v, logLabel = _logSomething(v, log)
if equalize:
nBins = kwargs.pop('nbins', 256)
assert nBins > 0, ValueError('nBins must be greater than zero')
v, dummy = histogramEqualize(v, nBins=nBins)
Zm = ma.masked_invalid(v, copy=False)
X, Y = np.meshgrid(mx, my)
pm = plt.pcolormesh(X, Y, Zm, **kwargs)
plt.xscale(xscale)
plt.yscale(yscale)
xlabel(getNameUnits(x))
ylabel(getNameUnits(y))
if (not noColorBar):
if (equalize):
cbar = plt.colorbar(pm, extend='both')
else:
cbar = plt.colorbar(pm)
if cl is None:
if (log):
clabel(cbar, logLabel + getNameUnits(values))
else:
clabel(cbar, getNameUnits(values))
else:
clabel(cbar, cl)
if flipX:
ax.set_xlim(ax.get_xlim()[::-1])
if flipY:
ax.set_ylim(ax.get_ylim()[::-1])
if grid:
ax.minorticks_on()
for i, xmin in enumerate(ax.xaxis.get_majorticklocs()):
if (xmin == np.round(xmin) and xmin > mx[0] and xmin < mx[-1]):
ax.axvline(x=xmin, c='k', **kwargs)
for i, xmin in enumerate(ax.xaxis.get_minorticklocs()):
if (xmin == np.round(xmin) and xmin > mx[0] and xmin < mx[-1]):
ax.axvline(x=xmin, c='k', **kwargs)
for i, ymin in enumerate(ax.yaxis.get_majorticklocs()):
if (ymin == np.round(ymin) and ymin > my[0] and ymin < my[-1]):
ax.axhline(y=ymin, c='k', **kwargs)
for i, ymin in enumerate(ax.yaxis.get_minorticklocs()):
if (ymin == np.round(ymin) and ymin > my[0] and ymin < my[-1]):
ax.axhline(y=ymin, c='k', **kwargs)
return ax
def infmean(arr):
return pylab.mean(masked_invalid(arr))
import numpy as np import numpy.ma as ma arr = np.array([1, 2, 3, np.nan, 5, 6, np.inf, 8]) ma_arr = ma.masked_invalid(arr) print(ma_arr)
from filter import Filter
#from sclouds.io import Filter
files = glob.glob(os.path.join(read_dir, '*.nc'))
# This should read all files....
import numpy as np
import numpy.ma as ma
#print(ma.corrcoef(a[msk],b[msk]))
#files = glob.glob(os.path.join(read_dir, '*2012*03*.nc'))
#print(files)
data = xr.open_mfdataset(files, compat='no_conflicts')
ref_data = data['tcc'].values
print('ref data shape {}'.format(ref_data.shape))
a = ma.masked_invalid(ref_data)
msk = ~a.mask
print('msk shape {}'.format(msk.shape))
storang = np.zeros((81, 161, 4))
dictionary_to_store = {}
for k, var in enumerate(['r', 'q', 't2m', 'sp']):
print('Variable {}'.format(var))
dta = data[var].values
print('dta data shape {}'.format(dta.shape))
for i in range(81):
for j in range(161):
#print(ref_data[msk][:, i, j].shape)
def main():
args = parse_args()
process_args(args)
print_args(args)
dims = len(args.dimensions)
srcs = len(args.source_ratios)
if args.texture is Texture.NONE:
textures = [Texture.OEU, Texture.OET, Texture.OUT]
if args.plot_table:
textures = [Texture.OET, Texture.OUT]
else:
textures = [args.texture]
texs = len(textures)
prefix = ''
# prefix = 'noprior'
# Initialise data structure.
statistic_arr = np.full((dims, srcs, texs, args.segments, 2), np.nan)
print('Loading data')
argsc = deepcopy(args)
for idim, dim in enumerate(args.dimensions):
argsc.dimension = dim
datadir = args.datadir + '/DIM{0}'.format(dim)
# Array of scales to scan over.
boundaries = fr_utils.SCALE_BOUNDARIES[argsc.dimension]
eval_scales = np.linspace(boundaries[0], boundaries[1],
args.segments - 1)
eval_scales = np.concatenate([[-100.], eval_scales])
for isrc, src in enumerate(args.source_ratios):
argsc.source_ratio = src
for itex, texture in enumerate(textures):
argsc.texture = texture
if args.stat_method is StatCateg.BAYESIAN:
base_infile = datadir + '/{0}/{1}/'.format(
*map(parse_enum, [args.stat_method, args.data])
) + r'{0}/fr_stat'.format(prefix) + gen_identifier(argsc)
else:
base_infile = datadir + '/{0}/{1}/'.format(
*map(parse_enum, [StatCateg.BAYESIAN, args.data])
) + r'{0}/fr_maxllh'.format(prefix) + gen_identifier(argsc)
print('== {0:<25} = {1}'.format('base_infile', base_infile))
if args.split_jobs:
for idx_sc, scale in enumerate(eval_scales):
infile = base_infile + '_scale_{0:.0E}'.format(
np.power(10, scale))
try:
print('Loading from {0}'.format(infile + '.npy'))
statistic_arr[idim][isrc][itex][idx_sc] = \
np.load(infile+'.npy')[0]
except:
print('Unable to load file {0}'.format(infile +
'.npy'))
# raise
continue
else:
print('Loading from {0}'.format(base_infile + '.npy'))
try:
statistic_arr[idim][isrc][itex] = \
np.load(base_infile+'.npy')
except:
print('Unable to load file {0}'.format(base_infile +
'.npy'))
continue
data = ma.masked_invalid(statistic_arr)
print('data', data)
if args.plot_statistic:
print('Plotting statistic')
for idim, dim in enumerate(args.dimensions):
argsc.dimension = dim
# Array of scales to scan over.
boundaries = fr_utils.SCALE_BOUNDARIES[argsc.dimension]
eval_scales = np.linspace(boundaries[0], boundaries[1],
args.segments - 1)
eval_scales = np.concatenate([[-100.], eval_scales])
for isrc, src in enumerate(args.source_ratios):
argsc.source_ratio = src
for itex, texture in enumerate(textures):
argsc.texture = texture
if args.stat_method is StatCateg.BAYESIAN:
base_infile = args.datadir + '/DIM{0}/{1}/{2}/'.format(
dim, *map(parse_enum,
[args.stat_method, args.data
])) + r'{0}/fr_stat'.format(
prefix) + gen_identifier(argsc)
else:
base_infile = args.datadir + '/DIM{0}/{1}/{2}/'.format(
dim,
*map(parse_enum,
[StatCateg.BAYESIAN, args.data
])) + r'{0}/fr_maxllh'.format(
prefix) + gen_identifier(argsc)
basename = os.path.dirname(base_infile)
outfile = basename[:5] + basename[5:].replace(
'data', 'plots')
outfile += '/' + os.path.basename(base_infile)
label = r'$\text{Texture}=' + plot_utils.texture_label(
texture)[1:]
plot_utils.plot_statistic(data=data[idim][isrc][itex],
outfile=outfile,
outformat=['png'],
args=argsc,
scale_param=scale,
label=label)
basename = args.datadir[:5] + args.datadir[5:].replace('data', 'plots')
baseoutfile = basename + '/{0}/{1}/'.format(
*map(parse_enum,
[args.actual_stat_method, args.data])) + r'{0}'.format(prefix)
argsc = deepcopy(args)
if args.plot_x:
for idim, dim in enumerate(args.dimensions):
print('|||| DIM = {0}'.format(dim))
argsc.dimension = dim
plot_utils.plot_x(data=data[idim],
outfile=baseoutfile +
'/hese_x_DIM{0}'.format(dim),
outformat=['png', 'pdf'],
args=argsc,
normalize=True)
if args.plot_table:
plot_utils.plot_table_sens(data=data,
outfile=baseoutfile + '/hese_table',
outformat=['png', 'pdf'],
args=args,
show_lvatmo=True)
def _process_2D_plot_args(args, gridding_dz=1):
"""
Processes input to the plotting class functions. Allows plots to receive
one (2D) or three (1D) input arguements.
"""
from numpy import array, ma, nan, ndarray
from pandas import DataFrame, Series
from xarray import DataArray
from .mapping import grid_data
from .helpers import GliderToolsError
name = ''
if len(args) == 3:
x = array(args[0])
y = array(args[1]).astype(float)
z = args[2].copy()
if isinstance(z, ma.MaskedArray):
z[z.mask] = nan
elif isinstance(z, DataArray):
name = z.name if z.name is not None else ''
unit = ' [{}]'.format(z.units) if 'units' in z.attrs else ''
name = name + unit
z = ma.masked_invalid(array(z)).astype(float)
else:
z = ma.masked_invalid(array(z)).astype(float)
if (x.size == y.size) & (len(z.shape) == 1):
df = grid_data(x,
y,
z,
interp_lim=6,
verbose=False,
return_xarray=False)
x = df.columns
y = df.index
z = ma.masked_invalid(df.values)
return x, y, z, name
elif len(args) == 1:
z = args[0]
if isinstance(z, DataArray):
name = z.name if z.name is not None else ''
unit = ' [{}]'.format(z.units) if 'units' in z.attrs else ''
name = name + unit
if z.ndim == 1:
raise GliderToolsError(
'Please provide gridded DataArray or x and y coordinates')
elif z.ndim == 2:
z = z.to_series().unstack()
elif z.ndim > 2:
raise GliderToolsError(
'GliderTools plotting currently only supports 2 '
'dimensional plotting')
elif isinstance(z, (ndarray, Series)):
if z.ndim == 2:
z = DataFrame(z)
else:
raise IndexError('The input must be a 2D DataFrame or ndarray')
x = z.columns.values
y = z.index.values
z = ma.masked_invalid(z.values).astype(float)
return x, y, z, name
print('Processing data for year '+year+' ...')
for j in date_range:
date='%03d'%j
fname=MyDir+'/'+param+'/'+year+'/AMSRU_Mland_'+year+date+pass_type+'.'+param
if os.path.isfile(fname):
fid = open(fname,'rb');
data=np.fromfile(fid)
fid.close()
for i in list(range(len(data))):
if data[i]<=0.0:
data[i]=np.nan
data = [i*factor for i in data];
data = -np.log(data)
from mkgrid_global import mkgrid_global
datagrid = mkgrid_global(data)
datagridm = ma.masked_invalid(datagrid)
from scipy.interpolate import griddata
z=np.asarray(datagridm)
z=z.flatten()
grid_z = griddata((y,x), z, (grid_y, grid_x), method='linear')
grid_z = ma.masked_invalid(grid_z)
nrows,ncols = np.shape(grid_z)
nrows=np.int(nrows)
ncols=np.int(ncols)
xres = (xmax-xmin)/(ncols-1)
yres = (ymax-ymin)/(nrows-1)
geotransform=(xmin,xres,0,ymax,0, -yres)
arcpy.env.workspace=Dir_fig
os.chdir(Dir_fig)
output_raster = gdal.GetDriverByName('GTiff').Create('VOD_%s_%s_%s.tif' %(year,date,pass_type),ncols, nrows, 1 ,gdal.GDT_Float32,) # Open the file
output_raster.SetGeoTransform(geotransform) # Specify its coordinates
def plotAllSUPERDARN(self):
# make a plot of the results
# type is what we are plotting here
#
# get the current variables
date=self.date
type=self.pType
pdata=self.pFband
times=self.pTimes
freq=self.pF
#
# set here the parameters for power, spectral width, and velocity
# no log parameters here, but will need to implement that if this
# changes
if self.pType == 'power':
cblabel = 'Power [dB]'
vmin = 0
vmax = 20
elif self.pType == 'spectral':
cblabel = 'Spectral Width [m/s]'
vmin = 0
vmax = 50
elif self.pType == 'vel':
cblabel = 'L-o-s Velocity [m/s]'
vmin = -200
vmax = 50
#
# labels for the plots
labels=['13 MHz', '15 MHz', '16 MHz', '17 MHz', '18 MHz', '19 MHz']
#
# iterate through the potential frequencies
for i in range(len(labels)): # number of elements in data arrays
# start by opening a figure
fig=plt.figure()
#
# get the necessary plotting imports
from numpy import ma
os.chdir('/Users/loisks/Desktop/Functions/')
import colormaps as cmaps
os.chdir(self.path)
# this is a colormap I like a lot but I have to load it extra special because of the python version I have :)
plt.register_cmap(name='viridis', cmap=cmaps.viridis)
# for time parameters on plot
# go with time labels every hour and ticks every 10 minutes
days = DayLocator(interval=1)
hours = MinuteLocator(interval=30)
hours2 = MinuteLocator(interval=10)
daysFmt = DateFormatter('%H:%M')
fig.gca().xaxis.set_major_locator(hours)
fig.gca().xaxis.set_major_formatter(daysFmt)
fig.gca().xaxis.set_minor_locator(hours2)
#
# big font to make things easy to read
font = {'family' : 'normal',
'weight' : 'bold',
'size' : 22}
plt.rc('font', **font)
ax=fig.add_subplot(111)
plt.subplots_adjust(right=0.70, top=0.92, bottom=0.28, left=0.11)
ax.set_ylabel('Range Gate Number', fontsize=22, fontweight='bold')
time=dt.date2num(self.pTimes[i])
Altitudes = range(75) # for 75 range gates
X,Y=np.meshgrid(time, Altitudes)
dtemp=np.array(self.pFband[i])
#
# nan the 10000 data, which are nans in this data set
dtemp[dtemp==10000]=np.nan
data=ma.masked_invalid(dtemp).transpose()
ax.set_ylim(25,38) # relevant range gates here
ax.plot(time, np.ones(len(time))*32, lw=3, ls='--', c='k')
ax.set_xlabel("UT Time on " + self.date, fontsize=20, fontweight='bold')
ax.set_xlim(self.xlimLow, self.xlimHigh)
# plot!
try:
col=ax.pcolormesh(X,Y,data, cmap='viridis', vmin=vmin, vmax=vmax)
font = {'family' : 'normal',
'weight' : 'bold',
'size' : 22}
plt.rc('font', **font)
#
# add a colorbar
cbaxes = fig.add_axes([0.8, 0.27, 0.03, 0.65])
cb = plt.colorbar(col, cax = cbaxes,ticks=np.linspace(vmin,vmax,5))
cb.set_label(cblabel, fontsize=25)
fig.set_size_inches(13,9)
#
# save in new directory if directory doesn't already exist
os.chdir(self.path)
subdir_name='CUTLASS_All_Spectrograms'
if not os.path.exists(subdir_name):
os.umask(0) # unmask if necessary
os.makedirs(subdir_name)
os.chdir(subdir_name)#
plt.savefig(labels[i]+self.date+'_'+self.pType+'.png')
except: # if there is no data in this frequency band
print "No data for " + labels[i]
#os.chdir('..')
plt.close()
# PLOT ALL ON SAME
fig=plt.figure()
for i in range(len(labels)): # number of elements in data arrays
from numpy import ma
os.chdir('/Users/loisks/Desktop/Functions/')
import colormaps as cmaps
os.chdir(self.path)
# this is a colormap I like a lot but I have to load it extra special because of the python version I have :)
plt.register_cmap(name='viridis', cmap=cmaps.viridis)
# for time parameters on plot
# go with time labels every hour and ticks every 10 minutes
days = DayLocator(interval=1)
hours = MinuteLocator(interval=30)
hours2 = MinuteLocator(interval=10)
daysFmt = DateFormatter('%H:%M')
fig.gca().xaxis.set_major_locator(hours)
fig.gca().xaxis.set_major_formatter(daysFmt)
fig.gca().xaxis.set_minor_locator(hours2)
#
# big font to make things easy to read
font = {'family' : 'normal',
'weight' : 'bold',
'size' : 22}
plt.rc('font', **font)
ax=fig.add_subplot(111)
plt.subplots_adjust(right=0.70, top=0.92, bottom=0.28, left=0.11)
ax.set_ylabel('Range Gate Number', fontsize=22, fontweight='bold')
time=dt.date2num(self.pTimes[i])
Altitudes = range(75) # for 75 range gates
X,Y=np.meshgrid(time, Altitudes)
dtemp=np.array(self.pFband[i])
#
# nan the 10000 data, which are nans in this data set
dtemp[dtemp==10000]=np.nan
data=ma.masked_invalid(dtemp).transpose()
ax.set_ylim(25,38) # relevant range gates here
ax.plot(time, np.ones(len(time))*32, lw=3, ls='--', c='k')
ax.set_xlabel("UT Time on " + self.date, fontsize=20, fontweight='bold')
ax.set_xlim(self.xlimLow, self.xlimHigh)
# plot!
try:
col=ax.pcolormesh(X,Y,data, cmap='viridis', vmin=vmin, vmax=vmax)
font = {'family' : 'normal',
'weight' : 'bold',
'size' : 22}
plt.rc('font', **font)
#
# add a colorbar
cbaxes = fig.add_axes([0.8, 0.27, 0.03, 0.65])
cb = plt.colorbar(col, cax = cbaxes,ticks=np.linspace(vmin,vmax,5))
cb.set_label(cblabel, fontsize=25)
except: # if there is no data in this frequency band
print "No data for " + labels[i]
#os.chdir('..')
fig.set_size_inches(13,9)
#
# save in new directory if directory doesn't already exist
os.chdir(self.path)
subdir_name='CUTLASS_All_Spectrograms'
if not os.path.exists(subdir_name):
os.umask(0) # unmask if necessary
os.makedirs(subdir_name)
os.chdir(subdir_name)#
plt.savefig('All_'+self.date+'_'+self.pType+'.png')
plt.close()
for cube in cubes:
print("cube loop")
print("cube.var_name=", cube.var_name,
"dict_of_variable_names.keys()",
dict_of_variable_names.keys())
if (cube.var_name in dict_of_variable_names.keys()):
print("NAME IN DICT OF VARIABLE NAMES")
print("file = ", file)
print(cube.var_name)
print("")
# Set any negative values to np.NaN
#print("min bef = ",np.nanmin(cube.data))
cube.data[cube.data < 0.0] = np.NAN
cube.data = ma.masked_invalid(cube.data)
#print("min aft = ",np.nanmin(cube.data))
#print("")
# Convert any data from ng m-3 to ug m-3
#print("cube.units")
#print(cube.units)
if (cube.units == "ng m-3"):
cube.convert_units('ug m-3')
print(cube.units)
print("")
for coord in cube.coords():
#print("coord = ",coord)
def _tomasked(self, value):
try:
return ma.masked_invalid(value.values)
except AttributeError:
return ma.masked_invalid(value)
