def XXtest5_regrid(self):
srcF = cdms2.open(cdat_info.get_prefix() + \
'/sample_data/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc')
so = srcF('so')[0, 0, ...]
clt = cdms2.open(cdat_info.get_prefix() + '/sample_data/clt.nc')('clt')
dstData = so.regrid(clt.getGrid(),
regridTool='esmf',
regridMethod='conserve')
if self.pe == 0:
dstDataMask = (dstData == so.missing_value)
dstDataFltd = dstData * (1 - dstDataMask)
zeroValCnt = (dstData == 0).sum()
if so.missing_value > 0:
dstDataMin = dstData.min()
dstDataMax = dstDataFltd.max()
else:
dstDataMin = dstDataFltd.min()
dstDataMax = dstData.max()
zeroValCnt = (dstData == 0).sum()
print 'Number of zero valued cells', zeroValCnt
print 'min/max value of dstData: %f %f' % (dstDataMin, dstDataMax)
self.assertLess(dstDataMax, so.max())
if PLOT:
pylab.figure(1)
pylab.pcolor(so, vmin=20, vmax=40)
pylab.colorbar()
pylab.title('so')
pylab.figure(2)
pylab.pcolor(dstData, vmin=20, vmax=40)
pylab.colorbar()
pylab.title('dstData')
def test2_varRegrid(self):
print
print 'test2_varRegrid'
srcF = cdms2.open(cdat_info.get_sampledata_path() + \
'/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc')
so = srcF('so')[0, 0, ...]
clt = cdms2.open(cdat_info.get_sampledata_path() + '/clt.nc')('clt')
diag = {'srcAreas': None, 'dstAreas': None,
'srcAreaFractions': None, 'dstAreaFractions': None}
soInterp = so.regrid(clt.getGrid(),
regridTool = 'esmf',
regridMethod='conserve',
diag = diag)
if self.pe == 0:
totSrcArea = diag['srcAreas'].sum()
totDstArea = diag['dstAreas'].sum()
totSrcFrac = diag['srcAreaFractions'].sum()
self.assertEqual(numpy.isnan(totSrcFrac).sum(), 0)
self.assertLess(abs(totSrcArea - 4*pi)/(4*pi), 0.02)
self.assertLess(abs(totDstArea - 4*pi)/(4*pi), 0.01)
soMass = (so*diag['srcAreas']).sum()
inMass = (soInterp*diag['dstAreas']).sum()
print soMass, inMass
diff = abs(soMass - inMass)/soMass
self.assertLess(diff, 7.e-7)
if PLOT:
pylab.subplot(1, 2, 1)
pylab.pcolor(so, vmin = 20, vmax = 40)
pylab.colorbar()
pylab.title('so')
pylab.subplot(1, 2, 2)
pylab.pcolor(soInterp, vmin = 20, vmax = 40)
pylab.colorbar()
pylab.title('soInterp')
def plot_expansion(time, A_array, E_array, eig, c, file_to_save='exp.png',figuresize=(10,10)):
from matplotlib.colors import LogNorm
plt.figure(figsize=figuresize) # in inches!
plt.subplot(1,2,1)
plt.plot(A_array[0], time, label = r'$A_x$')
plt.plot(E_array[0], time, label = r'$E_x$')
plt.plot(A_array[1], time, label = r'$A_y$', ls = '--')
plt.plot(E_array[1], time, label = r'$E_y$', ls = '--')
plt.ylim(0, max(time))
plt.ylabel(r'$t, s$', fontsize=26)
plt.legend(prop={'size':16})
plt.subplot(1,2,2)
X,Y = np.meshgrid(eig.real, time)
plt.pcolor(X, Y, c, norm=LogNorm(vmin=1.0e-3, vmax=1.0))
plt.xlim(min(eig.real), max(eig.real))
plt.ylim(0.0, max(time))
plt.colorbar()
plt.xlabel(r'$E, eV$', fontsize=26)
plt.xlim(-0.3,0.3)
if file_to_save:
plt.savefig(file_to_save)
plt.close()
return None
def Ms_plot(Ms, x, additional_rows=None, additional_labels=None):
import matplotlib.pylab as plt
idx = 5
plt.gcf().add_axes([0, .6, 1, 1])
plt.pcolor(Ms[:, x], vmin=0, vmax=1)
plt.yticks([])
plt.xticks([])
plt.ylabel("$q$s from the confidence interval")
plt.title("Inspecting $q(\\cdot | %d)$" % idx)
plt.gcf().add_axes([0, .3, 1.0, .2])
plt.pcolor(qtilde[idx, None], vmin=0, vmax=1)
plt.yticks([])
plt.xticks([])
plt.ylabel("$\\tilde q$")
plt.gcf().add_axes([0, 0, 1.0, .2])
mappable = plt.pcolor(qxy_star[idx, None], vmin=0, vmax=1)
plt.yticks([])
plt.xticks([])
plt.ylabel("$q^*$")
plt.gcf().add_axes([1.1, 0, .1, 1.6])
plt.colorbar(mappable, cax=plt.gca())
def test2_3x4_to_5x7_cart(self):
# Test non-periodic grid returning double grid resolution
roESMP = CdmsRegrid(self.fromGrid3x4, self.toGrid5x7,
dtype = self.data3x4.dtype,
srcGridMask = self.data3x4.mask,
regridTool = 'ESMP',
periodicity = 0,
regridMethod = 'Conserve',
coordSys = 'cart')
diag = {'srcAreas':0, 'dstAreas':0,
'srcAreaFractions':0, 'dstAreaFractions':0}
ESMP5x7 = roESMP(self.data3x4, diag = diag)
dstMass = (ESMP5x7 * diag['dstAreas']).sum()
srcMass = (self.data3x4 * diag['srcAreas'] \
* diag['srcAreaFractions']).sum()
if False:
pylab.figure(1)
pylab.pcolor(self.data3x4)
pylab.colorbar()
pylab.title('original self.data3x4')
pylab.figure(2)
pylab.pcolor(ESMP5x7)
pylab.colorbar()
pylab.title('interpolated ESMP5x7')
self.assertLess(abs(srcMass - dstMass), self.eps)
self.assertEqual(self.data3x4[0,0], ESMP5x7[0,0])
self.assertEqual(1.0, ESMP5x7[0,0])
self.assertEqual(0.25, ESMP5x7[1,1])
self.assertEqual(0.0, ESMP5x7[2,2])
def plotData(dataId):
from matplotlib import pylab
xtypep = c_int()
fillValuePtr = c_void_p()
gridId = c_int()
coordIds = (c_int * ndims)()
dataPtr = POINTER(c_double)()
latPtr = POINTER(c_double)()
lonPtr = POINTER(c_double)()
ier = pycf.nccf.nccf_get_data_pointer(dataId, byref(xtypep),
byref(dataPtr), byref(fillValuePtr))
assert (ier == pycf.NC_NOERR)
ier = pycf.nccf.nccf_get_data_pointer(dataId, byref(xtypep),
byref(dataPtr), byref(fillValuePtr))
assert (ier == pycf.NC_NOERR)
ier = pycf.nccf.nccf_inq_data_gridid(dataId, byref(gridId))
assert (ier == pycf.NC_NOERR)
ier = pycf.nccf.nccf_inq_grid_coordids(gridId, coordIds)
assert (ier == pycf.NC_NOERR)
ier = pycf.nccf.nccf_get_coord_data_pointer(coordIds[0], byref(latPtr))
assert (ier == pycf.NC_NOERR)
ier = pycf.nccf.nccf_get_coord_data_pointer(coordIds[1], byref(lonPtr))
assert (ier == pycf.NC_NOERR)
ntot, dims = inquireDataSizes(dataId)
data = numpy.ctypeslib.as_array(dataPtr, shape=tuple(dims))
lats = numpy.ctypeslib.as_array(latPtr, shape=tuple(dims))
lons = numpy.ctypeslib.as_array(lonPtr, shape=tuple(dims))
pylab.pcolor(lons, lats, data)
pylab.show()
def plot_expansion(time, A_array, E_array, eig, c, file_to_save='exp.png',figuresize=(10,10)):
from matplotlib.colors import LogNorm
plt.figure(figsize=figuresize) # in inches!
plt.subplot(1,2,1)
plt.plot(A_array[0], time, label = r'$A_x$')
plt.plot(E_array[0], time, label = r'$E_x$')
plt.plot(A_array[1], time, label = r'$A_y$', ls = '--')
plt.plot(E_array[1], time, label = r'$E_y$', ls = '--')
plt.ylim(0, max(time))
plt.ylabel(r'$t, s$', fontsize=26)
plt.legend(prop={'size':16})
plt.subplot(1,2,2)
X,Y = np.meshgrid(eig.real, time)
plt.pcolor(X, Y, c, norm=LogNorm(vmin=1.0e-3, vmax=1.0))
plt.xlim(min(eig.real), max(eig.real))
plt.ylim(0.0, max(time))
plt.colorbar()
plt.xlabel(r'$E, eV$', fontsize=26)
plt.xlim(-0.3,0.3)
if file_to_save:
plt.savefig(file_to_save)
plt.close()
return None
def testSingleTimeSingleElev(self):
"""
Interpolate over one level/time
"""
f = cdms2.open(sys.prefix + '/sample_data/clt.nc')
clt = f('clt')
v = f('v')[0,0,...]
srcGrid = v.getGrid()
dstGrid = clt.getGrid()
ro = CdmsRegrid(srcGrid = srcGrid,
dstGrid = dstGrid,
dtype = v.dtype)
vInterp = ro(v)
print 'min/max of v: %f %f' % (v.min(), v.max())
print 'min/max of vInterp: %f %f' % (vInterp.min(), vInterp.max())
if False:
pl.figure()
pl.pcolor(vInterp, vmin=-20, vmax=20)
pl.title('testSingleTimeSingleElev: vInterp')
pl.colorbar()
def make_correlation_matrix(df, title):
"""
Creates a matplotlib plot of values (typically a correlation dataframe from Pandas).
This plot has each value in a cell shaded on a color map for the entire
matrix. The color map is by default reverse gray scale.
requirements: matplotlib, pandas
"""
fig, ax = plt.subplots(figsize=(12, 12))
pcolor(df, cmap='gray_r')
plt.title(title)
for n, mc in enumerate(df.values):
for i, m in enumerate(mc):
plt.text(n + 0.35,
i + 0.35,
str(round(m, 2)),
color='white',
fontsize=24)
plt.xticks(np.arange(0.5, 5.5, 1))
plt.yticks(np.arange(0.5, 5.5, 1))
labels = list(
df.columns
) # ['Video Access Fraction', 'Video Access Density', 'Course Grade', 'FMCE Pre', 'FMCE post']
ax.set_xticklabels(labels, rotation=90, fontsize=12)
ax.set_yticklabels(labels, rotation=0, fontsize=12)
def testSingleTimeSingleElev(self):
"""
Interpolate over one level/time
"""
f = cdms2.open(cdat_info.get_prefix() + '/sample_data/clt.nc')
clt = f('clt')
v = f('v')[0,0,...]
srcGrid = v.getGrid()
dstGrid = clt.getGrid()
ro = CdmsRegrid(srcGrid = srcGrid,
dstGrid = dstGrid,
dtype = v.dtype)
vInterp = ro(v)
print 'min/max of v: %f %f' % (v.min(), v.max())
print 'min/max of vInterp: %f %f' % (vInterp.min(), vInterp.max())
if PLOT:
pl.figure()
pl.pcolor(vInterp, vmin=-20, vmax=20)
pl.title('testSingleTimeSingleElev: vInterp')
pl.colorbar()
def plot(nxG, nyG, iBeg, iEnd, jBeg, jEnd, data, title=''):
"""
Plot distributed array
@param nxG number of global cells in x
@param nyG number of global cells in y
@param iBeg global starting index in x
@param iEnd global ending index in x
@param jBeg global starting index in y
@param jEnd global ending index in y
@param data local array
@param title plot title
"""
sz = MPI.COMM_WORLD.Get_size()
rk = MPI.COMM_WORLD.Get_rank()
iBegs = MPI.COMM_WORLD.gather(iBeg, root=0)
iEnds = MPI.COMM_WORLD.gather(iEnd, root=0)
jBegs = MPI.COMM_WORLD.gather(jBeg, root=0)
jEnds = MPI.COMM_WORLD.gather(jEnd, root=0)
arrays = MPI.COMM_WORLD.gather(numpy.array(data), root=0)
if rk == 0:
bigArray = numpy.zeros((nxG, nyG), data.dtype)
for pe in range(sz):
bigArray[iBegs[pe]:iEnds[pe], jBegs[pe]:jEnds[pe]] = arrays[pe]
from matplotlib import pylab
pylab.pcolor(bigArray.transpose())
# add the decomp domains
for pe in range(sz):
pylab.plot([iBegs[pe], iBegs[pe]], [0, nyG - 1], 'w--')
pylab.plot([0, nxG - 1], [jBegs[pe], jBegs[pe]], 'w--')
pylab.title(title)
pylab.show()
def XXtest5_regrid(self):
srcF = cdms2.open(sys.prefix + \
'/sample_data/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc')
so = srcF('so')[0, 0, ...]
clt = cdms2.open(sys.prefix + '/sample_data/clt.nc')('clt')
dstData = so.regrid(clt.getGrid(),
regridTool = 'esmf',
regridMethod='conserve')
if self.pe == 0:
dstDataMask = (dstData == so.missing_value)
dstDataFltd = dstData * (1 - dstDataMask)
zeroValCnt = (dstData == 0).sum()
if so.missing_value > 0:
dstDataMin = dstData.min()
dstDataMax = dstDataFltd.max()
else:
dstDataMin = dstDataFltd.min()
dstDataMax = dstData.max()
zeroValCnt = (dstData == 0).sum()
print 'Number of zero valued cells', zeroValCnt
print 'min/max value of dstData: %f %f' % (dstDataMin, dstDataMax)
self.assertLess(dstDataMax, so.max())
if False:
pylab.figure(1)
pylab.pcolor(so, vmin=20, vmax=40)
pylab.colorbar()
pylab.title('so')
pylab.figure(2)
pylab.pcolor(dstData, vmin=20, vmax=40)
pylab.colorbar()
pylab.title('dstData')
def test2_3x4_to_5x7_cart(self):
# Test non-periodic grid returning double grid resolution
roESMP = CdmsRegrid(self.fromGrid3x4,
self.toGrid5x7,
dtype=self.data3x4.dtype,
srcGridMask=self.data3x4.mask,
regridTool='ESMP',
periodicity=0,
regridMethod='Conserve',
coordSys='cart')
diag = {
'srcAreas': 0,
'dstAreas': 0,
'srcAreaFractions': 0,
'dstAreaFractions': 0
}
ESMP5x7 = roESMP(self.data3x4, diag=diag)
dstMass = (ESMP5x7 * diag['dstAreas']).sum()
srcMass = (self.data3x4 * diag['srcAreas'] \
* diag['srcAreaFractions']).sum()
if PLOT:
pylab.figure(1)
pylab.pcolor(self.data3x4)
pylab.colorbar()
pylab.title('original self.data3x4')
pylab.figure(2)
pylab.pcolor(ESMP5x7)
pylab.colorbar()
pylab.title('interpolated ESMP5x7')
self.assertLess(abs(srcMass - dstMass), self.eps)
self.assertEqual(self.data3x4[0, 0], ESMP5x7[0, 0])
self.assertEqual(1.0, ESMP5x7[0, 0])
self.assertEqual(0.25, ESMP5x7[1, 1])
self.assertEqual(0.0, ESMP5x7[2, 2])
def density_cloud_by_tags(df, columns, silent=False):
"""Create density cloud of data for a given tag or group of tags
For example:
columns='DayOfWeek' --> Plots for Mon, Tue, Wed, Thur, ...
columns='Weekday' --> Plots of weekends vs weekday
columns=['Season','Weekday']
--> Plots of Summer, Spring, Winter, Fall Weekdays and Weekends
"""
figures = []
if columns == 'hr' or 'hr' in columns:
raise ValueError("Columns cannot contain hr tag")
# Create a profile for day of week
maxY = df['USAGE'].max()
for label, data in df.groupby(columns):
# Find mean
mean = data.groupby('hr')['USAGE'].agg('mean')
# Add in any missing hours
for h in set(range(24)) - set(data['hr']):
mean = mean.set_value(h, None)
# Create a density cloud of the MW
X = np.zeros([24, 100]) # Hours by resolution
Y = np.zeros([24, 100])
C = np.zeros([24, 100])
for hr, data2 in data.groupby('hr'):
freq = []
step = 1
rng = range(0,51,step)[1:]
freq += rng
bins = np.percentile(data2['USAGE'], rng)
rng = range(50,101,step)[1:]
freq += [100 - a for a in rng]
bins = np.hstack([bins, np.percentile(data2['USAGE'], rng)])
freq = np.array(freq)
X[hr,:] = np.ones(len(bins))*hr
Y[hr,:] = bins
C[hr,:] = freq
plt.figure()
#plt.xkcd()
plt.pcolor(X, Y, C, cmap=plt.cm.YlOrRd)
plt.plot(X[:,1], mean, color='k', label='Mean')
plt.colorbar().set_label('Probability Higher/Lower than Median')
plt.legend(loc='upper left')
plt.xlabel('Hour of Day')
plt.ylabel('Usage (kWh)')
plt.ylim([0, maxY])
plt.xlim([0,23])
plt.title('Typical usage on %s' % str(label))
plt.grid(axis='y')
figures.append(plt.gcf())
if not silent:
plt.show()
return figures
def plot(nxG, nyG, iBeg, iEnd, jBeg, jEnd, data, title=''):
"""
Plot distributed array
@param nxG number of global cells in x
@param nyG number of global cells in y
@param iBeg global starting index in x
@param iEnd global ending index in x
@param jBeg global starting index in y
@param jEnd global ending index in y
@param data local array
@param title plot title
"""
sz = MPI.COMM_WORLD.Get_size()
rk = MPI.COMM_WORLD.Get_rank()
iBegs = MPI.COMM_WORLD.gather(iBeg, root=0)
iEnds = MPI.COMM_WORLD.gather(iEnd, root=0)
jBegs = MPI.COMM_WORLD.gather(jBeg, root=0)
jEnds = MPI.COMM_WORLD.gather(jEnd, root=0)
arrays = MPI.COMM_WORLD.gather(numpy.array(data), root=0)
if rk == 0:
bigArray = numpy.zeros((nxG, nyG), data.dtype)
for pe in range(sz):
bigArray[iBegs[pe]:iEnds[pe], jBegs[pe]:jEnds[pe]] = arrays[pe]
from matplotlib import pylab
pylab.pcolor(bigArray.transpose())
# add the decomp domains
for pe in range(sz):
pylab.plot([iBegs[pe], iBegs[pe]], [0, nyG - 1], 'w--')
pylab.plot([0, nxG - 1], [jBegs[pe], jBegs[pe]], 'w--')
pylab.title(title)
pylab.show()
def test_2d_esmf(self):
# print 'running test_2d_esmf...'
f = cdms2.open(
cdat_info.get_sampledata_path() +
'/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc')
so = f('so')[0, 0, :, :]
clt = cdms2.open(cdat_info.get_sampledata_path() +
'/clt.nc')('clt')[0, :, :]
tic = time.time()
soInterp = so.regrid(clt.getGrid(),
regridTool='ESMF',
regridMethod='CONSERVE') # , periodicity=1)
soInterpInterp = soInterp.regrid(so.getGrid(),
regridTool='ESMF',
regridMethod='CONSERVE')
toc = time.time()
# print 'time to interpolate (ESMF linear) forward/backward: ', toc -
# tic
if has_mpi:
mype = MPI.COMM_WORLD.Get_rank()
else:
mype = 0
if mype == 0:
ntot = reduce(operator.mul, so.shape)
avgdiff = numpy.sum(so - soInterpInterp) / float(ntot)
# print 'avgdiff = ', avgdiff
self.assertLess(abs(avgdiff), 5.2e18)
if PLOT:
pylab.figure(2)
pylab.pcolor(abs(so - soInterpInterp), vmin=0.0, vmax=1.0)
pylab.colorbar()
pylab.title('ESMF linear')
def test2_varRegrid(self):
print
print 'test2_varRegrid'
srcF = cdms2.open(sys.prefix + \
'/sample_data/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc')
so = srcF('so')[0, 0, ...]
clt = cdms2.open(sys.prefix + '/sample_data/clt.nc')('clt')
diag = {'srcAreas': None, 'dstAreas': None,
'srcAreaFractions': None, 'dstAreaFractions': None}
soInterp = so.regrid(clt.getGrid(),
regridTool = 'esmf',
regridMethod='conserve',
diag = diag)
if self.pe == 0:
totSrcArea = diag['srcAreas'].sum()
totDstArea = diag['dstAreas'].sum()
totSrcFrac = diag['srcAreaFractions'].sum()
self.assertEqual(numpy.isnan(totSrcFrac).sum(), 0)
self.assertLess(abs(totSrcArea - 4*pi)/(4*pi), 0.02)
self.assertLess(abs(totDstArea - 4*pi)/(4*pi), 0.01)
soMass = (so*diag['srcAreas']).sum()
inMass = (soInterp*diag['dstAreas']).sum()
print soMass, inMass
diff = abs(soMass - inMass)/soMass
self.assertLess(diff, 7.e-7)
if False:
pylab.subplot(1, 2, 1)
pylab.pcolor(so, vmin = 20, vmax = 40)
pylab.colorbar()
pylab.title('so')
pylab.subplot(1, 2, 2)
pylab.pcolor(soInterp, vmin = 20, vmax = 40)
pylab.colorbar()
pylab.title('soInterp')
def plot_pcolor(self, X, Y, Z):
Xm, Ym = np.meshgrid(X, Y)
try:
plt.pcolor(Xm, Ym, Z)
except:
plt.pcolor(Z)
plt.colorbar()
return None
def plot_pcolor(self, X, Y, Z):
Xm, Ym = np.meshgrid(X, Y)
try:
plt.pcolor(Xm, Ym, Z)
except:
plt.pcolor(Z)
plt.colorbar()
return None
def plot_earth_model(self, type='perturbation'):
if (type == 'model'):
plt.pcolor(self.theta, self.radius, self.vs_array)
plt.colorbar()
plt.show()
elif (type == 'perturbation'):
plt.pcolor(self.theta, self.radius, self.dvs_array)
plt.colorbar()
plt.show()
def DrawMap(Q, model):
M = Q
M -= np.min([np.min(Q), 0])
if np.max(M) != 0:
M /= np.max(M)
M[np.where(model)] = -1
plt.pcolor(Q, cmap="brg")
plt.colorbar()
def test3(self):
"""
2D + level
"""
print 'This is a known failure for now. ESMF are looking into the error'
u = cdms2.open(cdat_info.get_prefix() + '/sample_data/clt.nc')('u')[0, :,...]
uCart = u.regrid( u.getGrid(),
regridTool='esmf', regridMethod='linear',
coordSys = 'cart',
periodicity = 1)
uDegr = u.regrid( u.getGrid(),
regridTool='esmf', regridMethod='linear',
coordSys = 'deg',
periodicity = 1)
n = reduce(lambda x,y: x*y, uCart.shape)
mask = (u == u.missing_value)
if PLOT:
import matplotlib.pylab as pl
fig = pl.figure()
fig.add_subplot(2,1,1)
pl.pcolor(u.getLongitude()[:], u.getLatitude()[:], uCart[1,...],
vmin = 0)
pl.colorbar()
pl.title('Cartiesian')
fig.add_subplot(2,1,2)
pl.pcolor(u.getLongitude()[:], u.getLatitude()[:], uDegr[1,...],
vmin = 0)
pl.colorbar()
pl.title('Degrees')
pl.show()
if PRINT:
print
print 'Level 0, u.min() = %11.3f, uInterp.min() = %11.3f' % \
(u[0,...].min(), uInterp[0,...].min())
print 'Level 1, u.min() = %11.3f, uInterp.min() = %11.3f' % \
(u[1,...].min(), uInterp[1,...].min())
print '\nFor indices 52,59 and 60, 68'
print 'm means missing'
print 'd means uInterp < u'
print '. means uInterp == u'
for i in range(52, 59):
string = ""
for j in range(60, 68):
if uInterp.mask[1,i,j] == True:
string = string+"m"
elif (uInterp[1,i,j] < u[1,i,j]):
string = string+"d"
else:
string = string + "."
print string
print
diff = abs(numpy.sum(u*(1-mask) - uCart)/float(n))
self.assertLess(diff, 1.e-3)
diff = abs(numpy.sum(u*(1-mask) - uDegr)/float(n))
self.assertLess(diff, 1.e-3)
def plotClusters2(clust_labels, n_clusters, original_data):
plt.figure()
for cluster_label in range(n_clusters):
plt.subplot(10,10,cluster_label + 1) ## lol watch out for +1 !!!!
plt.title(cluster_label + 1)
plt.gca().invert_yaxis()
one_cluster = original_data["shapes_n"][clust_labels == cluster_label, ...]
mean_image = np.mean(one_cluster, axis=0)
plt.pcolor(mean_image)
def draw(self):
x = np.arange(0, self.h_x, self.del_x)
y = np.arange(0, self.h_y, self.del_y)
X, Y = plt.meshgrid(x, y)
Z = self.map
plt.pcolor(X, Y, Z)
plt.colorbar()
plt.show()
def corr(data,labels,**kwargs):
data=np.transpose(data)
corrs=np.corrcoef(data)
labelsDict=dict((i,labels[i]) for i in range(len(labels)))
if 'makeGraph' in kwargs.keys():
if kwargs['makeGraph']==True:
fig,ax=plt.subplots()
# plt.pcolor(corrs)
plt.pcolor(corrs>=kwargs['Tresh'])
plt.xticks([i for i in range(44)],rotation=45)
ax.set_xticklabels(labels)
ax.set_yticklabels(labels)
plt.tick_params(axis='both', which='both', labelsize=7)
# plt.imshow(corrs>=kwargs['Tresh'],interpolation=None)
# plt.colorbar()
plt.show()
if 'undGraph' in kwargs:
plt.figure()
if kwargs['undGraph']==True:
gcorrs=np.copy(corrs)
if 'Tresh' in kwargs:
idx=np.where(corrs<=kwargs['Tresh'])
gcorrs[idx]=0
gcorrs=gcorrs-np.identity(gcorrs.shape[0])
G=nx.from_numpy_matrix(np.triu(gcorrs))
for node in nx.nodes(G):
edges=np.sum([ 1 for i in nx.all_neighbors(G, node)])
if edges==0:
G.remove_node(node)
labelsDict.pop(node)
G=nx.relabel_nodes(G,labelsDict)
pos=nx.spring_layout(G,iterations=200)
# pos=nx.shell_layout(G)
nx.draw_networkx(G,pos,font_size=9)
# nx.draw_spring(G)
# nx.draw(G,pos,font_size=9)
plt.show()
if 'ret' in kwargs.keys():
if kwargs['ret']==True:
corrs2=np.triu(corrs-np.diagflat(np.diag(corrs)))
i,j=np.where(np.abs(corrs2)>=kwargs['Tresh'])
# print corrs2[i,j]
# print i
# print j
feats=set(list(i)+list(j))
# print feats
return feats
def Xtest6_ESMF_Conserve_LevelTime_clt(self):
"""
Interpolate over level/time in addition to lat-lon
"""
f = cdms2.open(sys.prefix + '/sample_data/clt.nc')
clt = f('clt')
v = f('v')
# mask
srcGridMask = numpy.array(v[0, 0, ...] == v.missing_value, numpy.int32)
# v onto the ctl grid
srcGrd, srcNDims = regrid2.gsRegrid.makeCurvilinear(
[v.getLatitude(), v.getLongitude()])
dstGrd, dstNDims = regrid2.gsRegrid.makeCurvilinear(
[clt.getLatitude(), clt.getLongitude()])
ro = regrid2.GenericRegrid(srcGrd,
dstGrd,
regridMethod='conserve',
regridTool='esmp',
periodicity=1,
srcGridMask=srcGridMask)
ro.computeWeights()
vInterp = numpy.ones(
list(v.shape[:-2]) + list(clt.shape[-2:]),
v.dtype) * v.missing_value
ro.apply(numpy.array(v), vInterp, rootPe=0)
print 'min/max of v: %f %f' % (v.min(), v.max())
print 'min/max of vInterp: %f %f' % (vInterp.min(), vInterp.max())
if False:
nTimes = v.shape[0]
nLevels = v.shape[1]
for el in range(nTimes):
for k in range(nLevels):
pl.figure()
pl.subplot(1, 2, 1)
pl.pcolor(srcGrd[1],
srcGrd[0],
v[el, k, ...],
vmin=-20,
vmax=20)
pl.title('test6: v[%d, %d,...]' % (el, k))
pl.colorbar()
pl.subplot(1, 2, 2)
pl.pcolor(dstGrd[1],
dstGrd[0],
vInterp[el, k, ...],
vmin=-20,
vmax=20)
pl.title('test6: vInterp[%d, %d,...]' % (el, k))
pl.colorbar()
def plot_curvature(k):
sel = k.unstack()
T = sel.index
S = sel.columns*spacing
Z = sel.values
Ts = np.vstack([T]*len(S)).T
Ss = np.vstack([S]*len(T))
plt.pcolor(Ts, Ss, Z, cmap=plt.cm.viridis, vmin=-0.5, vmax=.5)
plt.colorbar(label="curvature $[\mu m^-1]$")
plt.xlabel("t [s]")
plt.ylabel(r"s $[\mu m]$")
def plot_contour(x, y, z, file_name):
xx, yy = meshgrid(y, x)
plt.figure()
plt.pcolor(xx, yy, z, vmax=abs(z).max(), vmin=abs(z).min())
cbar = plt.colorbar()
cbar.ax.set_ylabel('Scalar Modifier', rotation=270)
plt.xlabel('Calendar Years')
plt.ylabel('Age')
with open(str(file_name), 'wb') as f:
plt.savefig(f, format='png')
plt.close()
def test0(self):
"""
One way interpolation
"""
fnm = sys.prefix + '/sample_data/clt.nc'
f = cdms2.open(fnm)
s = f("clt")
grdori = s.getGrid()
nLon = s.shape[-1]
nLat = s.shape[-2]
xx = numpy.outer(numpy.ones((nLat, ), numpy.float32),
s.getLongitude()[:])
yy = numpy.outer(s.getLatitude(), numpy.ones((nLon, ), numpy.float32))
print s.shape, xx.shape, yy.shape
#s[0,...] = 50.0*(1.0 + numpy.sin(4*numpy.pi * xx / 180.0) * numpy.cos(2*numpy.pi * yy / 180.0))
grid = cdms2.createGaussianGrid(32, 64)
sInterps = {}
sInterps['regrid2'] = s.regrid(grid, regridTool='regrid2')
sInterps['esmf linear'] = s.regrid(grid,
regridTool='esmf',
regridMethod='linear')
diag = {}
sInterps['libcf'] = s.regrid(grid, regridTool='libcf', diag=diag)
print diag
diag = {}
sInterps['esmf conserve'] = s.regrid(grid,
regridTool='esmf',
regridMethod='conserve',
diag=diag)
print diag
diff = abs(sInterps['esmf linear'] - sInterps['regrid2']).max()
self.assertLess(diff, 18.0)
diff = abs(sInterps['esmf conserve'] - sInterps['regrid2']).max()
self.assertLess(diff, 86.0)
diff = abs(sInterps['libcf'] - sInterps['regrid2']).max()
self.assertLess(diff, 18.0)
if PLOT:
row = 0
for mth in sInterps:
row += 1
pylab.subplot(2, 2, row)
pylab.pcolor(sInterps[mth][0, ...] -
sInterps['regrid2'][0, ...],
vmin=-10,
vmax=10)
pylab.colorbar()
pylab.title(mth + ' - regrid2')
def PlotFieldQuad(self):
if not os.path.isdir(self.outdir):
os.makedirs(self.outdir)
fig_path = self.outdir
from matplotlib import pylab as plt
plt.close('all')
plt.figure()
quad = self.quad.copy()
plt.plot(quad[0,:]/1e-15,quad[1,:],'g--',label='$\mathscr{S}$')
plt.plot(quad[0,:]/1e-15,quad[2,:],'r--',label='$\mathscr{I}$')
xlabel = 'time ($fs$)'
ylabel = '$\mathscr{S}$, $\mathscr{I}$ ($a.u.$)'
plt.xlabel(xlabel)
plt.ylabel(ylabel)
plt.legend(frameon=False)
plt.draw()
plt.savefig(os.path.join(fig_path,'si_vs_t.png'),dpi=240)
plt.close()
plt.figure()
plt.plot(quad[0,:]/1e-15,quad[3,:],'g--',label='$\partial_t \mathscr{S}$')
plt.plot(quad[0,:]/1e-15,quad[4,:],'r--',label='$\partial_t \mathscr{I}$')
xlabel = 'time ($fs$)'
ylabel = '$\partial_t$ $\mathscr{S}$, $\mathscr{I}$ ($fs^{-1}$)'
plt.xlabel(xlabel)
plt.ylabel(ylabel)
plt.legend(frameon=False)
plt.draw()
plt.savefig(os.path.join(fig_path,'dtsi_vs_t.png'),dpi=240)
plt.close()
plt.figure()
t = np.linspace(self.quad[0,0],self.quad[0,-1],len(self.quad[0,:]))
x = self._x.copy()
#print t.shape, x.shape,self.u.shape
T,X = np.meshgrid(t,x)
#print T[0::5,:].shape,X[0::5,:].shape,self.u[0::5,:].transpose().shape
u = self.u.transpose()
plt.pcolor(X[::10,:],T[::10,:],u[::10,:],shading='interp')
xlabel = '$x$ ($\mu m$)'
ylabel = 'time ($fs$)'
plt.xlabel(xlabel)
plt.ylabel(ylabel)
plt.colorbar()
plt.draw()
plt.savefig(os.path.join(fig_path,'u.png'),dpi=240)
plt.close()
plt.close('all')
def test_2d_esmf_conserv(self):
print 'running test_2d_esmf_conserv...'
f = cdms2.open(sys.prefix + \
'/sample_data/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc')
so = f('so')[0, 0, :, :]
clt = cdms2.open(sys.prefix + '/sample_data/clt.nc')('clt')[0, :, :]
tic = time.time()
soInterp = so.regrid(clt.getGrid(), regridTool='ESMF', regridMethod='Conservative')
soInterpInterp = soInterp.regrid(so.getGrid(), regridTool='ESMF',
regridMethod='Conservative')
toc = time.time()
print 'time to interpolate (ESMF conservative) forward/backward: ', toc - tic
ntot = reduce(operator.mul, so.shape)
avgdiff = numpy.sum(so - soInterpInterp) / float(ntot)
print 'avgdiff = ', avgdiff
if PLOT:
pylab.figure(2)
pylab.subplot(2, 2, 1)
pylab.pcolor(so, vmin=20.0, vmax=40.0)
pylab.colorbar()
pylab.title('ESMF conserve regrid: so')
pylab.subplot(2, 2, 2)
pylab.pcolor(soInterp, vmin=20.0, vmax=40.0)
pylab.colorbar()
pylab.title('ESMF conserve regrid: soInterp')
pylab.subplot(2, 2, 3)
pylab.pcolor(soInterpInterp, vmin=20.0, vmax=40.0)
pylab.colorbar()
pylab.title('ESMF conserve regrid: soInterpInterp')
pylab.subplot(2, 2, 4)
pylab.pcolor(so - soInterpInterp, vmin=-0.5, vmax=0.5)
pylab.colorbar()
pylab.title('ESMF conserve regrid: error')
def test_2d_esmf_conserv(self):
print 'running test_2d_esmf_conserv...'
f = cdms2.open(cdat_info.get_sampledata_path() + \
'/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc')
so = f('so')[0, 0, :, :]
clt = cdms2.open(cdat_info.get_sampledata_path() + '/clt.nc')('clt')[0, :, :]
tic = time.time()
soInterp = so.regrid(clt.getGrid(), regridTool='ESMF', regridMethod='Conservative')
soInterpInterp = soInterp.regrid(so.getGrid(), regridTool='ESMF',
regridMethod='Conservative')
toc = time.time()
print 'time to interpolate (ESMF conservative) forward/backward: ', toc - tic
ntot = reduce(operator.mul, so.shape)
avgdiff = numpy.sum(so - soInterpInterp) / float(ntot)
print 'avgdiff = ', avgdiff
if PLOT:
pylab.figure(2)
pylab.subplot(2, 2, 1)
pylab.pcolor(so, vmin=20.0, vmax=40.0)
pylab.colorbar()
pylab.title('ESMF conserve regrid: so')
pylab.subplot(2, 2, 2)
pylab.pcolor(soInterp, vmin=20.0, vmax=40.0)
pylab.colorbar()
pylab.title('ESMF conserve regrid: soInterp')
pylab.subplot(2, 2, 3)
pylab.pcolor(soInterpInterp, vmin=20.0, vmax=40.0)
pylab.colorbar()
pylab.title('ESMF conserve regrid: soInterpInterp')
pylab.subplot(2, 2, 4)
pylab.pcolor(so - soInterpInterp, vmin=-0.5, vmax=0.5)
pylab.colorbar()
pylab.title('ESMF conserve regrid: error')
def test2_tasWithLevelsUsingLibCF(self):
test = 'Time test using regrid2 gsRegrid'
r2 = regrid2.GenericRegrid(self.gGrid, self.fGrid,
dtype = self.gtas.dtype,
regridMethod='linear',
regridTool='libcf')
r2.computeWeights()
fShape = list(self.gtas.shape[:-2]) + list(self.fGrid[0].shape)
ftas = np.ma.masked_array(np.ones(fShape, self.gtas.dtype)*self.gtas.missing_value,
mask = np.zeros(fShape))
ftas.missing_value = self.gtas.missing_value
r2.apply(self.gtas, ftas)
if PLOT:
vmin = self.gtas.min()
vmax = self.gtas.max()
pylab.subplot(1,4,1)
pylab.pcolor(self.gtas[-1,...], vmin=vmin, vmax=vmax)
pylab.colorbar()
pylab.title('self.gtas[-1,...]')
pylab.subplot(1,4,2)
pylab.pcolor(ftasOld[-1,...], vmin=vmin, vmax=vmax)
pylab.colorbar()
pylab.title('ftasOld[-1,...]')
pylab.subplot(1,4,3)
pylab.pcolor(ftas[0,...], vmin=vmin, vmax=vmax)
pylab.colorbar()
pylab.title('ftas[-1,...]')
pylab.subplot(1,4,4)
pylab.pcolor(ftas[-1,...] - ftasOld[-1,...], vmin=-1, vmax=1)
pylab.colorbar()
pylab.title('ftas[-1,...] - ftasOld[-1,...]')
self.assertEqual(ftas[0,...].shape, self.fclt[0,...].shape)
self.assertEqual(ftas.shape[0], self.gtas.shape[0])
self.assertGreater(abs(ftas[ftas>0].mean()-self.gtas.mean()), 1)
def draw2(self):
x = np.arange(0, self.h_x + 1, self.del_x)
y = np.arange(0, self.h_y + 1, self.del_y)
X, Y = plt.meshgrid(x, y)
Z = self.u.reshape([self.x_div, self.y_div])
plt.xlabel("x")
plt.ylabel("y")
# plt.xlim([0, self.h_x])
# plt.ylim([0, self.h_y])
plt.pcolor(X, Y, Z.T)
plt.colorbar()
plt.show()
def plotCellAreas(cube):
coords = cube.coords()
yy = coords[1].points
xx = coords[0].points
dyy = yy[1:, :] - yy[:-1, :]
dxx = xx[:, 1:] - xx[:, :-1]
dyyMid = 0.5 * (dyy[:, :-1] + dyy[:, 1:])
dxxMid = 0.5 * (dxx[:-1, :] + dxx[1:, :])
areas = numpy.zeros(yy.shape, yy.dtype)
areas[:-1, :-1] = dyyMid * dxxMid
negativeAreas = numpy.zeros(yy.shape, yy.dtype)
nj, ni = yy.shape
negativeAreas[numpy.where(areas < 0)] = 1.0
pylab.pcolor(xx, yy, negativeAreas, cmap='bone_r')
def plotCellAreas(cube):
coords = cube.coords()
lats = coords[0].points
lons = coords[1].points
dlats = lats[1:, :] - lats[:-1, :]
dlons = lons[:, 1:] - lons[:, :-1]
dlatMid = 0.5 * (dlats[:, :-1] + dlats[:, 1:])
dlonMid = 0.5 * (dlons[:-1, :] + dlons[1:, :])
areas = numpy.zeros(lats.shape, lats.dtype)
areas[:-1, :-1] = dlatMid * dlonMid
negativeAreas = numpy.zeros(lats.shape, lats.dtype)
nlat, nlon = lats.shape
area_max = 10 * (180. / nlat) * (360. / nlon)
negativeAreas[numpy.where(numpy.abs(areas) > area_max)] = 1.0
pylab.pcolor(lons, lats, negativeAreas, cmap='bone_r')
def plotClustersPL(clust_labels, n_clusters, original_data, trans_dict):
d = createTransDict(clust_labels, 100)
plt.figure()
for cluster_label in range(n_clusters):
plt.subplot(10,10,cluster_label + 1) ## lol watch out for +1 !!!!
plt.xlim([0, 31])
plt.ylim([0, 63])
plt.gca().invert_yaxis()
plt.gca().axes.get_xaxis().set_visible(False)
plt.gca().axes.get_yaxis().set_visible(False)
plt.title(trans_dict[d[cluster_label]])
one_cluster = original_data["shapes_n"][clust_labels == cluster_label, ...]
mean_image = np.mean(one_cluster, axis=0)
plt.pcolor(mean_image)
plt.subplots_adjust(hspace=0.45)
def heatmap(self,
title='',
norm=False,
savefig=False,
xticks_loc='default',
xticks_label='default',
yticks_interval='default'):
#xticks_loc=np.linspace(0,3000, 7),
#xticks_label=np.arange(0, 35, 5), yticks_interval=5):
"""
Draw a heatmap of the spectral information returned by
mrDMD.fit()
Parameters
--------
title : string
Titles the plot and names the png file
norm : boolean
The local level of the mrDMD. l is in [0, L)
savefig : boolean
Saves figure in current directory with title given
xticks_loc : array-like
Locations of xticks (time axis)
xticks_label : array-like
Labels at xticks (time axis)
yticks_interval : int
Skip interval for yticks (frequency axis)
"""
if xticks_loc is 'default':
t = range(self.time_steps) * np.tile(self.time_resolution,
self.time_steps)
plt.figure()
if norm:
spec = self.spec / np.max(self.spec)
else:
spec = self.spec
plt.pcolor(t, self.freq_space, spec, cmap='hot')
#plt.yticks(np.arange(len(self.freq_space) -
#1)[::yticks_interval], [int(i) for i in
#self.freq_space[::yticks_interval]])
#plt.xticks(xticks_loc, xticks_label)
plt.ylabel('Frequency (Hz)')
plt.xlabel('Time (s)')
plt.title(title)
#plt.colorbar()
if savefig:
plt.savefig('%s.png' % title.replace(' ', ''))
def plotBasisFunctions(self, eigenvalues, eigenvectors):
'''3d plot of the basis function. Right now I am plotting eigenvectors,
so each coordinate of the eigenvector correspond to the value to be
plotted for the correspondent state.'''
for i in range(len(eigenvalues)):
# fig1 = plt.figure()
# ax1 = fig1.add_subplot(111)
#
# ax1.set_title('Simple Heatmap with matplotlib and plotly')
#
# plotly_fig = tls.mpl_to_plotly(fig1)
# Z = eigenvectors[:, i].reshape(self.numCols, self.numRows)
# plotly_fig['data'] = [dict(z=Z, type="heatmap", zmin=np.min(Z), zmax=np.max(Z), colorscale='Viridis')]
# plotly_fig['layout']['xaxis'].update({'autorange': True})
# plotly_fig['layout']['yaxis'].update({'autorange': True})
#
# plot_url = py.plot(plotly_fig, filename='mpl-basic-heatmap')
Z = eigenvectors[:, i].reshape(self.numCols, self.numRows)
# fig, ax = plt.subplots(subplot_kw=dict(projection='3d'))
X, Y = np.meshgrid(np.arange(self.numRows),
np.arange(self.numCols))
for ii in range(len(X)):
for j in range(int(len(X[ii]) / 2)):
tmp = X[ii][j]
X[ii][j] = X[ii][len(X[ii]) - j - 1]
X[ii][len(X[ii]) - j - 1] = tmp
new_Z = Z[X][Y]
plt.pcolor(X, Y, Z, cmap=cm.Blues)
# my_col = cm.jet(np.random.rand(Z.shape[0], Z.shape[1]))
# surf = ax.plot_surface(X, Y, Z, rstride=1, cstride=1,
# cmap=cm.Blues, linewidth=0, antialiased=False)
# ax.zaxis.set_major_locator(LinearLocator(10))
# ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))
# Add a color bar which maps values to colors.
# fig.colorbar(surf, shrink=0.5, aspect=5)
# plt.gca().view_init(elev=30, azim=30)
plt.savefig(
os.path.join(self.outputPath,
("Eigenvector" + str(i) + '_eig' + '.png')))
plt.close()
plt.plot(eigenvalues, 'o')
plt.savefig(self.outputPath + 'eigenvalues.png')
def test3(self):
"""
2D + level
"""
print "This is a known failure for now. ESMF are looking into the error"
u = cdms2.open(sys.prefix + "/sample_data/clt.nc")("u")[0, :, ...]
uCart = u.regrid(u.getGrid(), regridTool="esmf", regridMethod="linear", coordSys="cart", periodicity=1)
uDegr = u.regrid(u.getGrid(), regridTool="esmf", regridMethod="linear", coordSys="deg", periodicity=1)
n = reduce(lambda x, y: x * y, uCart.shape)
mask = u == u.missing_value
if PLOT:
import matplotlib.pylab as pl
fig = pl.figure()
fig.add_subplot(2, 1, 1)
pl.pcolor(u.getLongitude()[:], u.getLatitude()[:], uCart[1, ...], vmin=0)
pl.colorbar()
pl.title("Cartiesian")
fig.add_subplot(2, 1, 2)
pl.pcolor(u.getLongitude()[:], u.getLatitude()[:], uDegr[1, ...], vmin=0)
pl.colorbar()
pl.title("Degrees")
pl.show()
if PRINT:
print
print "Level 0, u.min() = %11.3f, uInterp.min() = %11.3f" % (u[0, ...].min(), uInterp[0, ...].min())
print "Level 1, u.min() = %11.3f, uInterp.min() = %11.3f" % (u[1, ...].min(), uInterp[1, ...].min())
print "\nFor indices 52,59 and 60, 68"
print "m means missing"
print "d means uInterp < u"
print ". means uInterp == u"
for i in range(52, 59):
string = ""
for j in range(60, 68):
if uInterp.mask[1, i, j] == True:
string = string + "m"
elif uInterp[1, i, j] < u[1, i, j]:
string = string + "d"
else:
string = string + "."
print string
print
diff = abs(numpy.sum(u * (1 - mask) - uCart) / float(n))
self.assertLess(diff, 1.0e-3)
diff = abs(numpy.sum(u * (1 - mask) - uDegr) / float(n))
self.assertLess(diff, 1.0e-3)
def test1_regrid(self):
clt = cdms2.open(cdat_info.get_sampledata_path() + '/clt.nc')('clt')[0,...]
ta = cdms2.open(cdat_info.get_sampledata_path() + '/ta_ncep_87-6-88-4.nc')('ta')[0, 0,...]
diag = {}
cltInterp = clt.regrid( ta.getGrid(),
regridTool = 'libcf',
mkCyclic = True, verbose = True,
diag = diag )
print cltInterp.sum()
n = reduce(lambda x,y: x*y, cltInterp.shape)
self.assertLess(abs(cltInterp.sum() - 696921.0)/n, 0.3)
if PLOT:
pylab.pcolor(ta.getLongitude()[:], ta.getLatitude()[:], cltInterp)
pylab.colorbar()
pylab.title('cltInterp')
self.assertEqual(True, True)
def plot_feature_corr_matrix(df, title, figsave):
corr = df.corr()
xmax = df.shape[1] + 0.5
save_index = df.columns[0].split('_')[0]
plt.figure(figsize=(14, 11))
plt.pcolor(corr, cmap='rainbow')
plt.colorbar()
plt.xticks(np.arange(0.5, xmax), corr.columns, rotation=90, fontsize=8)
plt.yticks(np.arange(0.5, xmax), corr.columns, fontsize=8)
plt.title(title)
plt.xlim(0, xmax - 0.5)
plt.ylim(0, xmax - 0.5)
plt.savefig('Graphs/' + save_index + 'FeaturesHeatMap.png',
bbox_inches='tight')
def plot_comparison_2D(X, Y, elec_pos, true_csd, true_pots, rec_csd, rec_pots,
err_csd, err_pot):
"""
Plot for comparing model and reconstructed LFP and CSD in 2D.
"""
fig = plt.figure()
ax11 = fig.add_subplot(2, 3, 1)
pc1 = plt.pcolor(X, Y, true_csd)
plt.colorbar(pc1)
ax11.set_title('True CSD')
ax12 = fig.add_subplot(2, 3, 2)
pc2 = plt.pcolor(X, Y, rec_csd)
plt.colorbar(pc2)
ax12.set_title('Reconstructed CSD')
ax13 = fig.add_subplot(2, 3, 3)
pc3 = ax13.pcolor(X, Y, err_csd)
plt.colorbar(pc3)
ax13.set_title('CSD reconstruction error [%]')
ax21 = fig.add_subplot(2, 3, 4)
pc4 = plt.pcolor(X, Y, true_pots, cmap='RdYlBu')
plt.colorbar(pc4)
ax21.set_title('True LFP (forward scheme calculation from CSD)')
ax22 = fig.add_subplot(2, 3, 5)
pc5 = plt.pcolor(X, Y, rec_pots, cmap='RdYlBu')
plt.colorbar(pc5)
ax22.scatter(elec_pos[:, 0],
elec_pos[:, 1],
marker='o',
c='b',
s=3,
zorder=10)
ax22.set_xlim([np.min(X), np.max(X)])
ax22.set_ylim([np.min(Y), np.max(Y)])
ax22.set_title('Reconstructed LFP')
ax23 = fig.add_subplot(2, 3, 6)
pc6 = ax23.pcolor(X, Y, err_pot, cmap='RdYlBu')
plt.colorbar(pc6)
ax23.set_title('LFP reconstruction Error [%]')
plt.show()
def bagging():
X_train, X_test, y_train, y_test = train_test_split(x,
y,
random_state=35,
test_size=0.2)
x1_test = np.zeros((X_test.shape[0], len(classifiers))) #存储第一层测试集的输出结果
accuracy = np.zeros(len(classifiers)) #每个模型的准确率
for train_index, test_index in sss.split(X_train, y_train):
x_train, x_test = x[train_index], x[test_index]
y_train, y_test = y[train_index], y[test_index]
clf_num = 0
for clf in classifiers:
clf_name = clf.__class__.__name__
clf.fit(x_train, y_train)
x1_test[:, clf_num] += clf.predict(X_test) #直接对测试集进行预测，总共有十次，进行平均
accuracy[clf_num] += (
y_test == clf.predict(x_test)).mean() #该模型的准确率，十次平均
clf_num += 1
x1_test = x1_test / 10
accuracy = accuracy / 10
plt.bar(np.arange(len(classifiers)), accuracy, width=0.5, color='b')
plt.xlabel('Alog')
plt.ylabel('Accuracy')
plt.xticks(
np.arange(len(classifiers)) + 0.25, [
'KNN', 'DT', 'RF', 'SVC', 'AdaB', 'GBC', 'GNB', 'LDA', 'QDA', 'LR',
'xgb'
])
pyl.pcolor(np.corrcoef(x1_test.T), cmap='Blues')
pyl.colorbar()
pyl.xticks(np.arange(0.5, 11.5), [
'KNN', 'DT', 'RF', 'SVC', 'AdaB', 'GBC', 'GNB', 'LDA', 'QDA', 'LR',
'xgb'
])
pyl.yticks(np.arange(0.5, 11.5), [
'KNN', 'DT', 'RF', 'SVC', 'AdaB', 'GBC', 'GNB', 'LDA', 'QDA', 'LR',
'xgb'
])
pyl.show()
import pandas as pd
index = [0, 1, 2, 5, 9]
linear_prediction = x1_test[:, index].mean(axis=1)
print np.shape(linear_prediction)
def Xtest1(self):
"""
Forward/backward interpolation
"""
fnm=vcs.sample_data+'/clt.nc'
f=cdms2.open(fnm)
s=f("clt")
grdori = s.getGrid()
nLon = s.shape[-1]
nLat = s.shape[-2]
xx = numpy.outer(numpy.ones( (nLat,), numpy.float32 ), s.getLongitude()[:])
yy = numpy.outer(s.getLatitude(), numpy.ones( (nLon,), numpy.float32) )
print s.shape, xx.shape, yy.shape
#s[0,...] = 50.0*(1.0 + numpy.sin(4*numpy.pi * xx / 180.0) * numpy.cos(2*numpy.pi * yy / 180.0))
grid = cdms2.createGaussianGrid(64,128)
sInterps = {}
sInterps['regrid2'] = s.regrid(grid, regridTool='regrid2').regrid(grdori, regridTool='regrid2')
sInterps['esmf linear'] = s.regrid(grid, regridTool='esmf', regridMethod = 'linear').regrid(grdori, regridTool='esmf', regridMethod = 'linear')
diag = {}
sInterps['libcf'] = s.regrid(grid, regridTool='libcf', diag=diag).regrid(grdori, regridTool='libcf', diag=diag)
print diag
diag = {}
sInterps['esmf conserve'] = s.regrid(grid, regridTool='esmf', regridMethod = 'conserve', diag=diag).regrid(grdori, regridTool='esmf', regridMethod = 'conserve')
print diag
diff = abs(sInterps['regrid2'] - s).max()
self.assertLess(diff, 63.0)
diff = abs(sInterps['esmf linear'] - s).max()
self.assertLess(diff, 45.0)
diff = abs(sInterps['esmf conserve'] - s).max()
self.assertLess(diff, 103.534)
diff = abs(sInterps['libcf'] - s).max()
self.assertLess(diff, 45.0)
if PLOT:
row = 0
for mth in sInterps:
row += 1
pylab.subplot(2, 2, row)
pylab.pcolor(sInterps[mth][0,...] - s[0,...], vmin = -10, vmax = 10)
pylab.colorbar()
pylab.title(mth + ' - original')
def Xtest6_ESMF_Conserve_LevelTime_clt(self):
"""
Interpolate over level/time in addition to lat-lon
"""
f = cdms2.open(sys.prefix + '/sample_data/clt.nc')
clt = f('clt')
v = f('v')
# mask
srcGridMask = numpy.array(v[0,0,...] == v.missing_value, numpy.int32)
# v onto the ctl grid
srcGrd, srcNDims = regrid2.gsRegrid.makeCurvilinear([v.getLatitude(),
v.getLongitude()])
dstGrd, dstNDims = regrid2.gsRegrid.makeCurvilinear([clt.getLatitude(),
clt.getLongitude()])
ro = regrid2.GenericRegrid(srcGrd, dstGrd,
regridMethod = 'conserve',
regridTool = 'esmp',
periodicity = 1,
srcGridMask = srcGridMask)
ro.computeWeights()
vInterp = numpy.ones(list(v.shape[:-2]) + list(clt.shape[-2:]),
v.dtype) * v.missing_value
ro.apply(numpy.array(v), vInterp, rootPe = 0)
print 'min/max of v: %f %f' % (v.min(), v.max())
print 'min/max of vInterp: %f %f' % (vInterp.min(), vInterp.max())
if False:
nTimes = v.shape[0]
nLevels = v.shape[1]
for el in range(nTimes):
for k in range(nLevels):
pl.figure()
pl.subplot(1,2,1)
pl.pcolor(srcGrd[1], srcGrd[0], v[el, k,...], vmin=-20, vmax=20)
pl.title('test6: v[%d, %d,...]' % (el, k))
pl.colorbar()
pl.subplot(1,2,2)
pl.pcolor(dstGrd[1], dstGrd[0], vInterp[el, k,...], vmin=-20, vmax=20)
pl.title('test6: vInterp[%d, %d,...]' % (el, k))
pl.colorbar()
def Plot():
u, G = Calculator()
print 'G is', G
print np.amax(G), np.amin(G)
zvals = G
# make a color map of fixed colors
# tell imshow about color map so that only set colors are used
#img = plt.imshow(zvals)
img = plt.imshow(zvals,interpolation='nearest')
# make a color bar
plt.pcolor(zvals, vmin=0, vmax=0.05)
plt.colorbar()
#plt.colorbar(img,cmap=cmap,
# norm=norm,boundaries=bounds,ticks=[-5,0,5])
plt.show()
def draw_out_put_digit(data, n, ans, recog):
"""
学習済みのモデルが判定した画像を描画します.
@param data 画像データ
@param n 画像の通し番号
@param ans 正解ラベル
@param recog 推定した数字
"""
plt.subplot(10, 10, n)
Z = data.reshape(SIZE, SIZE)
Z = Z[::-1, :]
plt.xlim(0, 27)
plt.ylim(0, 27)
plt.pcolor(Z)
plt.title('ans=%d, recog=%d' % (ans, recog), size=8)
plt.gray()
plt.tick_params(labelbottom='off')
plt.tick_params(labelleft='off')
def histogram2d(x, y, bins=10, range=None, normed=False, weights=None,
log = False,
filename = None,
**formating):
hist, x, y = numpy.histogram2d(x, y, bins, range, normed, weights)
if log is True:
hist = numpy.log(hist)
X, Y = pylab.meshgrid(x,y)
pylab.pcolor(X, Y,hist.transpose())
pylab.colorbar()
doFormating(**formating)
pylab.show()
if filename is not None:
pylab.savefig(filename)
pylab.clf()
def test5_LibCF_LevelTime(self):
"""
Interpolate over one level/time
"""
f = cdms2.open(sys.prefix + '/sample_data/clt.nc')
clt = f('clt')
v = f('v')
# mask
srcGridMask = numpy.array(v[0,0,...] == v.missing_value, numpy.int32)
# v onto the ctl grid
srcGrd, srcNDims = regrid2.gsRegrid.makeCurvilinear([v.getLatitude(),
v.getLongitude()])
dstGrd, dstNDims = regrid2.gsRegrid.makeCurvilinear([clt.getLatitude(),
clt.getLongitude()])
ro = regrid2.GenericRegrid(srcGrd, dstGrd,
clt.dtype,
regridMethod = 'linear',
regridTool = 'esmf',
periodicity = 1,
coordSys = 'cart',
srcGridMask = srcGridMask)
ro.computeWeights()
vInterp = numpy.ones(clt.shape[-2:],
v.dtype) * v.missing_value
ro.apply(numpy.array(v[0,0,...]), vInterp, rootPe = 0)
print 'min/max of v: %f %f' % (v.min(), v.max())
print 'min/max of vInterp: %f %f' % (vInterp.min(), vInterp.max())
if False:
pl.figure()
pl.subplot(1,2,1)
pl.pcolor(srcGrd[1], srcGrd[0], v[0, 0,...], vmin=-20, vmax=20)
pl.title('test5: v[0, 0,...]')
pl.colorbar()
pl.subplot(1,2,2)
pl.pcolor(dstGrd[1], dstGrd[0], vInterp, vmin=-20, vmax=20)
pl.title('test5: vInterp')
pl.colorbar()
def testSalinityModel(self):
print "\nACCESS Salinity model"
srcGrid = self.so.getGrid()
dstGrid = self.clt.getGrid()
ro = CdmsRegrid(srcGrid = srcGrid,
dstGrid = dstGrid,
dtype = self.so.dtype,
regridTool = 'gsregrid', # "ESMp",
regridMethod = "Linear")
soInterp = ro(self.so)
print 'type(soInterp) = ', type(soInterp)
soMin, soMax = self.so.min(), self.so.max()
print "min/max of self.so: %f %f" % (soMin, soMax)
soInterpMin, soInterpMax = soInterp.min(), soInterp.max()
print "min/max of soInterp: %f %f" % (soInterpMin, soInterpMax)
self.assertEqual(self.so.missing_value, soInterp.missing_value)
self.assertLess(soInterpMax, 1.01*soMax)
self.assertLess(0.99*soMin, soInterpMin)
self.assertEqual(soInterp.shape[0], self.so.shape[0])
self.assertEqual(soInterp.shape[1], self.so.shape[1])
self.assertEqual(soInterp.shape[2], dstGrid.shape[0])
self.assertEqual(soInterp.shape[3], dstGrid.shape[1])
if False:
nTimes = self.so.shape[0]
nLevels = 3
for time in range(nTimes):
pl.figure(time)
f = 1
for k in [0,15,30]:
pl.subplot(3,2,f)
pl.pcolor(self.so[time, k, ...], vmin = 20, vmax = 40)
pl.title("so[%d, %d,...]" % (time, k))
pl.colorbar()
pl.subplot(3,2,f+1)
pl.pcolor(soInterp[time, k, ...], vmin = 20, vmax = 40)
pl.title("so[%d, %d,...]" % (time, k))
pl.colorbar()
f+=2
pl.suptitle("ACCESS Salinity Test for Time + Levels")
def testMultipleTimesAndElevations(self):
"""
Interpolate over time and elevation axes
"""
f = cdms2.open(sys.prefix + '/sample_data/clt.nc')
clt = f('clt')
v = f('v')
srcGrid = v.getGrid()
dstGrid = clt.getGrid()
ro = CdmsRegrid(srcGrid = srcGrid,
dstGrid = dstGrid,
dtype = v.dtype)
vInterp = ro(v)
mask = (v == v.missing_value)
if self.rank == 0:
vMin, vMax = v.min(), (v*(1-mask)).max()
vInterpMin, vInterpMax = vInterp.min(), (vInterp).max()
print 'min/max of v: %f %f' % (vMin, vMax)
print 'min/max of vInterp: %f %f' % (vInterpMin, vInterpMax)
self.assertLess(abs(vMin - vInterpMin), 0.4)
self.assertLess(abs(vMax - vInterpMax), 0.2)
if False and self.rank == 0:
nTimes = v.shape[0]
nLevels = v.shape[1]
for el in range(nTimes):
for k in range(nLevels):
pl.figure()
pl.subplot(1,2,1)
pl.pcolor(v[el, k,...], vmin=-20, vmax=20)
pl.title('testMultipleTimesAndElevations: v[%d, %d,...]' % (el, k))
pl.colorbar()
pl.subplot(1,2,2)
pl.pcolor(vInterp[el, k,...], vmin=-20, vmax=20)
pl.title('testMultipleTimesAndElevations: vInterp[%d, %d,...]' % (el, k))
pl.colorbar()
def plot_comparison_2D(X, Y, elec_pos, true_csd, true_pots, rec_csd, rec_pots, err_csd, err_pot):
"""
Plot for comparing model and reconstructed LFP and CSD in 2D.
"""
fig = plt.figure()
ax11 = fig.add_subplot(2, 3, 1)
pc1 = plt.pcolor(X, Y, true_csd)
plt.colorbar(pc1)
ax11.set_title('True CSD')
ax12 = fig.add_subplot(2, 3, 2)
pc2 = plt.pcolor(X, Y, rec_csd)
plt.colorbar(pc2)
ax12.set_title('Reconstructed CSD')
ax13 = fig.add_subplot(2, 3, 3)
pc3 = ax13.pcolor(X, Y, err_csd)
plt.colorbar(pc3)
ax13.set_title('CSD reconstruction error [%]')
ax21 = fig.add_subplot(2, 3, 4)
pc4 = plt.pcolor(X, Y, true_pots, cmap='RdYlBu')
plt.colorbar(pc4)
ax21.set_title('True LFP (forward scheme calculation from CSD)')
ax22 = fig.add_subplot(2, 3, 5)
pc5 = plt.pcolor(X, Y, rec_pots, cmap='RdYlBu')
plt.colorbar(pc5)
ax22.scatter(elec_pos[:, 0], elec_pos[:, 1],
marker='o', c='b', s=3, zorder=10)
ax22.set_xlim([np.min(X), np.max(X)])
ax22.set_ylim([np.min(Y), np.max(Y)])
ax22.set_title('Reconstructed LFP')
ax23 = fig.add_subplot(2, 3, 6)
pc6 = ax23.pcolor(X, Y, err_pot, cmap='RdYlBu')
plt.colorbar(pc6)
ax23.set_title('LFP reconstruction Error [%]')
plt.show()
def test_2d_libcf(self):
#print 'running test_2d_libcf...'
f = cdms2.open(cdat_info.get_sampledata_path() + \
'/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc')
so = f('so')[0, 0, :, :]
clt = cdms2.open(cdat_info.get_sampledata_path() + '/clt.nc')('clt')[0, :, :]
tic = time.time()
soInterp = so.regrid(clt.getGrid(), regridTool='libcf')
soInterpInterp = soInterp.regrid(so.getGrid(), regridTool='libcf')
toc = time.time()
#print 'time to interpolate forward/backward (gsRegrid): ', toc - tic
ntot = reduce(operator.mul, so.shape)
avgdiff = numpy.sum(so - soInterpInterp) / float(ntot)
#print 'avgdiff = ', avgdiff
self.assertLess(abs(avgdiff), 7.e-3)
if PLOT:
pylab.figure(1)
pylab.pcolor(abs(so - soInterpInterp), vmin=0.0, vmax=1.0)
pylab.colorbar()
pylab.title('gsRegrid')
def test_2d_esmf(self):
#print 'running test_2d_esmf...'
f = cdms2.open(sys.prefix + \
'/sample_data/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc')
so = f('so')[0, 0, :, :]
clt = cdms2.open(sys.prefix + '/sample_data/clt.nc')('clt')[0, :, :]
tic = time.time()
soInterp = so.regrid(clt.getGrid(), regridTool='ESMF') #, periodicity=1)
soInterpInterp = soInterp.regrid(so.getGrid(), regridTool='ESMF')
toc = time.time()
#print 'time to interpolate (ESMF linear) forward/backward: ', toc - tic
ntot = reduce(operator.mul, so.shape)
avgdiff = numpy.sum(so - soInterpInterp) / float(ntot)
#print 'avgdiff = ', avgdiff
self.assertLess(abs(avgdiff), 5.2e18)
if PLOT:
pylab.figure(2)
pylab.pcolor(abs(so - soInterpInterp), vmin=0.0, vmax=1.0)
pylab.colorbar()
pylab.title('ESMF linear')
def test5_esmf_conserve(self):
diagDeg = {'srcAreaFractions':None, 'srcAreas':None, 'dstAreas':None}
diagCar = {'srcAreaFractions':None, 'srcAreas':None, 'dstAreas':None}
soDegree = self.giss.regrid(self.tasGood.getGrid(), regridTool='esmf',
regridMethod='conserve',
coordSys='degrees', diag = diagDeg)
soCartes = self.giss.regrid(self.tasGood.getGrid(), regridTool='esmf',
regridMethod='conserve',
coordSys='cart', diag = diagCar)
smDeg = (self.giss * diagDeg['srcAreaFractions'] * diagDeg['srcAreas']).sum()
dmDeg = (self.giss * diagDeg['srcAreaFractions'] * diagDeg['srcAreas']).sum()
smCar = (self.giss * diagCar['srcAreaFractions'] * diagCar['srcAreas']).sum()
dmCar = (self.giss * diagCar['srcAreaFractions'] * diagCar['srcAreas']).sum()
if self.pe == 0:
self.assertLess(abs(smDeg - dmDeg), 1.e-5)
self.assertLess(abs(smCar - dmCar), 1.e-5)
if Plot:
fig = pl.figure('Giss -> HadGem2')
fig.add_subplot(2,2,1)
pl.pcolor(self.giss, vmin = 20, vmax = 40)
pl.colorbar()
pl.title('self.giss')
fig.add_subplot(2,2,2)
pl.pcolor(soDegree, vmin = 20, vmax = 40)
pl.colorbar()
pl.title('soDegree')
fig.add_subplot(2,2,3)
pl.pcolor(soCartes, vmin = 20, vmax = 40)
pl.colorbar()
pl.title('soCartes')
def split_solution(self,outdir=None,split_q=True,split_aux=True,update_aux=None,debug=False,ptc_split=None):
if outdir is None: outdir = self.outdir
if update_aux is None: update_aux = self.update_aux
if ptc_split is None: ptc_split=self.mpi_split
sampling = self.sampling
if debug: plt.figure()
for frame in self.frame_range:
if ptc_split:
if PETSc.COMM_WORLD.rank==0: print frame
if split_q:
ptcname = self._ptc_base+str(frame).zfill(4)
ptcfile = os.path.join(outdir,ptcname)
self.ptc_split(outdir=outdir,ptcfile=ptcfile,num_var=self.num_eqn,
affix=['sol'+str(frame).zfill(4),'s'],
irange=self.s_range)
if split_aux:
auxname = self._aux_base+str(frame).zfill(4)
auxfile = os.path.join(outdir,auxname)
self.ptc_split(outdir=outdir,ptcfile=auxfile,num_var=self.num_aux,
affix=['sol_aux'+str(frame).zfill(4),'n'],
poynting=False,irange=self.aux_range)
if not update_aux: split_aux=False
else:
solution = Solution()
solution.read(frame,path=outdir,file_format='petsc',read_aux=split_aux)
q = solution.state.get_q_global()
if self.num_dim==2:
if self.expand_2d:
q = q[:,::sampling,::sampling,np.newaxis]
else:
q = q[:,::sampling,::sampling,::sampling]
if debug:
plt.subplot(3,1,1)
plt.pcolor(q[1,:,:,16])
plt.subplot(3,1,2)
plt.pcolor(q[1,:,:,0])
self.write_split(q,'q',frame=frame,irange=self.q_range,outdir=outdir)
if split_aux:
aux = solution.state.get_aux_global()
if self.num_dim==2:
if self.expand_2d:
aux = aux[:,::sampling,::sampling,np.newaxis]
else:
aux = aux[:,::sampling,::sampling,::sampling]
if debug:
plt.subplot(3,1,3)
plt.pcolor(aux[0,:,:,16])
plt.show()
self.write_split(aux,'aux',frame=frame,irange=self.aux_range,outdir=outdir)
if not update_aux: split_aux=False
def createAndSaveHeatMap(dataArray, figFileRoot, xLabel="", yLabel="", xMin=0,
xMax=-1, yMin=0, yMax=-1, colorMap=0, maxInt=0,
vMin=0, vMax=0, fontSize=20, majorFontSize=18,
pngDPI=150, svgDPI=75, svgFlag=0):
"""Make a 2D intensity map of the data and save it to file."""
colorList = [None, plt.cm.Reds, plt.cm.Greens, plt.cm.Greys]
if vMin == -10:
vMin = 0
if vMax == 0:
for a in dataArray.flat:
if not isnan(a) and a > vMax:
vMax = a
plt.pcolor(dataArray, cmap=colorList[colorMap], vmin=vMin, vmax=vMax)
plt.xlabel(xLabel, fontsize=fontSize)
plt.ylabel(yLabel, fontsize=fontSize)
plt.colorbar(orientation="vertical")
ax = plt.gca()
for tick in ax.xaxis.get_major_ticks():
tick.label1.set_fontsize(majorFontSize)
for tick in ax.yaxis.get_major_ticks():
tick.label1.set_fontsize(majorFontSize)
if xMax == -1:
plt.xlim()
else:
plt.xlim(xMin, xMax)
if yMax == -1:
plt.ylim()
else:
plt.ylim(yMin, yMax)
if maxInt != 0:
plt.clim(0, maxInt)
plt.savefig(figFileRoot + '.png', dpi=pngDPI)
if svgFlag == 1:
plt.savefig(figFileRoot + '.svg', dpi=svgDPI)
plt.clf()
