def xxxxtest_median_model():
x = Data(None, None)
x.label = 'nothing'
d = np.random.random((100, 1, 1))
x.data = np.ma.array(d, mask= d!=d)
# odd number and no masked values
a = x.copy()
a.data[:, 0, 0] = 1.
b = x.copy()
b.data[:, 0, 0] = 3.
c = x.copy()
c.data[:, 0, 0] = 2.
d = x.copy()
d.data[:, 0, 0] = 5.
e = x.copy()
e.data[:, 0, 0] = 4.
m = MedianModel()
m.add_member(a)
m.add_member(b)
m.add_member(c)
m.add_member(d)
m.add_member(e)
m.ensmedian()
# should give the value of 3. for all timesteps
del m
# even number and no masked values
a = x.copy()
a.data[:, 0, 0] = 1.
b = x.copy()
b.data[:, 0, 0] = 3.
c = x.copy()
c.data[:, 0, 0] = 2.
d = x.copy()
c.data[:, 0, 0] = 4.
m = MedianModel()
m.add_member(a)
m.add_member(b)
m.add_member(c)
m.add_member(d)
m.ensmedian()
# should give the value of 2.5 for all timesteps
del m
def xxxxtest_median_model():
x = Data(None, None)
x.label = 'nothing'
d = np.random.random((100, 1, 1))
x.data = np.ma.array(d, mask=d != d)
# odd number and no masked values
a = x.copy()
a.data[:, 0, 0] = 1.
b = x.copy()
b.data[:, 0, 0] = 3.
c = x.copy()
c.data[:, 0, 0] = 2.
d = x.copy()
d.data[:, 0, 0] = 5.
e = x.copy()
e.data[:, 0, 0] = 4.
m = MedianModel()
m.add_member(a)
m.add_member(b)
m.add_member(c)
m.add_member(d)
m.add_member(e)
m.ensmedian()
# should give the value of 3. for all timesteps
del m
# even number and no masked values
a = x.copy()
a.data[:, 0, 0] = 1.
b = x.copy()
b.data[:, 0, 0] = 3.
c = x.copy()
c.data[:, 0, 0] = 2.
d = x.copy()
c.data[:, 0, 0] = 4.
m = MedianModel()
m.add_member(a)
m.add_member(b)
m.add_member(c)
m.add_member(d)
m.ensmedian()
# should give the value of 2.5 for all timesteps
del m
# -*- coding: utf-8 -*- """ This file is part of pyCMBS. (c) 2012- Alexander Loew For COPYING and LICENSE details, please refer to the LICENSE files """ from pycmbs.data import Data from pycmbs.plots import map_difference import matplotlib.pyplot as plt file_name = '../../../pycmbs/examples/example_data/air.mon.mean.nc' A = Data(file_name, 'air', lat_name='lat', lon_name='lon', read=True, label='air temperature') B = A.copy() B.mulc(2.3, copy=False) a = A.get_climatology(return_object=True) b = B.get_climatology(return_object=True) # a quick plot as well as a projection plot f1 = map_difference(a, b, show_stat=False, vmin=-30., vmax=30., dmin=-60., dmax=60.) # unprojected plt.show()
# -*- coding: utf-8 -*-
"""
This file is part of pyCMBS.
(c) 2012- Alexander Loew
For COPYING and LICENSE details, please refer to the LICENSE file
"""
from pycmbs.data import Data
from pycmbs.diagnostic import PatternCorrelation
import matplotlib.pyplot as plt
import numpy as np
file_name = '../../../pycmbs/examples/example_data/air.mon.mean.nc'
A = Data(file_name,
'air',
lat_name='lat',
lon_name='lon',
read=True,
label='air temperature')
B = A.copy()
B.mulc(2.3, copy=False)
B.data = B.data + np.random.random(B.shape) * 100.
# calculate spatial correlation for all timesteps ...
P = PatternCorrelation(A, B)
# ... and vizalize it
P.plot()
plt.show()
COPYRIGHT.md """ from pycmbs.data import Data import numpy as np fname = '../pycmbs/examples/example_data/air.mon.mean.nc' d = Data(fname, 'air', read=True) c = d.get_climatology(return_object=True) print 'c raw: ', c.fldmean() print c.date print '' # create some invalid data d1 = d.copy() t = d1.time * 1. d1.time[20:] = t[0:-20] d1.time[0:20] = t[-20:] tmp = d1.data * 1. d1.data[20:, :, :] = tmp[0:-20, :, :] d1.data[0:20, :, :] = tmp[-20:, :, :] c1 = d1.get_climatology(return_object=True, ensure_start_first=True) print '' print c1.date print c1.fldmean()
class TestEOF(TestCase):
def setUp(self):
#init Data object for testing
n=4 #slows down significantly! constraint is percentile test
x = sc.randn(n)*100. #generate dummy data
self.D = Data(None,None)
d=np.ones((n,1,2))
self.D.data = d
self.D.data[:,0,0]=x
self.D.data = np.ma.array(self.D.data,mask=self.D.data != self.D.data)
self.D.verbose = True
self.D.unit = 'myunit'
self.D.time = np.arange(n) + pl.datestr2num('2001-01-01') - 1
def test_eof_analysis(self):
#test of EOF
#example taken from:
# http://www.atmos.washington.edu/~dennis/552_Notes_4.pdf , page 80
#assign sampled data
D = self.D.copy()
# d1 = np.array([2,4,-6,8])
# d2 = np.array([1,2,-3,4])
#d1 = np.array([2,1])
#d2 = np.array([4,2])
#d3 = np.array([-6,-3])
#d4 = np.array([8,4])
#D.data[:,0,0] = d1; D.data[:,0,1] = d2
#D.data[:,0,2] = d3; D.data[:,0,3] = d4
# D.data[:,0,0] = d1
# D.data[:,0,1] = d2
#
# E = EOF(D,cov_norm=False) #do not normalize covariance matrix, as this is also not done in example
# print E.C #covariance matrix
# shape of EOF is wrong !!!!!!!
#
# something is not really working here !!!!
#
#
# irgendwie ist hier ein problem
# warum einmal 4x4 und einmal 2x2 ???
# print 'eigval'
# print E.eigval
#
# print 'eigvec'
# print E.eigvec
pass
class TestData(TestCase):
def setUp(self):
# init Data object for testing
n=100 # slows down significantly! constraint is percentile test
x = sc.randn(n)*100. # generate dummy data
self.D = Data(None, None)
d=np.ones((n, 1, 1))
self.D.data = d
self.D.data[:,0,0]=x
self.D.data = np.ma.array(self.D.data, mask=self.D.data != self.D.data)
self.D.verbose = True
self.D.unit = 'myunit'
self.D.label = 'testlabel'
self.D.filename = 'testinputfilename.nc'
self.D.varname = 'testvarname'
self.D.long_name = 'This is the longname'
self.D.time = np.arange(n) + pl.datestr2num('2001-01-01') - 1
self.D.time_str = "days since 0001-01-01 00:00:00"
self.D.calendar = 'gregorian'
self.D.cell_area = np.ones_like(self.D.data[0,:,:])
@unittest.skip('wait for bug free scipy')
def test_pattern_correlation(self):
"""
test pattern correlation function
"""
x = self.D.copy()
# correlation with random values
y = self.D.copy()
tmp = np.random.random(y.shape)
y.data = np.ma.array(tmp, mask=tmp != tmp)
P2 = PatternCorrelation(x, y)
P2._correlate()
self.assertEqual(x.nt,len(P2.r_value))
self.assertEqual(x.nt,len(P2.t))
for i in xrange(x.nt):
slope, intercept, r_value, p_value, std_err = stats.mstats.linregress(x.data[i,:,:].flatten(),y.data[i,:,:].flatten())
self.assertEqual(P2.r_value[i], r_value)
self.assertEqual(P2.p_value[i], p_value)
self.assertEqual(P2.slope[i], slope)
self.assertEqual(P2.intercept[i], intercept)
self.assertEqual(P2.std_err[i], std_err)
def test_gleckler_index(self):
"""
test Reichler index/Gleckler plot
"""
# generate sample data
# sample data
tmp = np.zeros((5, 3, 1))
tmp[:,0,0] = np.ones(5)*1.
tmp[:,1,0] = np.ones(5)*2.
tmp[:,2,0] = np.ones(5)*5.
# The data is like ...
#| 1 | 2 | 5 |
#| 1 | 2 | 5 |
#| 1 | 2 | 5 |
#| 1 | 2 | 5 |
#| 1 | 2 | 5 |
x = self.D.copy()
x._temporal_subsetting(0, 4)
x.data = np.ma.array(tmp, mask=tmp!=tmp)
x.std = np.ones(x.data.shape)
x.time[0] = pl.datestr2num('2000-02-15')
x.time[1] = pl.datestr2num('2000-03-15')
x.time[2] = pl.datestr2num('2000-04-15')
x.time[3] = pl.datestr2num('2000-05-15')
x.time[4] = pl.datestr2num('2000-06-15')
y = self.D.copy()
y._temporal_subsetting(0, 4)
tmp = np.ones(x.data.shape) # sample data 2
y.data = np.ma.array(tmp, mask=tmp!=tmp)
y.time[0] = pl.datestr2num('2000-02-15')
y.time[1] = pl.datestr2num('2000-03-15')
y.time[2] = pl.datestr2num('2000-04-15')
y.time[3] = pl.datestr2num('2000-05-15')
y.time[4] = pl.datestr2num('2000-06-15')
# Case 1: same area weights
# cell area
tmp = np.ones((3, 1))
x.cell_area = tmp*1.
#| 1-1 | 2-1 | 5-1 |
#| 1-1 | 2-1 | 5-1 |
#| 1-1 | 2-1 | 5-1 |
#| 1-1 | 2-1 | 5-1 |
#| 1-1 | 2-1 | 5-1 |
#===================
#| 0 | 5 | 5*4**2=5*16. = 80 |
#==> E2 = sqrt(85./(15.))
D = GlecklerPlot()
r = D.calc_index(x, y, 'a', 'b', time_weighting=False)
wt = np.ones(5) / 5.
ref = np.sqrt(((85./15.) * wt).sum())
t = np.abs(1. - r / ref)
self.assertLess(t, 0.000001) # relative error
D = GlecklerPlot()
r = D.calc_index(x, y, 'a', 'b')
wt = np.asarray([29., 31., 30., 31., 30.])
wt = wt / wt.sum()
ref = np.sqrt(((85./15.) * wt).sum())
t = np.abs(1. - r / ref)
self.assertLess(t, 0.000001) # relative error
# Case 2: Different area weights
# cell area
tmp = np.ones((3, 1))
tmp[1, 0] = 2.
x.cell_area = tmp*1.
#| 1-1=0 | 2-1=1 | 5-1=16 |
#| 1-1=0 | 2-1=1 | 5-1=16 |
#| 1-1=0 | 2-1=1 | 5-1=16 |
#| 1-1=0 | 2-1=1 | 5-1=16 |
#| 1-1=0 | 2-1=1 | 5-1=16 |
#--------------------------
# w = 0.25 w = 0.5 w=0.25|
#--------------------------
# 0.25*0 + 0.5 * 1 + 0.25 * 16 = 0 + 0.5 + 4 = 4.5
# the mean of that is 4.5 for each timestep
# mean because the overall weights are calculated as such that
# they give a total weight if 1
# diagnostic
D = GlecklerPlot()
r = D.calc_index(x, y, 'a', 'b', time_weighting=False)
wt = np.ones(5) / 5.
ref = np.sqrt((4.5 * wt).sum())
t = np.abs(1. - r / ref)
self.assertLess(t, 0.000001) # relative error
wt = np.asarray([29., 31., 30., 31., 30.])
wt = wt / wt.sum()
ref = np.sqrt((4.5 * wt).sum())
t = np.abs(1. - r / ref)
self.assertLess(t, 0.000001) # relative error
# Case 3: use different std
x.std = np.ones(x.data.shape)
x.std[:, 2, 0] = 0.5
#| 1-1=0 | 2-1=1 | 5-1=16 / 0.5 |
#| 1-1=0 | 2-1=1 | 5-1=16 / 0.5 |
#| 1-1=0 | 2-1=1 | 5-1=16 / 0.5 |
#| 1-1=0 | 2-1=1 | 5-1=16 / 0.5 |
#| 1-1=0 | 2-1=1 | 5-1=16 / 0.5 |
#--------------------------------
# w = 0.25 w = 0.5 w=0.25|
# 0 + 0.5 + 0.25*32 = 0.5 + 8 = 8.5
D = GlecklerPlot()
r = D.calc_index(x, y, 'a', 'b', time_weighting=False)
wt = np.ones(5) / 5.
ref = np.sqrt((8.5 * wt).sum())
t = np.abs(1. - r / ref)
self.assertLess(t, 0.000001) # relative error
wt = np.asarray([29., 31., 30., 31., 30.])
wt = wt / wt.sum()
ref = np.sqrt((8.5 * wt).sum())
t = np.abs(1. - r / ref)
self.assertLess(t, 0.000001) # relative error
def test_RegionalAnalysis_xNone(self):
region = RegionIndex(55, 1, 1, 1, 1, label='test')
R = RegionalAnalysis(None, self.D, region)
self.assertEqual(R.x, None)
def test_RegionalAnalysis_InvalidX(self):
region = RegionIndex(77, 1, 1, 1, 1, label='test')
with self.assertRaises(ValueError):
R = RegionalAnalysis([123.], self.D, region)
def test_RegionalAnalysis_InvalidY(self):
region = RegionIndex(88, 1, 1, 1, 1, label='test')
with self.assertRaises(ValueError):
R = RegionalAnalysis(self.D, [123.], region)
def test_RegionalAnalysis_yNone(self):
region = RegionIndex(55, 1, 1, 1, 1, label='test')
R = RegionalAnalysis(self.D, None, region)
self.assertEqual(R.y, None)
def test_RegionalAnalysis_InvalidRegion(self):
region = 1.
with self.assertRaises(ValueError):
R = RegionalAnalysis(self.D, self.D, region)
def test_RegionalAnalysis_InvalidGeometry(self):
region = RegionIndex(99, 1, 1, 1, 1, label='test')
x = self.D.copy()
y = self.D.copy()
y.data = np.random.random((2,3,4,5))
with self.assertRaises(ValueError):
R = RegionalAnalysis(x, y, region)
@unittest.skip('wait for solving logplot proplem in map_plot')
def test_EOF(self):
x = np.random.random((self.D.nt, 20, 30))
self.D.data = np.ma.array(x, mask=x != x)
self.D.cell_area = np.ones_like(self.D.data[0,:,:])
E = EOF(self.D)
r = E.reconstruct_data()
c = E.get_correlation_matrix()
E.get_eof_data_correlation()
#~ E.plot_channnel_correlations(100000) #slow!!
E.plot_eof_coefficients(None, all=True)
E._calc_anomalies()
E.plot_EOF(None, all=True)
#~ def test_koeppen(self):
#~ T = self.D.copy()
#~ T.data = np.random.random((10,20,30))
#~ T.unit = 'K'
#~ P = self.D.copy()
#~ P.data = np.random.random((10,20,30))
#~ P.unit = 'kg/m^2s'
#~ lsm = self.D.copy()
#~ lsm.unit = 'fractional'
#~ lsm.data = np.ones((20,30))
#~
#~ k = Koeppen(temp=T, precip=P, lsm=lsm)
def test_koeppen_InvalidInput(self):
T = self.D.copy()
P = self.D.copy()
lsm = self.D.copy()
with self.assertRaises(ValueError):
k = Koeppen(temp=None, precip=P, lsm=lsm)
with self.assertRaises(ValueError):
k = Koeppen(temp=T, precip=None, lsm=lsm)
with self.assertRaises(ValueError):
k = Koeppen(temp=T, precip=P, lsm=None)
class TestPycmbsPlots(unittest.TestCase):
def setUp(self):
self.D = Data(None, None)
self.D._init_sample_object(nt=1000, ny=1, nx=1)
self._tmpdir = tempfile.mkdtemp()
def test_ReichlerPlotGeneral(self):
RP = ReichlerPlot()
for i in xrange(10):
RP.add([i * 12.], 'test' + str(i))
RP.simple_plot()
RP.bar(title='some title', vmin=-10., vmax=10.)
#~ RP.circle_plot()
def test_ScatterPlot_General(self):
x = self.D
S = ScatterPlot(x)
S.plot(x)
S.legend()
def test_rotate_ticks(self):
f = plt.figure()
ax = f.add_subplot(111)
ax.plot(np.random.random(1000))
rotate_ticks(ax, 20.)
def test_correlation_analysis(self):
x = self.D
y = self.D
C = CorrelationAnalysis(x, y)
C.do_analysis()
def test_ScatterPlot_GeneralWithNormalization(self):
x = self.D
S = ScatterPlot(x, normalize_data=True)
S.plot(x)
S.legend()
def test_ScatterPlot_FldemeanFalse(self):
x = self.D
S = ScatterPlot(x)
S.plot(x, fldmean=False)
S.legend()
def test_ScatterPlot_InvalidShape(self):
x = self.D
S = ScatterPlot(x)
y = self.D.copy()
y.data = np.random.random((10, 20, 30, 40))
with self.assertRaises(ValueError):
S.plot(y, fldmean=False)
def test_LinePlot_General(self):
x = self.D
L = LinePlot()
L1 = LinePlot(regress=True)
L.plot(x)
L1.plot(x)
def test_LinePlot_WithAxis(self):
x = self.D
f = plt.figure()
ax = f.add_subplot(111)
L = LinePlot(ax=ax)
L.plot(x)
def test_HistogrammPlot_General(self):
H = HistogrammPlot(normalize=True)
H.plot(self.D, bins=10, shown=True)
def test_ZonalPlot(self):
Z = ZonalPlot()
Z.plot(self.D)
def test_map_difference_General(self):
map_difference(self.D, self.D)
def test_GlecklerPlot_InvalidNumberOfObservations(self):
G = GlecklerPlot()
G.add_model('echam5')
G.add_model('mpi-esm')
G.add_variable('ta')
G.add_data('ta', 'echam5', 0.5, pos=1)
G.add_data('ta', 'echam5', 0.25, pos=2)
G.add_data('ta', 'echam5', -0.25, pos=3)
G.add_data('ta', 'mpi-esm', -0.25, pos=4)
G.add_data('ta', 'mpi-esm', -0.25, pos=5)
with self.assertRaises(ValueError):
G.plot()
def test_GlecklerPlot_4obs(self):
G = GlecklerPlot()
G.add_model('echam5')
G.add_model('mpi-esm')
G.add_variable('ta')
G.add_variable('P')
G.add_data('P', 'echam5', 0.5, pos=1)
G.add_data('P', 'echam5', 0.25, pos=2)
G.add_data('P', 'echam5', -0.25, pos=3)
G.add_data('P', 'mpi-esm', -0.25, pos=4)
G.plot()
def test_GlecklerPlot(self):
G = GlecklerPlot()
G.add_model('echam5')
G.add_model('mpi-esm')
G.add_variable('ta')
G.add_variable('P')
G.add_data('ta', 'echam5', 0.5, pos=1)
G.add_data('P', 'echam5', 0.25, pos=1)
G.add_data('P', 'echam5', -0.25, pos=2)
G.add_data('P', 'mpi-esm', -0.25, pos=1)
G.plot()
G.plot_model_error('ta')
G.plot_model_ranking('ta')
G.write_ranking_table('ta',
self._tmpdir + os.sep + 'nix.tex',
fmt='latex')
self.assertTrue(os.path.exists(self._tmpdir + os.sep + 'nix.tex'))
if os.path.exists(self._tmpdir + os.sep + 'nix.tex'):
os.remove(self._tmpdir + os.sep + 'nix.tex')
G.write_ranking_table('ta',
self._tmpdir + os.sep + 'nix1',
fmt='latex')
self.assertTrue(os.path.exists(self._tmpdir + os.sep + 'nix1.tex'))
if os.path.exists(self._tmpdir + os.sep + 'nix1.tex'):
os.remove(self._tmpdir + os.sep + 'nix1.tex')
def test_old_map_plot(self):
xx_map_plot(self.D)
#~ xx_map_plot(self.D, use_basemap=True)
def test_HstackTimeSeries(self):
HT = HstackTimeseries()
for i in xrange(15):
x = np.random.random(100) * 2. - 1.
HT.add_data(x, 'model' + str(i).zfill(3))
HT.plot(cmap='RdBu_r',
interpolation='nearest',
vmin=-1.,
vmax=1.,
nclasses=15,
title='Testtitle')
def test_HstackTimeSeries_invalidData(self):
HT = HstackTimeseries()
x = np.random.random((50, 60)) * 2. - 1.
with self.assertRaises(ValueError):
HT.add_data(x, 'test')
def test_HstackTimeSeries_inconsistent_data_geometries(self):
HT = HstackTimeseries()
x = np.random.random(100) * 2. - 1.
y = np.random.random(1000) * 2. - 1.
HT.add_data(x, 'A')
with self.assertRaises(ValueError):
HT.add_data(y, 'B')
def test_HstackTimeSeries_duplicate_keys(self):
HT = HstackTimeseries()
x = np.random.random(100) * 2. - 1.
HT.add_data(x, 'A')
with self.assertRaises(ValueError):
HT.add_data(x, 'A')
def test_violin_plot(self):
Violin_example()
def test_globalmeanplot(self):
G = GlobalMeanPlot()
with self.assertRaises(ValueError):
G.plot(self.D, stat_type='no_stat_type')
G.plot(self.D, show_std=True)
G.plot_mean_result()
def test_HstackTimeSeries_monthly_ticks(self):
HT = HstackTimeseries()
f = plt.figure()
ax = f.add_subplot(111)
ax.plot(np.arange(12))
HT._set_monthly_xtick_labels(ax)
def test_HstackTimeSeries_noplotdone(self):
HT = HstackTimeseries()
with self.assertRaises(ValueError):
HT._add_colorbar()
def test_GlobalMean(self):
GM1 = GlobalMeanPlot(climatology=False)
self.assertEqual(GM1.nplots, 1)
f = plt.figure()
ax1 = f.add_subplot(111)
axA = f.add_subplot(211)
axB = f.add_subplot(212)
with self.assertRaises(ValueError):
GM2 = GlobalMeanPlot(climatology=True, ax=ax1)
GM2 = GlobalMeanPlot(climatology=True, ax=axA, ax1=axB)
self.assertEqual(GM2.nplots, 2)
def test_Hovmoeller(self):
H = HovmoellerPlot(self.D)
with self.assertRaises(ValueError):
H.plot()
H.plot(climits=[0., 1.])
class TestPycmbsBenchmarkingModels(unittest.TestCase):
def setUp(self):
n=1000 # slows down significantly! constraint is percentile test
x = sc.randn(n)*100. # generate dummy data
self.D = Data(None, None)
d=np.ones((n, 1, 1))
self.D.data = d
self.D.data[:,0,0]=x
self.D.data = np.ma.array(self.D.data, mask=self.D.data != self.D.data)
self.D.verbose = True
self.D.unit = 'myunit'
self.D.label = 'testlabel'
self.D.filename = 'testinputfilename.nc'
self.D.varname = 'testvarname'
self.D.long_name = 'This is the longname'
self.D.time = np.arange(n) + pl.datestr2num('2001-01-01')
self.D.time_str = "days since 0001-01-01 00:00:00"
self.D.calendar = 'gregorian'
self.D.oldtime=False
# generate dummy Model object
data_dir = './test/'
varmethods = {'albedo':'get_albedo()', 'sis': 'get_sis()'}
self.model = models.Model(data_dir, varmethods, name='testmodel', intervals='monthly')
sis = self.D.copy()
sis.mulc(5., copy=False)
sis.label='sisdummy'
alb = self.D.copy()
alb.label='albedodummy'
# add some dummy data variable
self.model.variables = {'albedo':alb, 'sis':sis}
def test_save_prefix_missing(self):
m = self.model
odir = './odir/'
with self.assertRaises(ValueError):
m.save(odir)
def test_save_create_odir(self):
m = self.model
odir = './odir/'
if os.path.exists(odir):
os.system('rm -rf ' + odir)
m.save(odir, prefix='test')
self.assertTrue(os.path.exists(odir))
os.system('rm -rf ' + odir)
def test_save(self):
m = self.model
odir = './odir/'
sisfile = odir + 'testoutput_SIS.nc'
albfile = odir + 'testoutput_ALBEDO.nc'
if os.path.exists(sisfile):
os.remove(sisfile)
if os.path.exists(albfile):
os.remove(albfile)
m.save(odir, prefix='testoutput')
self.assertTrue(os.path.exists(sisfile))
self.assertTrue(os.path.exists(albfile))
if os.path.exists(sisfile):
os.remove(sisfile)
if os.path.exists(albfile):
os.remove(albfile)
os.system('rm -rf ' + odir)
class TestPycmbsBenchmarkingModels(unittest.TestCase):
def setUp(self):
self.D = Data(None, None)
self.D._init_sample_object(nt=1000, ny=1, nx=1)
# generate dummy Model object
data_dir = './test/'
varmethods = {'albedo':'get_albedo()', 'sis': 'get_sis()'}
self.model = models.Model(data_dir, varmethods, name='testmodel', intervals='monthly')
sis = self.D.copy()
sis.mulc(5., copy=False)
sis.label='sisdummy'
alb = self.D.copy()
alb.label='albedodummy'
# add some dummy data variable
self.model.variables = {'albedo':alb, 'sis':sis}
def test_save_prefix_missing(self):
m = self.model
odir = tempfile.mkdtemp() + os.sep
with self.assertRaises(ValueError):
m.save(odir)
def test_save_create_odir(self):
m = self.model
odir = tempfile.mkdtemp() + os.sep
if os.path.exists(odir):
os.system('rm -rf ' + odir)
m.save(odir, prefix='test')
self.assertTrue(os.path.exists(odir))
os.system('rm -rf ' + odir)
def test_save(self):
m = self.model
odir = tempfile.mkdtemp() + os.sep
sisfile = odir + 'testoutput_SIS.nc'
albfile = odir + 'testoutput_ALBEDO.nc'
if os.path.exists(sisfile):
os.remove(sisfile)
if os.path.exists(albfile):
os.remove(albfile)
m.save(odir, prefix='testoutput')
self.assertTrue(os.path.exists(sisfile))
self.assertTrue(os.path.exists(albfile))
if os.path.exists(sisfile):
os.remove(sisfile)
if os.path.exists(albfile):
os.remove(albfile)
os.system('rm -rf ' + odir)
def test_cmip5_init_singlemember(self):
data_dir = tempfile.mkdtemp()
# invalid model identifier
with self.assertRaises(ValueError):
M = models.CMIP5RAW_SINGLE(data_dir, 'MPI-M:MPI-ESM-LR1', 'amip', {}, intervals='monthly')
with self.assertRaises(ValueError):
M = models.CMIP5RAW_SINGLE(data_dir, 'MPI-M:MPI-ESM-LR#1#2', 'amip', {}, intervals='monthly')
M1 = models.CMIP5RAW_SINGLE(data_dir, 'MPI-M:MPI-ESM-LR#1', 'amip', {}, intervals='monthly')
M2 = models.CMIP5RAW_SINGLE(data_dir, 'MPI-M:MPI-ESM-LR#728', 'amip', {}, intervals='monthly')
self.assertEqual(M1.ens_member, 1)
self.assertEqual(M2.ens_member, 728)
def test_cmip5_singlemember_filename(self):
data_dir = tempfile.mkdtemp()
# generate testfile
testfile = data_dir + os.sep + 'MPI-M' + os.sep + 'MPI-ESM-LR' + os.sep + 'amip' + os.sep + 'mon' + os.sep + 'atmos' + os.sep + 'Amon' + os.sep + 'r1i1p1' + os.sep + 'ta' + os.sep + 'ta_Amon_MPI-ESM-LR_amip_r1i1p1_197901-200812.nc'
os.makedirs(os.path.dirname(testfile))
os.system('touch ' + testfile)
self.assertTrue(os.path.exists(testfile))
M = models.CMIP5RAW_SINGLE(data_dir, 'MPI-M:MPI-ESM-LR#1', 'amip', {}, intervals='monthly')
f = M.get_single_ensemble_file('ta', mip='Amon', realm='atmos')
self.assertTrue(os.path.exists(f))
self.assertEqual(f, testfile)
class TestPycmbsPlots(unittest.TestCase):
def setUp(self):
self.D = Data(None, None)
self.D._init_sample_object(nt=1000, ny=1, nx=1)
self._tmpdir = tempfile.mkdtemp()
def test_ReichlerPlotGeneral(self):
RP = ReichlerPlot()
for i in xrange(10):
RP.add([i*12.], 'test'+str(i))
RP.simple_plot()
RP.bar(title='some title', vmin=-10., vmax=10.)
#~ RP.circle_plot()
def test_ScatterPlot_General(self):
x = self.D
S = ScatterPlot(x)
S.plot(x)
S.legend()
def test_rotate_ticks(self):
f = plt.figure()
ax=f.add_subplot(111)
ax.plot(np.random.random(1000))
rotate_ticks(ax, 20.)
def test_correlation_analysis(self):
x = self.D
y = self.D
C = CorrelationAnalysis(x, y)
C.do_analysis()
def test_ScatterPlot_GeneralWithNormalization(self):
x = self.D
S = ScatterPlot(x, normalize_data=True)
S.plot(x)
S.legend()
def test_ScatterPlot_FldemeanFalse(self):
x = self.D
S = ScatterPlot(x)
S.plot(x, fldmean=False)
S.legend()
def test_ScatterPlot_InvalidShape(self):
x = self.D
S = ScatterPlot(x)
y = self.D.copy()
y.data = np.random.random((10,20,30,40))
with self.assertRaises(ValueError):
S.plot(y, fldmean=False)
def test_LinePlot_General(self):
x = self.D
L = LinePlot()
L1 = LinePlot(regress=True)
L.plot(x)
L1.plot(x)
def test_LinePlot_WithAxis(self):
x = self.D
f = plt.figure()
ax = f.add_subplot(111)
L = LinePlot(ax=ax)
L.plot(x)
def test_HistogrammPlot_General(self):
H = HistogrammPlot(normalize=True)
H.plot(self.D, bins=10, shown=True)
def test_ZonalPlot(self):
Z = ZonalPlot()
Z.plot(self.D)
def test_map_difference_General(self):
map_difference(self.D, self.D)
def test_GlecklerPlot_InvalidNumberOfObservations(self):
G = GlecklerPlot()
G.add_model('echam5')
G.add_model('mpi-esm')
G.add_variable('ta')
G.add_data('ta', 'echam5', 0.5,pos=1)
G.add_data('ta', 'echam5',0.25,pos=2)
G.add_data('ta', 'echam5',-0.25,pos=3)
G.add_data('ta', 'mpi-esm',-0.25,pos=4)
G.add_data('ta', 'mpi-esm',-0.25,pos=5)
with self.assertRaises(ValueError):
G.plot()
def test_GlecklerPlot_4obs(self):
G = GlecklerPlot()
G.add_model('echam5')
G.add_model('mpi-esm')
G.add_variable('ta')
G.add_variable('P')
G.add_data('P', 'echam5', 0.5,pos=1)
G.add_data('P', 'echam5',0.25,pos=2)
G.add_data('P', 'echam5',-0.25,pos=3)
G.add_data('P', 'mpi-esm',-0.25,pos=4)
G.plot()
def test_GlecklerPlot(self):
G = GlecklerPlot()
G.add_model('echam5')
G.add_model('mpi-esm')
G.add_variable('ta')
G.add_variable('P')
G.add_data('ta', 'echam5', 0.5,pos=1)
G.add_data('P', 'echam5',0.25,pos=1)
G.add_data('P', 'echam5',-0.25,pos=2)
G.add_data('P', 'mpi-esm',-0.25,pos=1)
G.plot()
G.plot_model_error('ta')
G.plot_model_ranking('ta')
G.write_ranking_table('ta', self._tmpdir + os.sep + 'nix.tex', fmt='latex')
self.assertTrue(os.path.exists(self._tmpdir + os.sep + 'nix.tex'))
if os.path.exists(self._tmpdir + os.sep + 'nix.tex'):
os.remove(self._tmpdir + os.sep + 'nix.tex')
G.write_ranking_table('ta', self._tmpdir + os.sep + 'nix1', fmt='latex')
self.assertTrue(os.path.exists(self._tmpdir + os.sep + 'nix1.tex'))
if os.path.exists(self._tmpdir + os.sep + 'nix1.tex'):
os.remove(self._tmpdir + os.sep + 'nix1.tex')
def test_old_map_plot(self):
xx_map_plot(self.D)
#~ xx_map_plot(self.D, use_basemap=True)
def test_HstackTimeSeries(self):
HT = HstackTimeseries()
for i in xrange(15):
x = np.random.random(100)*2.-1.
HT.add_data(x, 'model' + str(i).zfill(3) )
HT.plot(cmap='RdBu_r', interpolation='nearest', vmin=-1., vmax=1., nclasses=15, title='Testtitle')
def test_HstackTimeSeries_invalidData(self):
HT = HstackTimeseries()
x = np.random.random((50, 60))*2.-1.
with self.assertRaises(ValueError):
HT.add_data(x, 'test' )
def test_HstackTimeSeries_inconsistent_data_geometries(self):
HT = HstackTimeseries()
x = np.random.random(100)*2.-1.
y = np.random.random(1000)*2.-1.
HT.add_data(x, 'A')
with self.assertRaises(ValueError):
HT.add_data(y, 'B')
def test_HstackTimeSeries_duplicate_keys(self):
HT = HstackTimeseries()
x = np.random.random(100)*2.-1.
HT.add_data(x, 'A')
with self.assertRaises(ValueError):
HT.add_data(x, 'A')
def test_violin_plot(self):
Violin_example()
def test_globalmeanplot(self):
G = GlobalMeanPlot()
with self.assertRaises(ValueError):
G.plot(self.D, stat_type='no_stat_type')
G.plot(self.D, show_std=True)
G.plot_mean_result()
def test_HstackTimeSeries_monthly_ticks(self):
HT = HstackTimeseries()
f = plt.figure()
ax = f.add_subplot(111)
ax.plot(np.arange(12))
HT._set_monthly_xtick_labels(ax)
def test_HstackTimeSeries_noplotdone(self):
HT = HstackTimeseries()
with self.assertRaises(ValueError):
HT._add_colorbar()
def test_GlobalMean(self):
GM1 = GlobalMeanPlot(climatology=False)
self.assertEqual(GM1.nplots, 1)
f = plt.figure()
ax1 = f.add_subplot(111)
axA = f.add_subplot(211)
axB = f.add_subplot(212)
with self.assertRaises(ValueError):
GM2 = GlobalMeanPlot(climatology=True, ax=ax1)
GM2 = GlobalMeanPlot(climatology=True, ax=axA, ax1=axB)
self.assertEqual(GM2.nplots, 2)
def test_Hovmoeller(self):
H = HovmoellerPlot(self.D)
with self.assertRaises(ValueError):
H.plot()
H.plot(climits=[0., 1.])
class TestPycmbsBenchmarkingModels(unittest.TestCase):
def setUp(self):
self.D = Data(None, None)
self.D._init_sample_object(nt=1000, ny=1, nx=1)
# generate dummy Model object
data_dir = '.' + os.sep + 'test' + os.sep
varmethods = {'albedo': 'get_albedo()', 'sis': 'get_sis()'}
self.model = models.Model(data_dir,
varmethods,
name='testmodel',
intervals='monthly')
sis = self.D.copy()
sis.mulc(5., copy=False)
sis.label = 'sisdummy'
alb = self.D.copy()
alb.label = 'albedodummy'
# add some dummy data variable
self.model.variables = {'albedo': alb, 'sis': sis}
def test_save_prefix_missing(self):
m = self.model
odir = tempfile.mkdtemp() + os.sep
with self.assertRaises(ValueError):
m.save(odir)
def test_save_create_odir(self):
m = self.model
odir = tempfile.mkdtemp() + os.sep
if os.path.exists(odir):
os.system('rm -rf ' + odir)
m.save(odir, prefix='test')
self.assertTrue(os.path.exists(odir))
os.system('rm -rf ' + odir)
def test_save(self):
m = self.model
odir = tempfile.mkdtemp() + os.sep
sisfile = odir + 'testoutput_SIS.nc'
albfile = odir + 'testoutput_ALBEDO.nc'
if os.path.exists(sisfile):
os.remove(sisfile)
if os.path.exists(albfile):
os.remove(albfile)
m.save(odir, prefix='testoutput')
self.assertTrue(os.path.exists(sisfile))
self.assertTrue(os.path.exists(albfile))
if os.path.exists(sisfile):
os.remove(sisfile)
if os.path.exists(albfile):
os.remove(albfile)
os.system('rm -rf ' + odir)
def test_cmip5_init_singlemember(self):
data_dir = tempfile.mkdtemp()
# invalid model identifier
with self.assertRaises(ValueError):
M = models.CMIP5RAW_SINGLE(data_dir,
'MPI-M:MPI-ESM-LR1',
'amip', {},
intervals='monthly')
with self.assertRaises(ValueError):
M = models.CMIP5RAW_SINGLE(data_dir,
'MPI-M:MPI-ESM-LR#1#2',
'amip', {},
intervals='monthly')
M1 = models.CMIP5RAW_SINGLE(data_dir,
'MPI-M:MPI-ESM-LR#1',
'amip', {},
intervals='monthly')
M2 = models.CMIP5RAW_SINGLE(data_dir,
'MPI-M:MPI-ESM-LR#728',
'amip', {},
intervals='monthly')
self.assertEqual(M1.ens_member, 1)
self.assertEqual(M2.ens_member, 728)
def test_cmip5_singlemember_filename(self):
data_dir = tempfile.mkdtemp()
# generate testfile
testfile = data_dir + os.sep + 'MPI-M' + os.sep + 'MPI-ESM-LR' + os.sep + 'amip' + os.sep + 'mon' + os.sep + 'atmos' + os.sep + 'Amon' + os.sep + 'r1i1p1' + os.sep + 'ta' + os.sep + 'ta_Amon_MPI-ESM-LR_amip_r1i1p1_197901-200812.nc'
os.makedirs(os.path.dirname(testfile))
os.system('touch ' + testfile)
self.assertTrue(os.path.exists(testfile))
M = models.CMIP5RAW_SINGLE(data_dir,
'MPI-M:MPI-ESM-LR#1',
'amip', {},
intervals='monthly')
kwargs = {
'CMIP5RAWSINGLE': {
'mip': 'Amon',
'realm': 'atmos',
'temporal_resolution': 'mon'
}
}
f = M.get_raw_filename('ta', **kwargs)
self.assertTrue(os.path.exists(f))
self.assertEqual(f, testfile)
For COPYING and LICENSE details, please refer to the LICENSE file """ from pycmbs.data import Data import numpy as np fname = '../pycmbs/examples/example_data/air.mon.mean.nc' d=Data(fname, 'air', read=True) c=d.get_climatology(return_object=True) print 'c raw: ', c.fldmean() print c.date print '' # create some invalid data d1=d.copy() t = d1.time*1. d1.time[20:] = t[0:-20] d1.time[0:20] = t[-20:] tmp = d1.data*1. d1.data[20:,:,:] = tmp[0:-20,:,:] d1.data[0:20,:,:] = tmp[-20:,:,:] c1=d1.get_climatology(return_object=True, ensure_start_first=True) print '' print c1.date print c1.fldmean()
class TestData(TestCase):
def setUp(self):
# init Data object for testing
n = 100 # slows down significantly! constraint is percentile test
x = sc.randn(n) * 100. # generate dummy data
self.D = Data(None, None)
d = np.ones((n, 1, 1))
self.D.data = d
self.D.data[:, 0, 0] = x
self.D.data = np.ma.array(self.D.data, mask=self.D.data != self.D.data)
self.D.verbose = True
self.D.unit = 'myunit'
self.D.label = 'testlabel'
self.D.filename = 'testinputfilename.nc'
self.D.varname = 'testvarname'
self.D.long_name = 'This is the longname'
self.D.time = np.arange(n) + pl.datestr2num('2001-01-01') - 1
self.D.time_str = "days since 0001-01-01 00:00:00"
self.D.calendar = 'gregorian'
self.D.cell_area = np.ones_like(self.D.data[0, :, :])
@unittest.skip('wait for bug free scipy')
def test_pattern_correlation(self):
"""
test pattern correlation function
"""
x = self.D.copy()
# correlation with random values
y = self.D.copy()
tmp = np.random.random(y.shape)
y.data = np.ma.array(tmp, mask=tmp != tmp)
P2 = PatternCorrelation(x, y)
P2._correlate()
self.assertEqual(x.nt, len(P2.r_value))
self.assertEqual(x.nt, len(P2.t))
for i in xrange(x.nt):
slope, intercept, r_value, p_value, std_err = stats.mstats.linregress(
x.data[i, :, :].flatten(), y.data[i, :, :].flatten())
self.assertEqual(P2.r_value[i], r_value)
self.assertEqual(P2.p_value[i], p_value)
self.assertEqual(P2.slope[i], slope)
self.assertEqual(P2.intercept[i], intercept)
self.assertEqual(P2.std_err[i], std_err)
def test_gleckler_index(self):
"""
test Reichler index/Gleckler plot
"""
# generate sample data
# sample data
tmp = np.zeros((5, 3, 1))
tmp[:, 0, 0] = np.ones(5) * 1.
tmp[:, 1, 0] = np.ones(5) * 2.
tmp[:, 2, 0] = np.ones(5) * 5.
# The data is like ...
#| 1 | 2 | 5 |
#| 1 | 2 | 5 |
#| 1 | 2 | 5 |
#| 1 | 2 | 5 |
#| 1 | 2 | 5 |
x = self.D.copy()
x._temporal_subsetting(0, 4)
x.data = np.ma.array(tmp, mask=tmp != tmp)
x.std = np.ones(x.data.shape)
x.time[0] = pl.datestr2num('2000-02-15')
x.time[1] = pl.datestr2num('2000-03-15')
x.time[2] = pl.datestr2num('2000-04-15')
x.time[3] = pl.datestr2num('2000-05-15')
x.time[4] = pl.datestr2num('2000-06-15')
y = self.D.copy()
y._temporal_subsetting(0, 4)
tmp = np.ones(x.data.shape) # sample data 2
y.data = np.ma.array(tmp, mask=tmp != tmp)
y.time[0] = pl.datestr2num('2000-02-15')
y.time[1] = pl.datestr2num('2000-03-15')
y.time[2] = pl.datestr2num('2000-04-15')
y.time[3] = pl.datestr2num('2000-05-15')
y.time[4] = pl.datestr2num('2000-06-15')
# Case 1: same area weights
# cell area
tmp = np.ones((3, 1))
x.cell_area = tmp * 1.
#| 1-1 | 2-1 | 5-1 |
#| 1-1 | 2-1 | 5-1 |
#| 1-1 | 2-1 | 5-1 |
#| 1-1 | 2-1 | 5-1 |
#| 1-1 | 2-1 | 5-1 |
#===================
#| 0 | 5 | 5*4**2=5*16. = 80 |
#==> E2 = sqrt(85./(15.))
D = GlecklerPlot()
r = D.calc_index(x, y, 'a', 'b', time_weighting=False)
wt = np.ones(5) / 5.
ref = np.sqrt(((85. / 15.) * wt).sum())
t = np.abs(1. - r / ref)
self.assertLess(t, 0.000001) # relative error
D = GlecklerPlot()
r = D.calc_index(x, y, 'a', 'b')
wt = np.asarray([29., 31., 30., 31., 30.])
wt = wt / wt.sum()
ref = np.sqrt(((85. / 15.) * wt).sum())
t = np.abs(1. - r / ref)
self.assertLess(t, 0.000001) # relative error
# Case 2: Different area weights
# cell area
tmp = np.ones((3, 1))
tmp[1, 0] = 2.
x.cell_area = tmp * 1.
#| 1-1=0 | 2-1=1 | 5-1=16 |
#| 1-1=0 | 2-1=1 | 5-1=16 |
#| 1-1=0 | 2-1=1 | 5-1=16 |
#| 1-1=0 | 2-1=1 | 5-1=16 |
#| 1-1=0 | 2-1=1 | 5-1=16 |
#--------------------------
# w = 0.25 w = 0.5 w=0.25|
#--------------------------
# 0.25*0 + 0.5 * 1 + 0.25 * 16 = 0 + 0.5 + 4 = 4.5
# the mean of that is 4.5 for each timestep
# mean because the overall weights are calculated as such that
# they give a total weight if 1
# diagnostic
D = GlecklerPlot()
r = D.calc_index(x, y, 'a', 'b', time_weighting=False)
wt = np.ones(5) / 5.
ref = np.sqrt((4.5 * wt).sum())
t = np.abs(1. - r / ref)
self.assertLess(t, 0.000001) # relative error
wt = np.asarray([29., 31., 30., 31., 30.])
wt = wt / wt.sum()
ref = np.sqrt((4.5 * wt).sum())
t = np.abs(1. - r / ref)
self.assertLess(t, 0.000001) # relative error
# Case 3: use different std
x.std = np.ones(x.data.shape)
x.std[:, 2, 0] = 0.5
#| 1-1=0 | 2-1=1 | 5-1=16 / 0.5 |
#| 1-1=0 | 2-1=1 | 5-1=16 / 0.5 |
#| 1-1=0 | 2-1=1 | 5-1=16 / 0.5 |
#| 1-1=0 | 2-1=1 | 5-1=16 / 0.5 |
#| 1-1=0 | 2-1=1 | 5-1=16 / 0.5 |
#--------------------------------
# w = 0.25 w = 0.5 w=0.25|
# 0 + 0.5 + 0.25*32 = 0.5 + 8 = 8.5
D = GlecklerPlot()
r = D.calc_index(x, y, 'a', 'b', time_weighting=False)
wt = np.ones(5) / 5.
ref = np.sqrt((8.5 * wt).sum())
t = np.abs(1. - r / ref)
self.assertLess(t, 0.000001) # relative error
wt = np.asarray([29., 31., 30., 31., 30.])
wt = wt / wt.sum()
ref = np.sqrt((8.5 * wt).sum())
t = np.abs(1. - r / ref)
self.assertLess(t, 0.000001) # relative error
def test_RegionalAnalysis_xNone(self):
region = RegionIndex(55, 1, 1, 1, 1, label='test')
R = RegionalAnalysis(None, self.D, region)
self.assertEqual(R.x, None)
def test_RegionalAnalysis_InvalidX(self):
region = RegionIndex(77, 1, 1, 1, 1, label='test')
with self.assertRaises(ValueError):
R = RegionalAnalysis([123.], self.D, region)
def test_RegionalAnalysis_InvalidY(self):
region = RegionIndex(88, 1, 1, 1, 1, label='test')
with self.assertRaises(ValueError):
R = RegionalAnalysis(self.D, [123.], region)
def test_RegionalAnalysis_yNone(self):
region = RegionIndex(55, 1, 1, 1, 1, label='test')
R = RegionalAnalysis(self.D, None, region)
self.assertEqual(R.y, None)
def test_RegionalAnalysis_InvalidRegion(self):
region = 1.
with self.assertRaises(ValueError):
R = RegionalAnalysis(self.D, self.D, region)
def test_RegionalAnalysis_InvalidGeometry(self):
region = RegionIndex(99, 1, 1, 1, 1, label='test')
x = self.D.copy()
y = self.D.copy()
y.data = np.random.random((2, 3, 4, 5))
with self.assertRaises(ValueError):
R = RegionalAnalysis(x, y, region)
@unittest.skip('wait for solving logplot proplem in map_plot')
def test_EOF(self):
x = np.random.random((self.D.nt, 20, 30))
self.D.data = np.ma.array(x, mask=x != x)
self.D.cell_area = np.ones_like(self.D.data[0, :, :])
E = EOF(self.D)
r = E.reconstruct_data()
c = E.get_correlation_matrix()
E.get_eof_data_correlation()
#~ E.plot_channnel_correlations(100000) #slow!!
E.plot_eof_coefficients(None, all=True)
E._calc_anomalies()
E.plot_EOF(None, all=True)
#~ def test_koeppen(self):
#~ T = self.D.copy()
#~ T.data = np.random.random((10,20,30))
#~ T.unit = 'K'
#~ P = self.D.copy()
#~ P.data = np.random.random((10,20,30))
#~ P.unit = 'kg/m^2s'
#~ lsm = self.D.copy()
#~ lsm.unit = 'fractional'
#~ lsm.data = np.ones((20,30))
#~
#~ k = Koeppen(temp=T, precip=P, lsm=lsm)
def test_koeppen_InvalidInput(self):
T = self.D.copy()
P = self.D.copy()
lsm = self.D.copy()
with self.assertRaises(ValueError):
k = Koeppen(temp=None, precip=P, lsm=lsm)
with self.assertRaises(ValueError):
k = Koeppen(temp=T, precip=None, lsm=lsm)
with self.assertRaises(ValueError):
k = Koeppen(temp=T, precip=P, lsm=None)
