def test_io():
from pygeode.formats import netcdf as nc
from pygeode.timeaxis import StandardTime
from pygeode.axis import Pres
from pygeode.dataset import Dataset
import numpy as np
tm = StandardTime(values=np.arange(365), units='days', startdate={'year':2001})
p = Pres(np.arange(100.))
v = (tm * p).rename('v')
# Save the dataset, then reload it immediately
before = Dataset([v])
nc.save('issue004_test.nc', before)
after = nc.open('issue004_test.nc')
# Compare all vars/axes/attributes
for var in before:
assert var.name in after, "Can't find var '%s'"%var.name
var2 = getattr(after,var.name)
assert var2.atts == var.atts, "mismatched metadata. Input: %s, Output %s"%(var.atts,var2.atts)
for axis in before.axes:
axis2 = [a for a in after.axes if a.name == axis.name]
assert len(axis2) == 1, "can't find axis '%s'"%axis.name
axis2 = axis2[0]
# assert axis2.atts == axis.atts, "mismatched metadata. Input: %s, Output %s"%(axis.atts,axis2.atts)
for attname in list(axis.atts.keys()):
assert attname in axis2.atts, "attribute '%s' not found"%attname
assert axis.atts[attname] == axis2.atts[attname], "attribute '%s' changed from '%s' to '%s'"%(attname, axis.atts[attname], axis2.atts[attname])
assert type(axis2) == type(axis), "mismatched axis types. Input: %s, Output %s"%(type(axis), type(axis2))
def test_io():
from pygeode.formats import netcdf as nc
from pygeode.timeaxis import StandardTime
from pygeode.axis import Pres
from pygeode.dataset import Dataset
import numpy as np
tm = StandardTime(values=np.arange(365), units='days', startdate={'year':2001})
p = Pres(np.arange(100.))
v = (tm * p).rename('v')
# Save the dataset, then reload it immediately
before = Dataset([v])
nc.save('issue004_test.nc', before)
after = nc.open('issue004_test.nc')
# Compare all vars/axes/attributes
for var in before:
assert var.name in after, "Can't find var '%s'"%var.name
var2 = getattr(after,var.name)
assert var2.atts == var.atts, "mismatched metadata. Input: %s, Output %s"%(var.atts,var2.atts)
for axis in before.axes:
axis2 = [a for a in after.axes if a.name == axis.name]
assert len(axis2) == 1, "can't find axis '%s'"%axis.name
axis2 = axis2[0]
# assert axis2.atts == axis.atts, "mismatched metadata. Input: %s, Output %s"%(axis.atts,axis2.atts)
for attname in axis.atts.keys():
assert attname in axis2.atts, "attribute '%s' not found"%attname
assert axis.atts[attname] == axis2.atts[attname], "attribute '%s' changed from '%s' to '%s'"%(attname, axis.atts[attname], axis2.atts[attname])
assert type(axis2) == type(axis), "mismatched axis types. Input: %s, Output %s"%(type(axis), type(axis2))
def test_opener():
from pygeode.formats import netcdf
from pygeode.tutorial import t1
from pygeode.formats.multifile import openall
netcdf.save("issue090.data", t1)
my_opener = lambda filename: netcdf.open(filename)
f = openall("issue090.d???", opener=my_opener)
def test_opener():
from pygeode.formats import netcdf
from pygeode.tutorial import t1
from pygeode.formats.multifile import openall
netcdf.save("issue090.data", t1)
my_opener = lambda filename: netcdf.open(filename)
f = openall("issue090.d???", opener=my_opener)
def test_issue005():
from pygeode.timeaxis import ModelTime365
from pygeode.axis import TAxis
import numpy as np
from pygeode.var import Var
from pygeode.formats import netcdf as nc
from pygeode import timeutils
# Make a time axis starting at year 0
startdate = dict(year=0,month=1,day=1)
taxis = ModelTime365(values=10200, startdate=startdate, units='days')
# Make some dummy variable
np.random.seed(len(taxis))
values = np.random.randn(len(taxis))
var = Var(axes=[taxis], values=values, name='x')
# Save it
nc.save("issue005_test.nc", var)
# Load it
f = nc.open("issue005_test.nc")
# Make sure we have a regular time axis
# (no climatologies!)
assert f.time.__class__ == ModelTime365
assert hasattr(f.time,'year')
# Okay, now reload it, but override the axis coming in
f = nc.open("issue005_test.nc", dimtypes=dict(time=TAxis(taxis.values)))
# Make sure we dimtypes is still working properly
assert f.x.axes[0].__class__ == TAxis
# For good measure, test that climatologies are still produced
taxis = timeutils.modify(taxis,exclude='year',uniquify=True)
values = np.random.randn(len(taxis))
var = Var(axes=[taxis], values=values, name='c')
nc.save("issue005_test.nc", var)
f = nc.open("issue005_test.nc")
assert not hasattr(f.time,'year')
def testIntersect(self):
from pygeode.formats.netcdf import open
from myDatasets.utils import intersectDatasets
# load new test data
file = "TPfittest.nc"
newData = open(self.rootFolder + file)
# merge datasets with different slicing
mergedData = intersectDatasets(newData, self.data)
print(mergedData)
def testIntersect(self): from pygeode.formats.netcdf import open from myDatasets.utils import intersectDatasets # load new test data file = "TPfittest.nc" newData = open(self.rootFolder+file) # merge datasets with different slicing mergedData = intersectDatasets(newData,self.data) print(mergedData)
def test_issue005():
from pygeode.timeaxis import ModelTime365
from pygeode.axis import TAxis
import numpy as np
from pygeode.var import Var
from pygeode.formats import netcdf as nc
from pygeode import timeutils
# Make a time axis starting at year 0
startdate = dict(year=0, month=1, day=1)
taxis = ModelTime365(values=10200, startdate=startdate, units='days')
# Make some dummy variable
np.random.seed(len(taxis))
values = np.random.randn(len(taxis))
var = Var(axes=[taxis], values=values, name='x')
# Save it
nc.save("issue005_test.nc", var)
# Load it
f = nc.open("issue005_test.nc")
# Make sure we have a regular time axis
# (no climatologies!)
assert f.time.__class__ == ModelTime365
assert hasattr(f.time, 'year')
# Okay, now reload it, but override the axis coming in
f = nc.open("issue005_test.nc", dimtypes=dict(time=TAxis(taxis.values)))
# Make sure we dimtypes is still working properly
assert f.x.axes[0].__class__ == TAxis
# For good measure, test that climatologies are still produced
taxis = timeutils.modify(taxis, exclude='year', uniquify=True)
values = np.random.randn(len(taxis))
var = Var(axes=[taxis], values=values, name='c')
nc.save("issue005_test.nc", var)
f = nc.open("issue005_test.nc")
assert not hasattr(f.time, 'year')
def test_encode_decode():
from pygeode.formats import netcdf
import numpy as np
x = make_var()
netcdf.save("issue068_test.nc", x)
y = netcdf.open("issue068_test.nc").dummy
type1 = type(x.station)
type2 = type(y.station)
assert type1 is type2, (type1, type2)
assert x.station == y.station
assert np.all(x.get() == y.get())
def test_encode_decode():
from pygeode.formats import netcdf
import numpy as np
x = make_var()
netcdf.save("issue068_test.nc", x)
y = netcdf.open("issue068_test.nc").dummy
type1 = type(x.station)
type2 = type(y.station)
assert type1 is type2, (type1, type2)
assert x.station == y.station
assert np.all(x.get() == y.get())
def test_issue025():
lat = Lat([80,70,60])
var = Var(axes=[lat], values=[1,2,3], name='2B')
# Save the variable
nc.save ("issue025_test.nc", var)
# This may crash in some versions of the netcdf library.
# Even if it doesn't crash, it's a good idea to enforce the legal
# netcdf names
f = nc.open("issue025_test.nc")
assert len(f.vars) == 1
# Must not start with a digit (should have been filtered)
assert not f.vars[0].name[0].isdigit()
def test_issue015():
from pygeode.formats import netcdf as nc
from pygeode.axis import Lat, Lon, Pres
# Create a simple variable
lat = Lat([75,85,95])
lon = Lon([100, 110, 120, 130])
pres = Pres([1000,900,800,700])
x = (lat-80)**2 + lon - pres/2.
x.name = "stuff"
# Save as a netcdf file
nc.save("issue015_test.nc", x)
# Reload
f = nc.open("issue015_test.nc")
y = f.stuff.load()
# Issue 22 - plot attributes get overridden by NamedAxis defaults
# https://github.com/pygeode/pygeode/issues/22
# Make a sample file with a non-annotated pressure axis
from pygeode.axis import NamedAxis, Pres
from pygeode.formats import netcdf as nc
lat = NamedAxis(values=[-80,-70,-60,-50], name='lat')
p1 = NamedAxis(values=[1000.,900.,800.], name='p1')
x = lat * p1
x.name = 'x'
nc.save("issue022_test.nc", x)
# Load it back in
d = nc.open("issue022_test.nc", dimtypes={'p1':Pres})
# plotatts should be from the new (dimtypes) axis, not copied from the old one?
# (also implicitly asserts that other metadata (such as the name) are still
# copied from the old axis)
assert d.p1.plotatts == Pres.plotatts
# Issue 22 - plot attributes get overridden by NamedAxis defaults
# https://github.com/pygeode/pygeode/issues/22
# Make a sample file with a non-annotated pressure axis
from pygeode.axis import NamedAxis, Pres
from pygeode.formats import netcdf as nc
lat = NamedAxis(values=[-80, -70, -60, -50], name='lat')
p1 = NamedAxis(values=[1000., 900., 800.], name='p1')
x = lat * p1
x.name = 'x'
nc.save("issue022_test.nc", x)
# Load it back in
d = nc.open("issue022_test.nc", dimtypes={'p1': Pres})
# plotatts should be from the new (dimtypes) axis, not copied from the old one?
# (also implicitly asserts that other metadata (such as the name) are still
# copied from the old axis)
assert d.p1.plotatts == Pres.plotatts
def get_merra(time, print_time=False, action='middle'):
"""Reads 3h average MERRA data"""
Source = time.source
print "\nGathering MERRA data..."
file_times = [
"01:30", "04:30", "07:30", "10:30", "13:30", "16:30", "19:30", "22:30"
]
filename = time.strftime(_merra_fmt)
t_lon = Source.lon
t_lat = Source.lat
if action == 'middle':
time_indices = [int((time.decimaltime) // _merra_step)]
hours = [time.decimaltime]
elif action == 'beginning':
time_indices = [int((time.decimaltime - 1.5) // _merra_step)]
hours = [time.decimaltime]
elif action == 'interpolate':
tfloor = int((time.decimaltime - 1.5) // _merra_step)
time_indices = [tfloor, tfloor + 1]
hours = [(_merra_step * h + 1.5) for h in time_indices]
else:
raise ValueError('Invalid option "%s" for "action"' % action)
if print_time:
ftimes = [file_times[i] for i in time_indices]
print "Time {0}; using file for time {1}".format(
time.datetimeobj, ', '.join(ftimes))
if not os.path.exists(filename):
raise IOError("File {0} does not exist".format(filename))
u_interp = []
v_interp = []
for time_index in time_indices:
try:
merra = netcdf.open(filename)
except IOError:
raise
except:
print "Unexpected Error Encountered:"
raise
U = merra.U
V = merra.V
lat0 = U.lat[0]
lon0 = U.lon[0]
lat1 = U.lat[1]
lon1 = U.lon[1]
dlat = lat1 - lat0
dlon = lon1 - lon0
lat_row = int((t_lat - lat0) // dlat)
lon_col = int((t_lon - lon0) // dlon)
lat_used = U.lat[lat_row]
lon_used = U.lon[lon_col]
lat_error = lat_used - t_lat
lon_error = lon_used - t_lon
if lat_error > dlat or lon_error > dlon:
raise ValueError(
"Lat/lon disagree by more than the resolution: Errors are ({0}, {1})"
.format(lat_error, lon_error))
U = U[time_index, :, lat_row, lon_col]
V = V[time_index, :, lat_row, lon_col]
Heights = merra.H[time_index, :, lat_row, lon_col]
stack = Source.height
H_list = list(Heights)
i = 0
while i < len(H_list) and H_list[i] > stack:
i += 1
if i == len(H_list):
i = -1
u = U[i]
v = V[i]
# print("Using height {0}".format(H_list[i]))
else:
h1 = H_list[i]
h2 = H_list[i - 1]
u1 = U[i]
u2 = U[i - 1]
v1 = V[i]
v2 = V[i - 1]
# print("Interpolating between heights {0} and {1} for a stack height of {2}".format(h1,h2,stack))
u = numpy.interp(stack, [h1, h2], [u1, u2])
v = numpy.interp(stack, [h1, h2], [v1, v2])
u_interp.append(u)
v_interp.append(v)
if len(u_interp) == 1:
u_final = u_interp[0]
v_final = v_interp[0]
else:
u_final = numpy.interp(time.decimaltime, hours, u_interp)
v_final = numpy.interp(time.decimaltime, hours, v_interp)
return PST.Wind((u_final, v_final), stack)