def test_overriding_constructor(gfdl_data_loader, ds):
new = GFDLDataLoader(gfdl_data_loader,
data_direc=os.path.join('.', 'a'))
assert new.data_direc == os.path.join('.', 'a')
assert new.data_dur == gfdl_data_loader.data_dur
assert new.data_start_date == gfdl_data_loader.data_start_date
assert new.data_end_date == gfdl_data_loader.data_end_date
assert new.preprocess_func == gfdl_data_loader.preprocess_func
assert new.upcast_float32 == gfdl_data_loader.upcast_float32
new = GFDLDataLoader(gfdl_data_loader, data_dur=8)
assert new.data_dur == 8
new = GFDLDataLoader(gfdl_data_loader,
data_start_date=datetime.datetime(2001, 1, 1))
assert new.data_start_date == datetime.datetime(2001, 1, 1)
new = GFDLDataLoader(gfdl_data_loader,
data_end_date=datetime.datetime(2003, 12, 31))
assert new.data_end_date == datetime.datetime(2003, 12, 31)
new = GFDLDataLoader(gfdl_data_loader, preprocess_func=lambda ds: ds)
xr.testing.assert_identical(new.preprocess_func(ds), ds)
new = GFDLDataLoader(gfdl_data_loader, upcast_float32=True)
assert new.upcast_float32
new = GFDLDataLoader(gfdl_data_loader, data_vars='all')
assert new.data_vars == 'all'
new = GFDLDataLoader(gfdl_data_loader, coords='all')
assert new.coords == 'all'
def test_init_default_dates(self):
gdl = GFDLDataLoader(data_start_date=datetime.datetime(1, 1, 1),
data_end_date=datetime.datetime(1, 12, 31))
run_ = Run(data_loader=gdl)
self.assertEqual(run_.default_start_date, datetime.datetime(1, 1, 1))
self.assertEqual(run_.default_end_date, datetime.datetime(1, 12, 31))
ddl = DictDataLoader({'monthly': '/a/'})
run_ = Run(data_loader=ddl)
self.assertEqual(run_.default_start_date, None)
self.assertEqual(run_.default_end_date, None)
def test_overriding_constructor(gfdl_data_loader, ds_with_time_bounds):
new = GFDLDataLoader(gfdl_data_loader, data_direc=os.path.join('.', 'a'))
assert new.data_direc == os.path.join('.', 'a')
assert new.data_dur == gfdl_data_loader.data_dur
assert new.data_start_date == gfdl_data_loader.data_start_date
assert new.data_end_date == gfdl_data_loader.data_end_date
assert new.preprocess_func == gfdl_data_loader.preprocess_func
assert new.upcast_float32 == gfdl_data_loader.upcast_float32
new = GFDLDataLoader(gfdl_data_loader, data_dur=8)
assert new.data_dur == 8
new = GFDLDataLoader(gfdl_data_loader,
data_start_date=datetime.datetime(2001, 1, 1))
assert new.data_start_date == datetime.datetime(2001, 1, 1)
new = GFDLDataLoader(gfdl_data_loader,
data_end_date=datetime.datetime(2003, 12, 31))
assert new.data_end_date == datetime.datetime(2003, 12, 31)
new = GFDLDataLoader(gfdl_data_loader, preprocess_func=lambda ds: ds)
xr.testing.assert_identical(new.preprocess_func(ds_with_time_bounds),
ds_with_time_bounds)
new = GFDLDataLoader(gfdl_data_loader, upcast_float32=True)
assert new.upcast_float32
new = GFDLDataLoader(gfdl_data_loader, data_vars='all')
assert new.data_vars == 'all'
new = GFDLDataLoader(gfdl_data_loader, coords='all')
assert new.coords == 'all'
def test_overriding_constructor(self):
new = GFDLDataLoader(self.DataLoader,
data_direc=os.path.join('.', 'a'))
self.assertEqual(new.data_direc, os.path.join('.', 'a'))
self.assertEqual(new.data_dur, self.DataLoader.data_dur)
self.assertEqual(new.data_start_date, self.DataLoader.data_start_date)
self.assertEqual(new.data_end_date, self.DataLoader.data_end_date)
self.assertEqual(new.preprocess_func, self.DataLoader.preprocess_func)
new = GFDLDataLoader(self.DataLoader, data_dur=8)
self.assertEqual(new.data_dur, 8)
new = GFDLDataLoader(self.DataLoader,
data_start_date=datetime(2001, 1, 1))
self.assertEqual(new.data_start_date, datetime(2001, 1, 1))
new = GFDLDataLoader(self.DataLoader,
data_end_date=datetime(2003, 12, 31))
self.assertEqual(new.data_end_date, datetime(2003, 12, 31))
new = GFDLDataLoader(self.DataLoader, preprocess_func=lambda ds: ds)
xr.testing.assert_identical(new.preprocess_func(self.ds), self.ds)
def setUp(self):
super(TestGFDLDataLoader, self).setUp()
self.DataLoader = GFDLDataLoader(data_direc=os.path.join('.', 'test'),
data_dur=6,
data_start_date=datetime(2000, 1, 1),
data_end_date=datetime(2012, 12, 31))
class TestGFDLDataLoader(TestDataLoader):
def setUp(self):
super(TestGFDLDataLoader, self).setUp()
self.DataLoader = GFDLDataLoader(data_direc=os.path.join('.', 'test'),
data_dur=6,
data_start_date=datetime(2000, 1, 1),
data_end_date=datetime(2012, 12, 31))
def test_overriding_constructor(self):
new = GFDLDataLoader(self.DataLoader,
data_direc=os.path.join('.', 'a'))
self.assertEqual(new.data_direc, os.path.join('.', 'a'))
self.assertEqual(new.data_dur, self.DataLoader.data_dur)
self.assertEqual(new.data_start_date, self.DataLoader.data_start_date)
self.assertEqual(new.data_end_date, self.DataLoader.data_end_date)
self.assertEqual(new.preprocess_func, self.DataLoader.preprocess_func)
new = GFDLDataLoader(self.DataLoader, data_dur=8)
self.assertEqual(new.data_dur, 8)
new = GFDLDataLoader(self.DataLoader,
data_start_date=datetime(2001, 1, 1))
self.assertEqual(new.data_start_date, datetime(2001, 1, 1))
new = GFDLDataLoader(self.DataLoader,
data_end_date=datetime(2003, 12, 31))
self.assertEqual(new.data_end_date, datetime(2003, 12, 31))
new = GFDLDataLoader(self.DataLoader, preprocess_func=lambda ds: ds)
xr.testing.assert_identical(new.preprocess_func(self.ds), self.ds)
def test_maybe_apply_time_offset_inst(self):
inst_ds = xr.decode_cf(self.inst_ds)
self.generate_file_set_args['dtype_in_time'] = 'inst'
self.generate_file_set_args['intvl_in'] = '3hr'
da = inst_ds[self.var_name]
result = self.DataLoader._maybe_apply_time_shift(
da.copy(), **self.generate_file_set_args)[TIME_STR]
expected = da[TIME_STR] + np.timedelta64(-3, 'h')
expected[TIME_STR] = expected
assert result.identical(expected)
self.generate_file_set_args['intvl_in'] = 'daily'
da = inst_ds[self.var_name]
result = self.DataLoader._maybe_apply_time_shift(
da.copy(), **self.generate_file_set_args)[TIME_STR]
expected = da[TIME_STR]
expected[TIME_STR] = expected
assert result.identical(expected)
def test_maybe_apply_time_offset_ts(self):
ds = xr.decode_cf(self.ds)
da = ds[self.var_name]
result = self.DataLoader._maybe_apply_time_shift(
da.copy(), **self.generate_file_set_args)[TIME_STR]
assert result.identical(da[TIME_STR])
def test_generate_file_set(self):
with self.assertRaises(IOError):
self.DataLoader._generate_file_set(**self.generate_file_set_args)
def test_input_data_paths_gfdl(self):
expected = [
os.path.join('.', 'test', 'atmos', 'ts', 'monthly', '6yr',
'atmos.200601-201112.temp.nc')
]
result = self.DataLoader._input_data_paths_gfdl(
'temp', datetime(2010, 1, 1), datetime(2010, 12, 31), 'atmos',
'monthly', 'pressure', 'ts', None)
self.assertEqual(result, expected)
expected = [
os.path.join('.', 'test', 'atmos_daily', 'ts', 'daily', '6yr',
'atmos_daily.20060101-20111231.temp.nc')
]
result = self.DataLoader._input_data_paths_gfdl(
'temp', datetime(2010, 1, 1), datetime(2010, 12, 31), 'atmos',
'daily', 'pressure', 'ts', None)
self.assertEqual(result, expected)
expected = [
os.path.join('.', 'test', 'atmos_level', 'ts', 'monthly', '6yr',
'atmos_level.200601-201112.temp.nc')
]
result = self.DataLoader._input_data_paths_gfdl(
'temp', datetime(2010, 1, 1), datetime(2010, 12, 31), 'atmos',
'monthly', ETA_STR, 'ts', None)
self.assertEqual(result, expected)
expected = [
os.path.join('.', 'test', 'atmos', 'ts', 'monthly', '6yr',
'atmos.200601-201112.ps.nc')
]
result = self.DataLoader._input_data_paths_gfdl(
'ps', datetime(2010, 1, 1), datetime(2010, 12, 31), 'atmos',
'monthly', ETA_STR, 'ts', None)
self.assertEqual(result, expected)
expected = [
os.path.join('.', 'test', 'atmos_inst', 'ts', 'monthly', '6yr',
'atmos_inst.200601-201112.temp.nc')
]
result = self.DataLoader._input_data_paths_gfdl(
'temp', datetime(2010, 1, 1), datetime(2010, 12, 31), 'atmos',
'monthly', 'pressure', 'inst', None)
self.assertEqual(result, expected)
expected = [
os.path.join('.', 'test', 'atmos', 'av', 'monthly_6yr',
'atmos.2006-2011.jja.nc')
]
result = self.DataLoader._input_data_paths_gfdl(
'temp', datetime(2010, 1, 1), datetime(2010, 12, 31), 'atmos',
'monthly', 'pressure', 'av', 'jja')
self.assertEqual(result, expected)
def test_data_name_gfdl_annual(self):
for data_type in ['ts', 'inst']:
expected = 'atmos.2010.temp.nc'
result = io.data_name_gfdl('temp', 'atmos', data_type, 'annual',
2010, None, 2000, 1)
self.assertEqual(result, expected)
expected = 'atmos.2006-2011.temp.nc'
result = io.data_name_gfdl('temp', 'atmos', data_type, 'annual',
2010, None, 2000, 6)
self.assertEqual(result, expected)
for intvl_type in ['annual', 'ann']:
expected = 'atmos.2010.ann.nc'
result = io.data_name_gfdl('temp', 'atmos', 'av', intvl_type, 2010,
None, 2000, 1)
self.assertEqual(result, expected)
expected = 'atmos.2006-2011.ann.nc'
result = io.data_name_gfdl('temp', 'atmos', 'av', intvl_type, 2010,
None, 2000, 6)
self.assertEqual(result, expected)
expected = 'atmos.2006-2011.01-12.nc'
result = io.data_name_gfdl('temp', 'atmos', 'av_ts', 'annual', 2010,
None, 2000, 6)
self.assertEqual(result, expected)
def test_data_name_gfdl_monthly(self):
for data_type in ['ts', 'inst']:
expected = 'atmos.200601-201112.temp.nc'
result = io.data_name_gfdl('temp', 'atmos', data_type, 'monthly',
2010, 'jja', 2000, 6)
self.assertEqual(result, expected)
for intvl_type in ['monthly', 'mon']:
expected = 'atmos.2010.jja.nc'
result = io.data_name_gfdl('temp', 'atmos', 'av', intvl_type, 2010,
'jja', 2000, 1)
self.assertEqual(result, expected)
expected = 'atmos.2006-2011.jja.nc'
result = io.data_name_gfdl('temp', 'atmos', 'av', intvl_type, 2010,
'jja', 2000, 6)
self.assertEqual(result, expected)
expected = 'atmos.2006-2011.01-12.nc'
result = io.data_name_gfdl('temp', 'atmos', 'av_ts', 'monthly', 2010,
'jja', 2000, 6)
self.assertEqual(result, expected)
def test_data_name_gfdl_daily(self):
for data_type in ['ts', 'inst']:
expected = 'atmos.20060101-20111231.temp.nc'
result = io.data_name_gfdl('temp', 'atmos', data_type, 'daily',
2010, None, 2000, 6)
self.assertEqual(result, expected)
with self.assertRaises(NameError):
io.data_name_gfdl('temp', 'atmos', 'av', 'daily', 2010, None, 2000,
6)
expected = 'atmos.2006-2011.01-12.nc'
result = io.data_name_gfdl('temp', 'atmos', 'av_ts', 'daily', 2010,
None, 2000, 6)
self.assertEqual(result, expected)
def test_data_name_gfdl_hr(self):
for data_type in ['ts', 'inst']:
expected = 'atmos.2006010100-2011123123.temp.nc'
result = io.data_name_gfdl('temp', 'atmos', data_type, '3hr', 2010,
None, 2000, 6)
self.assertEqual(result, expected)
with self.assertRaises(NameError):
io.data_name_gfdl('temp', 'atmos', 'av', '3hr', 2010, None, 2000,
6)
expected = 'atmos.2006-2011.01-12.nc'
result = io.data_name_gfdl('temp', 'atmos', 'av_ts', '3hr', 2010, None,
2000, 6)
self.assertEqual(result, expected)
def test_data_name_gfdl_seasonal(self):
for data_type in ['ts', 'inst']:
with self.assertRaises(NameError):
io.data_name_gfdl('temp', 'atmos', data_type, 'seasonal', 2010,
None, 2000, 6)
for intvl_type in ['seasonal', 'seas']:
expected = 'atmos.2010.JJA.nc'
result = io.data_name_gfdl('temp', 'atmos', 'av', intvl_type, 2010,
'jja', 2000, 1)
self.assertEqual(result, expected)
expected = 'atmos.2006-2011.JJA.nc'
result = io.data_name_gfdl('temp', 'atmos', 'av', intvl_type, 2010,
'jja', 2000, 6)
self.assertEqual(result, expected)
expected = 'atmos.2006-2011.01-12.nc'
result = io.data_name_gfdl('temp', 'atmos', 'av_ts', 'seasonal', 2010,
None, 2000, 6)
self.assertEqual(result, expected)
from aospy import Run
from aospy import Model
from aospy import Proj
from aospy import Var
from aospy import Region
rootdir = os.path.abspath(
os.path.join(os.getcwd(), '..', 'gridded_data',
'AM2.1_1979-2000-AllForc_h1', 'pp'))
output_dir = os.path.abspath(
os.path.join(os.getcwd(), '..', 'gridded_data',
'AM2.1_1979-2000-AllForc_h1_results'))
base = GFDLDataLoader(
data_direc=rootdir,
data_dur=5,
data_start_date=datetime(1980, 1, 1),
data_end_date=datetime(1999, 12, 31),
)
example_run = Run(
name='relative humidity examination',
description='An examination of relative humidity 1980 => 2000',
data_loader=base)
example_model = Model(name='example_model',
runs=[example_run],
grid_file_paths=(os.path.join(rootdir, 'atmos', 'ts',
'monthly',
'zg_A1.199501-199912.nc')))
example_proj = Proj('example_proj',