def test_reindex_time(self):
"""Tests whether the reindex_time function works as expected."""
[yr, month, day] = [1850, 1, 15]
expected_times = xr.DataArray(
np.array([
cftime.DatetimeProlepticGregorian(yr, month, day, 0, 0),
cftime.DatetimeProlepticGregorian(yr, month + 1, day, 0, 0)
]))
testdates1 = xr.DataArray(
np.array([
np.datetime64(datetime.datetime(yr, month, day, 0, 0)),
np.datetime64(datetime.datetime(yr, month + 1, day, 0, 0))
]))
testdates2 = xr.DataArray(
np.array([
cftime.DatetimeNoLeap(yr, month, day, 0, 0),
cftime.DatetimeNoLeap(yr, month + 1, day, 0, 0)
]))
testdates3 = xr.DataArray(
np.array([
cftime.Datetime360Day(yr, month, day, 0, 0),
cftime.Datetime360Day(yr, month + 1, day, 0, 0)
]))
dates_to_test = [testdates1, testdates2, testdates3]
# Run test
dates_converted = True
for date_to_test in dates_to_test:
newtimes = reindex_time(date_to_test)
if not np.all((newtimes == expected_times).values):
dates_converted = False
else:
continue
self.assertTrue(dates_converted)
def test_360_day_calendar_nd_raw_date(self):
# Test the case where the input is an nd-array.
calendar = '360_day'
unit = 'days since 2000-01-01'
val = np.array([[cftime.Datetime360Day(2014, 8, 12)],
[cftime.Datetime360Day(2014, 8, 13)]])
result = NetCDFTimeConverter().default_units(val, None)
self.assertEqual(result, (calendar, unit, cftime.Datetime360Day))
def crop_years(self, new_start_year, new_end_year):
if new_start_year is not None:
self.data = self.data.where(self.data.t >= cftime.Datetime360Day(new_start_year, 1, 1),
drop=True)
self.start_year = new_start_year
if new_end_year is not None:
self.data = self.data.where(self.data.t <= cftime.Datetime360Day(new_end_year, 12, 30),
drop=True)
self.end_year = new_end_year
print("____ Data cropped to the new start and end years.")
return self
def test_360_day_calendar_raw_dates(self):
datetimes = [
cftime.Datetime360Day(1986, month, 30) for month in range(1, 6)
]
line1, = plt.plot(datetimes)
result_ydata = line1.get_ydata()
np.testing.assert_array_equal(result_ydata, datetimes)
def test_sel_multi_level_file(tmpdir):
ds = xr.open_mfdataset(
"/badc/cmip5/data/cmip5/output1/MOHC/HadGEM2-ES/rcp45/mon/atmos/Amon/r1i1p1/latest/ta/ta_Amon_HadGEM2-ES_rcp45_r1i1p1_209912-212411.nc"
)
subset = ds.ta.sel(
time=slice("2100-12-16", "2120-12-16"),
plev=slice(85000, 3000),
lat=slice(50, 59),
lon=slice(2, 352),
)
assert (2 <= subset["lon"].data).all() & (subset["lon"].data <= 352).all()
assert (50 <= subset["lat"].data).all() & (subset["lat"].data <= 59).all()
assert (3000 <= subset["plev"].data).all() & (subset["plev"].data <= 85000).all()
assert (cftime.Datetime360Day(2100, 12, 16) <= subset["time"].data).all() & (
subset["time"].data <= cftime.Datetime360Day(2120, 12, 16)
).all()
subset.to_netcdf(path=tmpdir.mkdir("test_dir").join("example_dataset.nc"))
def _handler(request, response):
model = request.inputs['model'][0].data
variable = request.inputs['variable'][0].data
# Throw manually with temporary bbox solution
if request.inputs['min_lon'][0].data < 0:
raise InvalidParameterValue(
'Minimum longitude input cannot be below 0')
if request.inputs['max_lon'][0].data > 360:
raise InvalidParameterValue(
'Maximum longitude input cannot be above 360')
if request.inputs['min_lat'][0].data < -90:
raise InvalidParameterValue(
'Minimum latitude input cannot be below -90')
if request.inputs['max_lat'][0].data > 90:
raise InvalidParameterValue(
'Minimum latitude input cannot be above 90')
d1 = cftime.Datetime360Day(2010, 1, 1)
d2 = cftime.Datetime360Day(2020, 1, 1)
nc_files_path = f'xarray'
files_path = os.path.join(str(Path.home()), nc_files_path)
files = glob.glob(files_path + '/tas*.nc')
#response.update_status("Reading through files", 10)
files_to_open = get_years(files)
new_dataset = open_mfdatasets(files_to_open)
#response.update_status("Calculating temporal average", 50)
sliced_dataset = new_dataset.sel(
time=slice(d1, d2),
lon=slice(request.inputs['min_lon'][0].data,
request.inputs['max_lon'][0].data),
lat=slice(request.inputs['min_lat'][0].data,
request.inputs['max_lat'][0].data))
# calculate temporal average across the time axis only
mean_array = sliced_dataset.mean(dim='time')
print(mean_array)
response.update_status("Writing results to a new NetCDF4 file", 50)
# result_nc_file = mean_array.to_netcdf('results.nc')
response.outputs['output'].nc_file = mean_array.to_netcdf(
'resultsssss.nc')
response.update_status("Done.", 100)
return response
def load_base_data(info):
if (info.verbose): print("Loading: "+info.atmdir + "/" + info.atmfile)
ds = xr.open_mfdataset(info.atmdir + "/" + info.atmfile.format(info.base_vars[0]))
for v in info.base_vars[1:]:
this_data = xr.open_mfdataset(info.atmdir + "/" + info.atmfile.format(v))
data = interpolate_data(this_data[v], ds)
ds[v] = data
# for example, this could do:
# ds = xr.open_mfdataset(["hus_6hrLev_CanESM2_historical_r1i1p1_198001010000-198012311800.nc",
# "ta_6hrLev_CanESM2_historical_r1i1p1_198001010000-198012311800.nc",
# "ua_6hrLev_CanESM2_historical_r1i1p1_198001010000-198012311800.nc",
# "va_6hrLev_CanESM2_historical_r1i1p1_198001010000-198012311800.nc",
# "orog_fx_CanESM2_historical_r0i0p0.nc",
# "sftlf_fx_CanESM2_historical_r0i0p0.nc"])
# perhaps as :
# ds = xr.open_mfdataset("*CanESM2*.nc")
if (info.verbose): print("Loading base file metadata (this could take a while)")
# Time objects have to be defined with the calendar used in the input GCM
start_time = ds.time.values[0]
if start_time.calendar == "360_day":
start_date = cftime.Datetime360Day(*info.start_date.timetuple())
end_date = cftime.Datetime360Day(*info.end_date.timetuple())
elif start_time.calendar == "noleap":
start_date = cftime.DatetimeNoLeap(*info.start_date.timetuple())
end_date = cftime.DatetimeNoLeap(*info.end_date.timetuple())
else:
start_date = cftime.Datetime(*info.start_date.timetuple())
end_date = cftime.Datetime(*info.end_date.timetuple())
# subset the dataset in space and time
return ds.sel(time= slice(start_date, end_date),
lat = slice(info.lat[0], info.lat[1]),
lon = slice(info.lon[0], info.lon[1]))
def test_time_360(self):
cube = Cube(np.array([[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]]),
long_name="ts")
time_unit = cf_units.Unit("days since 2000-01-01 00:00",
calendar=cf_units.CALENDAR_360_DAY)
time_coord = DimCoord([100.1, 200.2],
long_name="time",
units=time_unit)
cube.add_dim_coord(time_coord, 0)
expected_index = [
cftime.Datetime360Day(2000, 4, 11, 2, 24),
cftime.Datetime360Day(2000, 7, 21, 4, 48)
]
expected_columns = [0, 1, 2, 3, 4]
data_frame = iris.pandas.as_data_frame(cube)
self.assertArrayEqual(data_frame, cube.data)
self.assertArrayEqual(data_frame.index, expected_index)
self.assertArrayEqual(data_frame.columns, expected_columns)
def test_time_360(self):
cube = Cube(np.array([0, 1, 2, 3, 4]), long_name="ts")
time_unit = cf_units.Unit("days since 2000-01-01 00:00",
calendar=cf_units.CALENDAR_360_DAY)
time_coord = DimCoord([0, 100.1, 200.2, 300.3, 400.4],
long_name="time",
units=time_unit)
cube.add_dim_coord(time_coord, 0)
expected_index = [
cftime.Datetime360Day(2000, 1, 1, 0, 0),
cftime.Datetime360Day(2000, 4, 11, 2, 24),
cftime.Datetime360Day(2000, 7, 21, 4, 48),
cftime.Datetime360Day(2000, 11, 1, 7, 12),
cftime.Datetime360Day(2001, 2, 11, 9, 36)
]
series = iris.pandas.as_series(cube)
self.assertArrayEqual(series, cube.data)
self.assertArrayEqual(series.index, expected_index)
def datetime(
year, month, day, hour=0, minute=0, second=0, microsecond=0,
tzinfo=None, calendar='proleptic_gregorian'):
"""
Retrieves a datetime-like object with the requested calendar. Calendar types
other than proleptic_gregorian require the netcdftime module to be
installed.
Parameters
----------
year : int,
month : int,
day : int,
hour : int, optional
minute : int, optional
second : int, optional
microsecond : int, optional
tzinfo : datetime.tzinfo, optional
A timezone informaton class, such as from pytz. Can only be used with
'proleptic_gregorian' calendar, as netcdftime does not support
timezones.
calendar : string, optional
Should be one of 'proleptic_gregorian', 'no_leap', '365_day',
'all_leap', '366_day', '360_day', 'julian', or 'gregorian'. Default
is 'proleptic_gregorian', which returns a normal Python datetime.
Other options require the netcdftime module to be installed.
Returns
-------
datetime : datetime-like
The requested datetime. May be a Python datetime, or one of the
datetime-like types in netcdftime.
"""
kwargs = {
'year': year, 'month': month, 'day': day, 'hour': hour,
'minute': minute, 'second': second, 'microsecond': microsecond
}
if calendar.lower() == 'proleptic_gregorian':
return real_datetime(tzinfo=tzinfo, **kwargs)
elif tzinfo is not None:
raise ValueError('netcdftime does not support timezone-aware datetimes')
elif ct is None:
raise DependencyError(
"Calendars other than 'proleptic_gregorian' require the netcdftime "
"package, which is not installed.")
elif calendar.lower() in ('all_leap', '366_day'):
return ct.DatetimeAllLeap(**kwargs)
elif calendar.lower() in ('no_leap', 'noleap', '365_day'):
return ct.DatetimeNoLeap(**kwargs)
elif calendar.lower() == '360_day':
return ct.Datetime360Day(**kwargs)
elif calendar.lower() == 'julian':
return ct.DatetimeJulian(**kwargs)
elif calendar.lower() == 'gregorian':
return ct.DatetimeGregorian(**kwargs)
def make_year_constraint_all_calendars(start, end):
"""Utility function to create a dict of time constraints on year-basis
This create a dict of the same time constraint, but for different calendars.
Since comparisons between different calendar types are not (always) possible,
a calendar for a time coordinate should be compared to the specific constraint
with the same calendar.
The calendar type (as a string) can be obtained through the coordinate's `units`
attribute: `cube.coord('time').units.calendar`; the resulting string is the key
for the dict, which then as a value yields the correct constraint
Arguments
---------
start, end: integer
Start and end year. Month and day are 1, 1 for the starting year, and
31, 12 or 30, 12 (for a 360-day calendar) for the end year
"""
dates = {
'default': (cftime.datetime(start, 1, 1), cftime.datetime(end, 12,
31)),
'360_day':
(cftime.Datetime360Day(start, 1,
1), cftime.Datetime360Day(end, 12, 30)),
'365_day': (cftime.DatetimeNoLeap(start, 1, 1),
cftime.DatetimeNoLeap(end, 12, 31)),
'proleptic_gregorian': (cftime.DatetimeProlepticGregorian(start, 1, 1),
cftime.DatetimeProlepticGregorian(end, 12,
31)),
'gregorian': (cftime.DatetimeGregorian(start, 1, 1),
cftime.DatetimeGregorian(end, 12, 31)),
'julian': (cftime.DatetimeJulian(start, 1, 1),
cftime.DatetimeJulian(end, 12, 31)),
}
constraints = {
key: make_date_constraint(*value)
for key, value in dates.items()
}
return constraints
def state_list_with_inconsistent_calendars(base_state, numpy):
state_list = []
state_times = [
cftime.DatetimeNoLeap(2000, 1, 1),
cftime.Datetime360Day(2000, 1, 2)
]
for i in range(2):
new_state = copy.deepcopy(base_state)
for name in set(new_state.keys()).difference(["time"]):
new_state[name].view[:] = numpy.random.randn(
*new_state[name].extent)
state_list.append(new_state)
new_state["time"] = state_times[i]
return state_list
def _handler(request, response):
model = request.inputs['model'][0].data
variable = request.inputs['variable'][0].data
# Throw manually with temporary bbox solution
if request.inputs['min_lon'][0].data < 0:
raise InvalidParameterValue('Minimum longitude input cannot be below 0')
if request.inputs['max_lon'][0].data > 360:
raise InvalidParameterValue('Maximum longitude input cannot be above 360')
if request.inputs['min_lat'][0].data < -90:
raise InvalidParameterValue('Minimum latitude input cannot be below -90')
if request.inputs['max_lat'][0].data > 90:
raise InvalidParameterValue('Minimum latitude input cannot be above 90')
d1 = cftime.Datetime360Day(2010, 1, 1)
d2 = cftime.Datetime360Day(2020, 1, 1)
glob_pattern = f'/badc/cmip5/data/cmip5/output1/MOHC/{model}/rcp45/mon/atmos/Amon/r1i1p1/latest/{variable}/*.nc'
files = glob.glob(glob_pattern)
files_to_open = get_years(files)
new_dataset = open_mfdatasets(files_to_open)
sliced_dataset = new_dataset.sel(time=slice(d1, d2), lon=slice(request.inputs['min_lon'][0].data,
request.inputs['max_lon'][0].data),
lat=slice(request.inputs['min_lat'][0].data,
request.inputs['max_lat'][0].data))
# calculate temporal average across the time axis only
mean_array = sliced_dataset.mean(dim='time')
print(mean_array)
response.update_status("Writing results to a new NetCDF4 file", 50)
output_path = '/tmp/output.nc'
response.outputs['output'].nc_file = mean_array.to_netcdf(output_path)
print(f'Wrote: {output_path}')
response.update_status("Done.", 100)
return response
def import_coordinates(self):
self.lon, self.lat = sort_coordinates(self.grid.plon.values,
self.grid.plat.values)
self.lon_p, self.lat_p = cycle_coordinates(self.lon, self.lat)
self.transform_matrix = get_transform_matrix(self.grid.plon.values)
self.z = np.sort(self.sample_data.depth.values)
self.z_p = self.z
self.t = [
cftime.Datetime360Day(year, month, 1)
for year in np.arange(int(self.start_year),
int(self.end_year) + 1)
for month in util.months_to_number(self.months)
]
super(OCNMDS, self).import_coordinates()
# (C) Copyright 1996-2019 ECMWF.
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
# In applying this licence, ECMWF does not waive the privileges and immunities
# granted to it by virtue of its status as an intergovernmental organisation nor
# does it submit to any jurisdiction.
import cftime
import xarray as xr
import numpy as np
from Magics import macro as magics
ref = "xarray3"
ds = xr.open_dataset('tos_O1_2001-2002.nc')
time = cftime.Datetime360Day(2001, 1, 16, 0, 0, 0, 0, 5, 16)
png = magics.output(output_name_first_page_number="off", output_name=ref)
data = magics.mxarray(xarray_dataset=ds,
xarray_variable_name="tos",
xarray_dimension_settings={"time": time})
contour = magics.mcont(contour_automatic_setting="ecmwf")
magics.plot(png, data, contour, magics.mcoast())
#Merged file
blob_extents = pd.read_csv(fname)
blob_extents['year'] = blob_extents['time'].str.slice(0, 4)
blob_extents['month'] = blob_extents['time'].str.slice(5, 7)
blob_extents['day'] = blob_extents['time'].str.slice(8, 10)
blob_extents['sec'] = blob_extents['time'].str.slice(11, )
tvar_check = var_blob['time'].values
df_indices = pd.DataFrame()
for it, r in blob_extents.iterrows():
ysnum = int(r['year'])
msnum = int(r['month'])
dsnum = int(r['day'])
ssnum = int(r['sec']) / 3600
if (input_calendar == "360_day"):
sdate = cftime.Datetime360Day(ysnum, msnum, dsnum, ssnum)
else:
sdate = cftime.DatetimeNoLeap(ysnum, msnum, dsnum, ssnum)
xmin = r['minlon'] - 0.001
xmax = r['maxlon'] + 0.001
ymin = r['minlat'] - 0.001
ymax = r['maxlat'] + 0.001
if (sdate in tvar_check):
#print(sdate)
#Need to deal with the case where there's the periodic boundary
if (xmin > xmax):
b1 = var_blob.sel(time=sdate,
lat=slice(ymin, ymax),
lon=slice(xmin, 360))
b2 = var_blob.sel(time=sdate,
def test_cf_subsecond(self):
input_time = cftime.Datetime360Day(2018, 11, 19, 0, 29, 59, 800000)
output_time = _round_time(input_time, 60)
time_diff = output_time - cftime.Datetime360Day(2018, 11, 19, 0, 30, 0)
self.assertEqual(time_diff.days, 0)
self.assertEqual(time_diff.seconds, 0)
def test_cftime_np_array_raw_date(self):
val = np.array([cftime.Datetime360Day(2012, 6, 4)], dtype=np.object)
result = NetCDFTimeConverter().convert(val, None, None)
self.assertEqual(result, np.array([4473.]))
def test_360_day_calendar_list_raw_date(self):
calendar = '360_day'
unit = 'days since 2000-01-01'
val = [cftime.Datetime360Day(2014, 8, 12)]
result = NetCDFTimeConverter().default_units(val, None)
self.assertEqual(result, (calendar, unit, cftime.Datetime360Day))
dat_bsub = dat_bsub.sort_values('time')
sline = dat_bsub.iloc[0]
eline = dat_bsub.iloc[len(dat_bsub) - 1]
calendar = sline['calendar']
stime = sline['time']
etime = eline['time']
ysnum = int(stime[:4])
msnum = int(stime[5:7])
dsnum = int(stime[8:10])
ssnum = int(stime[11:]) / 3600
yenum = int(etime[:4])
menum = int(etime[5:7])
denum = int(etime[8:10])
senum = int(etime[11:]) / 3600
if calendar == "360_day":
sdate = cftime.Datetime360Day(ysnum, msnum, dsnum, ssnum)
edate = cftime.Datetime360Day(yenum, menum, denum, senum)
else:
sdate = cftime.DatetimeNoLeap(ysnum, msnum, dsnum, ssnum)
edate = cftime.DatetimeNoLeap(yenum, menum, denum, senum)
td = edate - sdate
num_days = td.days + 1
num_hrs = td.seconds / (3600 * 24)
num_tot_days = num_days + num_hrs
avg_clat = (sline['centlat'] + eline['centlat']) / 2.
dict_bsub = {
"var":
sline['var'],
"bnum":
b,
"region":
