def __init__(self, lon, lat, time, time_origin, mesh):
self.lon = lon
self.lat = lat
self.time = np.zeros(1, dtype=np.float64) if time is None else time
if not self.lon.dtype == np.float32:
logger.warning_once("Casting lon data to np.float32")
self.lon = self.lon.astype(np.float32)
if not self.lat.dtype == np.float32:
logger.warning_once("Casting lat data to np.float32")
self.lat = self.lat.astype(np.float32)
if not self.time.dtype == np.float64:
assert isinstance(self.time[0], (np.integer, np.floating, float, int)), 'Time vector must be an array of int or floats'
logger.warning_once("Casting time data to np.float64")
self.time = self.time.astype(np.float64)
self.time_full = self.time # needed for deferred_loaded Fields
self.time_origin = TimeConverter() if time_origin is None else time_origin
assert isinstance(self.time_origin, TimeConverter), 'time_origin needs to be a TimeConverter object'
self.mesh = mesh
self.cstruct = None
self.cell_edge_sizes = {}
self.zonal_periodic = False
self.zonal_halo = 0
self.meridional_halo = 0
self.lat_flipped = False
self.defer_load = False
self.lonlat_minmax = np.array([np.nanmin(lon), np.nanmax(lon), np.nanmin(lat), np.nanmax(lat)], dtype=np.float32)
self.periods = 0
def test_fieldset_defer_loading_with_diff_time_origin(tmpdir, fail, filename='test_parcels_defer_loading'):
filepath = tmpdir.join(filename)
data0, dims0 = generate_fieldset(10, 10, 1, 10)
dims0['time'] = np.arange(0, 10, 1) * 3600
fieldset_out = FieldSet.from_data(data0, dims0)
fieldset_out.U.grid.time_origin = TimeConverter(np.datetime64('2018-04-20'))
fieldset_out.V.grid.time_origin = TimeConverter(np.datetime64('2018-04-20'))
data1, dims1 = generate_fieldset(10, 10, 1, 10)
if fail:
dims1['time'] = np.arange(0, 10, 1) * 3600
else:
dims1['time'] = np.arange(0, 10, 1) * 1800 + (24+25)*3600
if fail:
Wtime_origin = TimeConverter(np.datetime64('2018-04-22'))
else:
Wtime_origin = TimeConverter(np.datetime64('2018-04-18'))
gridW = RectilinearZGrid(dims1['lon'], dims1['lat'], dims1['depth'], dims1['time'], time_origin=Wtime_origin)
fieldW = Field('W', np.zeros(data1['U'].shape), grid=gridW)
fieldset_out.add_field(fieldW)
fieldset_out.write(filepath)
fieldset = FieldSet.from_parcels(filepath, extra_fields={'W': 'W'})
assert fieldset.U.creation_log == 'from_parcels'
pset = ParticleSet.from_list(fieldset, pclass=JITParticle, lon=[0.5], lat=[0.5], depth=[0.5],
time=[datetime.datetime(2018, 4, 20, 1)])
pset.execute(AdvectionRK4_3D, runtime=delta(hours=4), dt=delta(hours=1))
def from_data(cls, data, dimensions, transpose=False, mesh='spherical',
allow_time_extrapolation=None, time_periodic=False, **kwargs):
"""Initialise FieldSet object from raw data
:param data: Dictionary mapping field names to numpy arrays.
Note that at least a 'U' and 'V' numpy array need to be given, and that
the built-in Advection kernels assume that U and V are in m/s
1. If data shape is [xdim, ydim], [xdim, ydim, zdim], [xdim, ydim, tdim] or [xdim, ydim, zdim, tdim],
whichever is relevant for the dataset, use the flag transpose=True
2. If data shape is [ydim, xdim], [zdim, ydim, xdim], [tdim, ydim, xdim] or [tdim, zdim, ydim, xdim],
use the flag transpose=False (default value)
3. If data has any other shape, you first need to reorder it
:param dimensions: Dictionary mapping field dimensions (lon,
lat, depth, time) to numpy arrays.
Note that dimensions can also be a dictionary of dictionaries if
dimension names are different for each variable
(e.g. dimensions['U'], dimensions['V'], etc).
:param transpose: Boolean whether to transpose data on read-in
:param mesh: String indicating the type of mesh coordinates and
units used during velocity interpolation, see also https://nbviewer.jupyter.org/github/OceanParcels/parcels/blob/master/parcels/examples/tutorial_unitconverters.ipynb:
1. spherical (default): Lat and lon in degree, with a
correction for zonal velocity U near the poles.
2. flat: No conversion, lat/lon are assumed to be in m.
:param allow_time_extrapolation: boolean whether to allow for extrapolation
(i.e. beyond the last available time snapshot)
Default is False if dimensions includes time, else True
:param time_periodic: boolean whether to loop periodically over the time component of the FieldSet
This flag overrides the allow_time_interpolation and sets it to False
"""
fields = {}
for name, datafld in data.items():
# Use dimensions[name] if dimensions is a dict of dicts
dims = dimensions[name] if name in dimensions else dimensions
cls.checkvaliddimensionsdict(dims)
if allow_time_extrapolation is None:
allow_time_extrapolation = False if 'time' in dims else True
lon = dims['lon']
lat = dims['lat']
depth = np.zeros(1, dtype=np.float32) if 'depth' not in dims else dims['depth']
time = np.zeros(1, dtype=np.float64) if 'time' not in dims else dims['time']
time = np.array(time) if not isinstance(time, np.ndarray) else time
if isinstance(time[0], np.datetime64):
time_origin = TimeConverter(time[0])
time = np.array([time_origin.reltime(t) for t in time])
else:
time_origin = TimeConverter(0)
grid = Grid.create_grid(lon, lat, depth, time, time_origin=time_origin, mesh=mesh)
if 'creation_log' not in kwargs.keys():
kwargs['creation_log'] = 'from_data'
fields[name] = Field(name, datafld, grid=grid, transpose=transpose,
allow_time_extrapolation=allow_time_extrapolation, time_periodic=time_periodic, **kwargs)
u = fields.pop('U', None)
v = fields.pop('V', None)
return cls(u, v, fields=fields)
def test_TimeConverter():
cf_datetime_names = _get_cftime_datetimes()
for cf_datetime in cf_datetime_names:
date = getattr(cftime, cf_datetime)(1990, 1, 1)
assert TimeConverter(date).calendar == date.calendar
assert TimeConverter(None).calendar is None
date_datetime64 = np.datetime64('2001-01-01T12:00')
assert TimeConverter(date_datetime64).calendar == "np_datetime64"
def __init__(self, lon, lat, time, time_origin, mesh):
self.xi = None
self.yi = None
self.zi = None
self.ti = -1
self.lon = lon
self.lat = lat
self.time = np.zeros(1, dtype=np.float64) if time is None else time
if not self.lon.dtype == np.float32:
self.lon = self.lon.astype(np.float32)
if not self.lat.dtype == np.float32:
self.lat = self.lat.astype(np.float32)
if not self.time.dtype == np.float64:
assert isinstance(
self.time[0],
(np.integer, np.floating, float,
int)), 'Time vector must be an array of int or floats'
self.time = self.time.astype(np.float64)
self.time_full = self.time # needed for deferred_loaded Fields
self.time_origin = TimeConverter(
) if time_origin is None else time_origin
assert isinstance(
self.time_origin,
TimeConverter), 'time_origin needs to be a TimeConverter object'
self.mesh = mesh
self.cstruct = None
self.cell_edge_sizes = {}
self.zonal_periodic = False
self.zonal_halo = 0
self.meridional_halo = 0
self.lat_flipped = False
self.defer_load = False
self.lonlat_minmax = np.array(
[np.nanmin(lon),
np.nanmax(lon),
np.nanmin(lat),
np.nanmax(lat)],
dtype=np.float32)
self.periods = 0
self.load_chunk = []
self.chunk_info = None
self.chunksize = None
self._add_last_periodic_data_timestep = False
self.depth_field = None
def from_data(cls, data, dimensions, transpose=False, mesh='spherical',
allow_time_extrapolation=None, time_periodic=False, **kwargs):
"""Initialise FieldSet object from raw data
:param data: Dictionary mapping field names to numpy arrays.
Note that at least a 'U' and 'V' numpy array need to be given
1. If data shape is [xdim, ydim], [xdim, ydim, zdim], [xdim, ydim, tdim] or [xdim, ydim, zdim, tdim],
whichever is relevant for the dataset, use the flag transpose=True
2. If data shape is [ydim, xdim], [zdim, ydim, xdim], [tdim, ydim, xdim] or [tdim, zdim, ydim, xdim],
use the flag transpose=False (default value)
3. If data has any other shape, you first need to reorder it
:param dimensions: Dictionary mapping field dimensions (lon,
lat, depth, time) to numpy arrays.
Note that dimensions can also be a dictionary of dictionaries if
dimension names are different for each variable
(e.g. dimensions['U'], dimensions['V'], etc).
:param transpose: Boolean whether to transpose data on read-in
:param mesh: String indicating the type of mesh coordinates and
units used during velocity interpolation:
1. spherical (default): Lat and lon in degree, with a
correction for zonal velocity U near the poles.
2. flat: No conversion, lat/lon are assumed to be in m.
:param allow_time_extrapolation: boolean whether to allow for extrapolation
(i.e. beyond the last available time snapshot)
Default is False if dimensions includes time, else True
:param time_periodic: boolean whether to loop periodically over the time component of the FieldSet
This flag overrides the allow_time_interpolation and sets it to False
"""
fields = {}
for name, datafld in data.items():
# Use dimensions[name] if dimensions is a dict of dicts
dims = dimensions[name] if name in dimensions else dimensions
if allow_time_extrapolation is None:
allow_time_extrapolation = False if 'time' in dims else True
lon = dims['lon']
lat = dims['lat']
depth = np.zeros(1, dtype=np.float32) if 'depth' not in dims else dims['depth']
time = np.zeros(1, dtype=np.float64) if 'time' not in dims else dims['time']
grid = RectilinearZGrid(lon, lat, depth, time, time_origin=TimeConverter(), mesh=mesh)
fields[name] = Field(name, datafld, grid=grid, transpose=transpose,
allow_time_extrapolation=allow_time_extrapolation, time_periodic=time_periodic, **kwargs)
u = fields.pop('U', None)
v = fields.pop('V', None)
return cls(u, v, fields=fields)
class Grid(object):
"""Grid class that defines a (spatial and temporal) grid on which Fields are defined
"""
def __init__(self, lon, lat, time, time_origin, mesh):
self.lon = lon
self.lat = lat
self.time = np.zeros(1, dtype=np.float64) if time is None else time
if not self.lon.dtype == np.float32:
logger.warning_once("Casting lon data to np.float32")
self.lon = self.lon.astype(np.float32)
if not self.lat.dtype == np.float32:
logger.warning_once("Casting lat data to np.float32")
self.lat = self.lat.astype(np.float32)
if not self.time.dtype == np.float64:
assert isinstance(self.time[0], (np.integer, np.floating, float, int)), 'Time vector must be an array of int or floats'
logger.warning_once("Casting time data to np.float64")
self.time = self.time.astype(np.float64)
self.time_full = self.time # needed for deferred_loaded Fields
self.time_origin = TimeConverter() if time_origin is None else time_origin
assert isinstance(self.time_origin, TimeConverter), 'time_origin needs to be a TimeConverter object'
self.mesh = mesh
self.cstruct = None
self.cell_edge_sizes = {}
self.zonal_periodic = False
self.zonal_halo = 0
self.meridional_halo = 0
self.lat_flipped = False
self.defer_load = False
self.lonlat_minmax = np.array([np.nanmin(lon), np.nanmax(lon), np.nanmin(lat), np.nanmax(lat)], dtype=np.float32)
self.periods = 0
@staticmethod
def create_grid(lon, lat, depth, time, time_origin, mesh, **kwargs):
if len(lon.shape) == 1:
if depth is None or len(depth.shape) == 1:
return RectilinearZGrid(lon, lat, depth, time, time_origin=time_origin, mesh=mesh, **kwargs)
else:
return RectilinearSGrid(lon, lat, depth, time, time_origin=time_origin, mesh=mesh, **kwargs)
else:
if depth is None or len(depth.shape) == 1:
return CurvilinearZGrid(lon, lat, depth, time, time_origin=time_origin, mesh=mesh, **kwargs)
else:
return CurvilinearSGrid(lon, lat, depth, time, time_origin=time_origin, mesh=mesh, **kwargs)
@property
def ctypes_struct(self):
# This is unnecessary for the moment, but it could be useful when going will fully unstructured grids
self.cgrid = cast(pointer(self.child_ctypes_struct), c_void_p)
cstruct = CGrid(self.gtype, self.cgrid.value)
return cstruct
@property
def child_ctypes_struct(self):
"""Returns a ctypes struct object containing all relevant
pointers and sizes for this grid."""
class CStructuredGrid(Structure):
# z4d is only to have same cstruct as RectilinearSGrid
_fields_ = [('xdim', c_int), ('ydim', c_int), ('zdim', c_int),
('tdim', c_int), ('z4d', c_int),
('mesh_spherical', c_int), ('zonal_periodic', c_int),
('tfull_min', c_double), ('tfull_max', c_double), ('periods', POINTER(c_int)),
('lonlat_minmax', POINTER(c_float)),
('lon', POINTER(c_float)), ('lat', POINTER(c_float)),
('depth', POINTER(c_float)), ('time', POINTER(c_double))
]
# Create and populate the c-struct object
if not self.cstruct: # Not to point to the same grid various times if grid in various fields
if not isinstance(self.periods, c_int):
self.periods = c_int()
self.periods.value = 0
self.cstruct = CStructuredGrid(self.xdim, self.ydim, self.zdim,
self.tdim, self.z4d,
self.mesh == 'spherical', self.zonal_periodic,
self.time_full[0], self.time_full[-1], pointer(self.periods),
self.lonlat_minmax.ctypes.data_as(POINTER(c_float)),
self.lon.ctypes.data_as(POINTER(c_float)),
self.lat.ctypes.data_as(POINTER(c_float)),
self.depth.ctypes.data_as(POINTER(c_float)),
self.time.ctypes.data_as(POINTER(c_double)))
return self.cstruct
def lon_grid_to_target(self):
if self.lon_remapping:
self.lon = self.lon_remapping.to_target(self.lon)
def lon_grid_to_source(self):
if self.lon_remapping:
self.lon = self.lon_remapping.to_source(self.lon)
def lon_particle_to_target(self, lon):
if self.lon_remapping:
return self.lon_remapping.particle_to_target(lon)
return lon
def advancetime(self, grid_new):
assert isinstance(grid_new.time_origin, type(self.time_origin)), 'time_origin of new and old grids must be either both None or both a date'
if self.time_origin:
grid_new.time = grid_new.time + self.time_origin.reltime(grid_new.time_origin)
if len(grid_new.time) != 1:
raise RuntimeError('New FieldSet needs to have only one snapshot')
if grid_new.time > self.time[-1]: # forward in time, so appending at end
self.time = np.concatenate((self.time[1:], grid_new.time))
return 1
elif grid_new.time < self.time[0]: # backward in time, so prepending at start
self.time = np.concatenate((grid_new.time, self.time[:-1]))
return -1
else:
raise RuntimeError("Time of field_new in Field.advancetime() overlaps with times in old Field")
def check_zonal_periodic(self):
if self.zonal_periodic or self.mesh == 'flat':
return
dx = (self.lon[1:] - self.lon[:-1]) if len(self.lon.shape) == 1 else self.lon[0, 1:] - self.lon[0, :-1]
dx = np.where(dx < -180, dx+360, dx)
dx = np.where(dx > 180, dx-360, dx)
self.zonal_periodic = sum(dx) > 359.9
def add_Sdepth_periodic_halo(self, zonal, meridional, halosize):
if zonal:
if len(self.depth.shape) == 3:
self.depth = np.concatenate((self.depth[:, :, -halosize:], self.depth,
self.depth[:, :, 0:halosize]), axis=len(self.depth.shape) - 1)
assert self.depth.shape[2] == self.xdim, "Third dim must be x."
else:
self.depth = np.concatenate((self.depth[:, :, :, -halosize:], self.depth,
self.depth[:, :, :, 0:halosize]), axis=len(self.depth.shape) - 1)
assert self.depth.shape[3] == self.xdim, "Fourth dim must be x."
if meridional:
if len(self.depth.shape) == 3:
self.depth = np.concatenate((self.depth[:, -halosize:, :], self.depth,
self.depth[:, 0:halosize, :]), axis=len(self.depth.shape) - 2)
assert self.depth.shape[1] == self.ydim, "Second dim must be y."
else:
self.depth = np.concatenate((self.depth[:, :, -halosize:, :], self.depth,
self.depth[:, :, 0:halosize, :]), axis=len(self.depth.shape) - 2)
assert self.depth.shape[2] == self.ydim, "Third dim must be y."
def computeTimeChunk(self, f, time, signdt):
nextTime_loc = np.infty * signdt
periods = self.periods.value if isinstance(self.periods, c_int) else self.periods
if self.update_status == 'not_updated':
if self.ti >= 0:
if (time - periods*(self.time_full[-1]-self.time_full[0]) < self.time[0] or time - periods*(self.time_full[-1]-self.time_full[0]) > self.time[2]):
self.ti = -1 # reset
elif (time - periods*(self.time_full[-1]-self.time_full[0]) < self.time_full[0] or time - periods*(self.time_full[-1]-self.time_full[0]) >= self.time_full[-1]):
self.ti = -1 # reset
elif signdt >= 0 and time - periods*(self.time_full[-1]-self.time_full[0]) >= self.time[1] and self.ti < len(self.time_full)-3:
self.ti += 1
self.time = self.time_full[self.ti:self.ti+3]
self.update_status = 'updated'
elif signdt == -1 and time - periods*(self.time_full[-1]-self.time_full[0]) <= self.time[1] and self.ti > 0:
self.ti -= 1
self.time = self.time_full[self.ti:self.ti+3]
self.update_status = 'updated'
if self.ti == -1:
self.time = self.time_full
self.ti, _ = f.time_index(time)
periods = self.periods.value if isinstance(self.periods, c_int) else self.periods
if self.ti > 0 and signdt == -1:
self.ti -= 1
if self.ti >= len(self.time_full) - 2:
self.ti = len(self.time_full) - 3
self.time = self.time_full[self.ti:self.ti+3]
self.tdim = 3
self.update_status = 'first_updated'
if signdt >= 0 and (self.ti < len(self.time_full)-3 or not f.allow_time_extrapolation):
nextTime_loc = self.time[2] + periods*(self.time_full[-1]-self.time_full[0])
elif signdt == -1 and (self.ti > 0 or not f.allow_time_extrapolation):
nextTime_loc = self.time[0] + periods*(self.time_full[-1]-self.time_full[0])
return nextTime_loc