def test_index():
a_nc_subset = EddiesObservations.load_file(
a_filename, indexs=dict(obs=slice(500, 1000))
)
a_zarr_subset = EddiesObservations.load_from_zarr(
memory_store, indexs=dict(obs=slice(500, 1000)), buffer_size=50
)
assert a_nc_subset == a_zarr_subset
def get_unused_data(self, raw_data=False):
"""
Add in track object all the observations which aren't selected
Returns: Unused Eddies
"""
self.reset_dataset_cache()
self.swap_dataset(self.datasets[0], raw_data=raw_data)
nb_dataset = len(self.datasets)
# Get the number of obs unused
nb_obs = 0
list_mask = list()
has_virtual = "virtual" in self[0].dtype.names
logger.debug("Count unused data ...")
for i, filename in enumerate(self.datasets):
last_dataset = i == (nb_dataset - 1)
if has_virtual and not last_dataset:
m_in = ~self[i]["virtual"]
else:
m_in = slice(None)
if i == 0:
eddies_used = self[i]["in"]
elif last_dataset:
eddies_used = self[i - 1]["out"]
else:
eddies_used = unique(
concatenate((self[i - 1]["out"], self[i]["in"][m_in])))
if not isinstance(filename, str):
filename = filename.astype(str)
with Dataset(filename) as h:
nb_obs_day = len(h.dimensions["obs"])
m = ones(nb_obs_day, dtype="bool")
m[eddies_used] = False
list_mask.append(m)
nb_obs += m.sum()
logger.debug("Count unused data OK")
eddies = EddiesObservations(
size=nb_obs,
track_extra_variables=self.current_obs.track_extra_variables,
track_array_variables=self.current_obs.track_array_variables,
array_variables=self.current_obs.array_variables,
raw_data=raw_data,
)
j = 0
for i, dataset in enumerate(self.datasets):
logger.debug("Loaf file : (%d) %s", i, dataset)
current_obs = self.class_method.load_file(dataset,
raw_data=raw_data)
if i == 0:
eddies.sign_type = current_obs.sign_type
unused_obs = current_obs.observations[list_mask[i]]
nb = unused_obs.shape[0]
eddies.observations[j:j + nb] = unused_obs
j += nb
return eddies
def get_unused_data(self, raw_data=False):
"""
Add in track object all the observations which aren't selected
Returns: Unused Eddies
"""
nb_dataset = len(self.datasets)
has_virtual = "virtual" in self[0].dtype.names
eddies = list()
for i, dataset in enumerate(self.datasets):
last_dataset = i == (nb_dataset - 1)
if has_virtual and not last_dataset:
m_in = ~self[i]["virtual"]
else:
m_in = slice(None)
if i == 0:
index_used = self[i]["in"]
elif last_dataset:
index_used = self[i - 1]["out"]
else:
index_used = unique(
concatenate((self[i - 1]["out"], self[i]["in"][m_in])))
logger.debug("Load file : %s", dataset)
if self.memory:
with open(dataset, "rb") as h:
current_obs = self.class_method.load_file(
h, raw_data=raw_data)
else:
current_obs = self.class_method.load_file(dataset,
raw_data=raw_data)
eddies.append(current_obs.index(index_used, reverse=True))
return EddiesObservations.concatenate(eddies)
- Circle
In the two case we use a least square algorithm
"""
from matplotlib import pyplot as plt
from numpy import cos, linspace, radians, sin
from py_eddy_tracker import data
from py_eddy_tracker.generic import coordinates_to_local, local_to_coordinates
from py_eddy_tracker.observations.observation import EddiesObservations
from py_eddy_tracker.poly import fit_circle_, fit_ellips
# %%
# Load example identification file
a = EddiesObservations.load_file(data.get_path("Anticyclonic_20190223.nc"))
# %%
# Function to draw circle or ellips from parameter
def build_circle(x0, y0, r):
angle = radians(linspace(0, 360, 50))
x_norm, y_norm = cos(angle), sin(angle)
return local_to_coordinates(x_norm * r, y_norm * r, x0, y0)
def build_ellips(x0, y0, a, b, theta):
angle = radians(linspace(0, 360, 50))
x = a * cos(theta) * cos(angle) - b * sin(theta) * sin(angle)
y = a * sin(theta) * cos(angle) + b * cos(theta) * sin(angle)
return local_to_coordinates(x, y, x0, y0)
def update_axes(ax, mappable=None):
ax.grid()
if mappable:
plt.colorbar(mappable, cax=ax.figure.add_axes([0.95, 0.05, 0.01, 0.9]))
# %%
# We load demo sample and take only first year.
#
# Replace by a list of filename to apply on your own dataset.
file_objects = get_remote_demo_sample(
"eddies_med_adt_allsat_dt2018/Anticyclonic_2010_2011_2012")[:365]
# %%
# Merge all identification datasets in one object
all_a = EddiesObservations.concatenate(
[EddiesObservations.load_file(i) for i in file_objects])
# %%
# We define polygon bound
x0, x1, y0, y1 = 15, 20, 33, 38
xs = np.array([[x0, x1, x1, x0, x0]], dtype="f8")
ys = np.array([[y0, y0, y1, y1, y0]], dtype="f8")
# Polygon object created for the match function use.
polygon = dict(contour_lon_e=xs,
contour_lat_e=ys,
contour_lon_s=xs,
contour_lat_s=ys)
# %%
# Geographic frequency of eddies
step = 0.125
import py_eddy_tracker.gui
from py_eddy_tracker import data
from py_eddy_tracker.dataset.grid import RegularGridDataset
from py_eddy_tracker.observations.observation import EddiesObservations
# %%
# Load Input grid, ADT is used to detect eddies
g = RegularGridDataset(
data.get_path("dt_med_allsat_phy_l4_20160515_20190101.nc"), "longitude",
"latitude")
# Compute u/v from height
g.add_uv("adt")
# %%
# Load detection files
a = EddiesObservations.load_file(data.get_path("Anticyclonic_20160515.nc"))
c = EddiesObservations.load_file(data.get_path("Cyclonic_20160515.nc"))
# %%
# Quiver from u/v with eddies
fig = plt.figure(figsize=(10, 5))
ax = fig.add_axes([0, 0, 1, 1], projection="full_axes")
ax.set_xlim(19, 30), ax.set_ylim(31, 36.5), ax.grid()
x, y = np.meshgrid(g.x_c, g.y_c)
a.filled(ax, facecolors="r", alpha=0.1), c.filled(ax,
facecolors="b",
alpha=0.1)
_ = ax.quiver(x.T, y.T, g.grid("u"), g.grid("v"), scale=20)
# %%
pcolormesh = ax.pcolorfast(x_g_, y_g_, lavd, **kw_vorticity)
update_axes(ax, pcolormesh)
_ = VideoAnimation(ax.figure, update, **kw_video)
# %%
# Final LAVD
# ^^^^^^^^^^
# %%
# Format LAVD data
lavd = RegularGridDataset.with_array(
coordinates=("lon", "lat"),
datas=dict(
lavd=lavd.T,
lon=x_g,
lat=y_g,
),
centered=True,
)
# %%
# Display final LAVD with py eddy tracker detection.
# Period used for LAVD integration (8 days) is too short for a real use, but choose for example efficiency.
fig, ax, _ = start_ax()
mappable = lavd.display(ax, "lavd", **kw_vorticity)
EddiesObservations.load_file(get_path("Anticyclonic_20160515.nc")).display(
ax, color="k")
EddiesObservations.load_file(get_path("Cyclonic_20160515.nc")).display(
ax, color="k")
_ = update_axes(ax, mappable)
from py_eddy_tracker.observations.network import NetworkObservations
from py_eddy_tracker.observations.observation import EddiesObservations, Table
from py_eddy_tracker.observations.tracking import TrackEddiesObservations
# %%
# Eddies can be stored in 2 formats with the same structure:
#
# - zarr (https://zarr.readthedocs.io/en/stable/), which allow efficiency in IO,...
# - NetCDF4 (https://unidata.github.io/netcdf4-python/), well-known format
#
# Each field are stored in column, each row corresponds at 1 observation,
# array field like contour/profile are 2D column.
# %%
# Eddies files (zarr or netcdf) can be loaded with ```load_file``` method:
eddies_collections = EddiesObservations.load_file(
get_demo_path("Cyclonic_20160515.nc"))
eddies_collections.field_table()
# offset and scale_factor are used only when data is stored in zarr or netCDF4
# %%
# Field access
# ------------
# To access the total field, here ```amplitude```
eddies_collections.amplitude
# To access only a specific part of the field
eddies_collections.amplitude[4:15]
# %%
# Data matrix is a numpy ndarray
eddies_collections.obs
import zarr
from netCDF4 import Dataset
from py_eddy_tracker.data import get_demo_path
from py_eddy_tracker.featured_tracking.area_tracker import AreaTracker
from py_eddy_tracker.observations.observation import EddiesObservations
from py_eddy_tracker.tracking import Correspondances
filename = get_demo_path("Anticyclonic_20190223.nc")
a0 = EddiesObservations.load_file(filename)
a1 = a0.copy()
def test_area_tracking_parameter():
delta = 0.2
# All eddies will be shift of delta in longitude and latitude
for k in (
"lon",
"lon_max",
"contour_lon_s",
"contour_lon_e",
"lat",
"lat_max",
"contour_lat_s",
"contour_lat_e",
):
a1[k][:] -= delta
a1.time[:] += 1
# wrote in memory a0 and a1
h0, h1 = zarr.group(), zarr.group()
a0.to_zarr(h0), a1.to_zarr(h1)
# Merge of group, ref over etu
for i_, j_ in zip(ii[m], ij[m]):
g0, g1 = gr_i[i_], gr_j[j_]
apply_replace(gr, g0, g1)
NETWORK_GROUPS.append((i, j, gr.copy()))
return gr
# %%
# Movie period
t0 = (datetime(2005, 5, 1) - datetime(1950, 1, 1)).days
t1 = (datetime(2005, 6, 1) - datetime(1950, 1, 1)).days
# %%
# Get data from period and area
e = EddiesObservations.load_file(data.get_demo_path("network_med.nc"))
e = e.extract_with_mask((e.time >= t0) * (e.time < t1)).extract_with_area(
dict(llcrnrlon=25, urcrnrlon=35, llcrnrlat=31, urcrnrlat=37.5))
# %%
# Reproduce individual daily identification(for demonstration)
EDDIES_BY_DAYS = list()
for i, b0, b1 in e.iter_on("time"):
EDDIES_BY_DAYS.append(e.index(i))
# need for display
e = EddiesObservations.concatenate(EDDIES_BY_DAYS)
# %%
# Run network building group to intercept every step
n = MyNetwork.from_eddiesobservations(EDDIES_BY_DAYS, window=7)
_ = n.group_observations(minimal_area=True)
import zarr
from py_eddy_tracker.data import get_path
from py_eddy_tracker.observations.observation import EddiesObservations
a_filename, c_filename = (
get_path("Anticyclonic_20190223.nc"),
get_path("Cyclonic_20190223.nc"),
)
a = EddiesObservations.load_file(a_filename)
a_raw = EddiesObservations.load_file(a_filename, raw_data=True)
memory_store = zarr.group()
# Dataset was raw loaded from netcdf and save in zarr
a_raw.to_zarr(memory_store, chunck_size=100000)
# We load zarr data without raw option
a_zarr = EddiesObservations.load_from_zarr(memory_store)
c = EddiesObservations.load_file(c_filename)
def test_merge():
new = a.merge(c)
assert len(new) == len(a) + len(c)
def test_zarr_raw():
assert a == a_zarr
def test_index():
a_nc_subset = EddiesObservations.load_file(
a_filename, indexs=dict(obs=slice(500, 1000)))
