def test_box_notok(self):
for box in [[], list(range(0, 12))]:
with pytest.raises(ValueError):
is_box(box)
with pytest.raises(ValueError):
is_box(box, errors="raise")
assert not is_box(box, errors="ignore")
def init(self, box: list, **kw):
""" Create Argo data loader
Parameters
----------
box : list(float, float, float, float, float, float, str, str)
The box domain to load all Argo data for:
box = [lon_min, lon_max, lat_min, lat_max, pres_min, pres_max]
or:
box = [lon_min, lon_max, lat_min, lat_max, pres_min, pres_max, datim_min, datim_max]
"""
# if len(box) == 6:
# Select the last months of data:
# end = pd.to_datetime('now')
# start = end - pd.DateOffset(months=1)
# box.append(start.strftime('%Y-%m-%d'))
# box.append(end.strftime('%Y-%m-%d'))
is_box(box, errors="raise")
self.BOX = box
if self.dataset_id == "phy":
self.definition = "Ifremer erddap Argo data fetcher for a space/time region"
elif self.dataset_id == "ref":
self.definition = (
"Ifremer erddap Argo REFERENCE data fetcher for a space/time region"
)
return self
def init(self, box: list):
""" Create Argo data loader
Parameters
----------
box : list()
The box domain to load all Argo data for, with one of the following convention:
- box = [lon_min, lon_max, lat_min, lat_max, pres_min, pres_max]
- box = [lon_min, lon_max, lat_min, lat_max, pres_min, pres_max, datim_min, datim_max]
"""
# We use a full domain definition (x, y, z, t) as argument for compatibility with the other fetchers
# but at this point, we internally work only with x, y and t.
is_box(box)
self.BOX = box
self.indexBOX = [box[ii] for ii in [0, 1, 2, 3]]
if len(box) == 8:
self.indexBOX = [box[ii] for ii in [0, 1, 2, 3, 6, 7]]
# if len(box) == 6:
# # Select the last months of data:
# end = pd.to_datetime('now')
# start = end - pd.DateOffset(months=1)
# self.BOX.append(start.strftime('%Y-%m-%d'))
# self.BOX.append(end.strftime('%Y-%m-%d'))
self.fs_index = indexstore(self.cache,
self.cachedir,
os.path.sep.join([self.local_ftp, "ar_index_global_prof.txt"]))
def test_box_invalid_str(self):
for i in [6, 7]:
box = self.BOX4d
box[i] = "str"
with pytest.raises(ValueError):
is_box(box)
with pytest.raises(ValueError):
is_box(box, errors="raise")
assert not is_box(box, errors="ignore")
def test_box_invalid_range(self):
for i in [0, 1, 2, 3, 4, 5]:
box = self.BOX3d
box[i] = -1000
with pytest.raises(ValueError):
is_box(box)
with pytest.raises(ValueError):
is_box(box, errors="raise")
assert not is_box(box, errors="ignore")
def init(self, box: list):
""" Create Argo data loader
Parameters
----------
box : list(float, float, float, float, float, float, str, str)
The box domain to load all Argo data for:
box = [lon_min, lon_max, lat_min, lat_max, pres_min, pres_max]
or:
box = [lon_min, lon_max, lat_min, lat_max, pres_min, pres_max, datim_min, datim_max]
"""
if len(box) == 6:
# Select the last months of data:
end = pd.to_datetime("now")
start = end - pd.DateOffset(months=1)
box.append(start.strftime("%Y-%m-%d"))
box.append(end.strftime("%Y-%m-%d"))
is_box(box)
self.BOX = box
self.definition = "?"
if self.dataset_id == "phy":
self.definition = "Argovis Argo data fetcher for a space/time region"
return self
def test_box_ok(self):
assert is_box(self.BOX3d)
assert is_box(self.BOX4d)
def test_chunk_box4d(self):
C = Chunker({"box": self.BOX4d})
assert all([is_box(chunk) for chunk in C.fit_transform()])
C = Chunker({"box": self.BOX4d}, chunks="auto")
assert all([is_box(chunk) for chunk in C.fit_transform()])
C = Chunker({"box": self.BOX4d},
chunks={
"lon": 2,
"lat": 1,
"dpt": 1,
"time": 1
})
assert all([is_box(chunk) for chunk in C.fit_transform()])
assert len(C.fit_transform()) == 2
C = Chunker(
{"box": self.BOX4d},
chunks={
"lat": 1,
"dpt": 1,
"time": 1
},
chunksize={"lon": 10},
)
chunks = C.fit_transform()
assert all([is_box(chunk) for chunk in chunks])
assert chunks[0][1] - chunks[0][0] == 10
C = Chunker({"box": self.BOX4d},
chunks={
"lon": 1,
"lat": 2,
"dpt": 1,
"time": 1
})
assert all([is_box(chunk) for chunk in C.fit_transform()])
assert len(C.fit_transform()) == 2
C = Chunker(
{"box": self.BOX4d},
chunks={
"lon": 1,
"dpt": 1,
"time": 1
},
chunksize={"lat": 10},
)
chunks = C.fit_transform()
assert all([is_box(chunk) for chunk in chunks])
assert chunks[0][3] - chunks[0][2] == 10
C = Chunker({"box": self.BOX4d},
chunks={
"lon": 1,
"lat": 1,
"dpt": 2,
"time": 1
})
assert all([is_box(chunk) for chunk in C.fit_transform()])
assert len(C.fit_transform()) == 2
C = Chunker(
{"box": self.BOX4d},
chunks={
"lon": 1,
"lat": 1,
"time": 1
},
chunksize={"dpt": 10},
)
chunks = C.fit_transform()
assert all([is_box(chunk) for chunk in chunks])
assert chunks[0][5] - chunks[0][4] == 10
C = Chunker({"box": self.BOX4d},
chunks={
"lon": 1,
"lat": 1,
"dpt": 1,
"time": 2
})
assert all([is_box(chunk) for chunk in C.fit_transform()])
assert len(C.fit_transform()) == 2
C = Chunker(
{"box": self.BOX4d},
chunks={
"lon": 1,
"lat": 1,
"dpt": 1
},
chunksize={"time": 5},
)
chunks = C.fit_transform()
assert all([is_box(chunk) for chunk in chunks])
assert np.timedelta64(
pd.to_datetime(chunks[0][7]) - pd.to_datetime(chunks[0][6]),
"D") <= np.timedelta64(5, "D")
with pytest.raises(ValueError):
Chunker({"box": self.BOX4d}, chunks=["lon", 1])
C = Chunker({"box": self.BOX4d})
assert isinstance(C.this_chunker, types.FunctionType) or isinstance(
C.this_chunker, types.MethodType)
def test_chunk_box3d(self):
C = Chunker({"box": self.BOX3d})
assert all([is_box(chunk) for chunk in C.fit_transform()])
C = Chunker({"box": self.BOX3d}, chunks="auto")
assert all([is_box(chunk) for chunk in C.fit_transform()])
C = Chunker({"box": self.BOX3d},
chunks={
"lon": 12,
"lat": 1,
"dpt": 1
})
assert all([is_box(chunk) for chunk in C.fit_transform()])
assert len(C.fit_transform()) == 12
C = Chunker({"box": self.BOX3d},
chunks={
"lat": 1,
"dpt": 1
},
chunksize={"lon": 10})
chunks = C.fit_transform()
assert all([is_box(chunk) for chunk in chunks])
assert chunks[0][1] - chunks[0][0] == 10
C = Chunker({"box": self.BOX3d},
chunks={
"lon": 1,
"lat": 12,
"dpt": 1
})
assert all([is_box(chunk) for chunk in C.fit_transform()])
assert len(C.fit_transform()) == 12
C = Chunker({"box": self.BOX3d},
chunks={
"lon": 1,
"dpt": 1
},
chunksize={"lat": 10})
chunks = C.fit_transform()
assert all([is_box(chunk) for chunk in chunks])
assert chunks[0][3] - chunks[0][2] == 10
C = Chunker({"box": self.BOX3d},
chunks={
"lon": 1,
"lat": 1,
"dpt": 12
})
assert all([is_box(chunk) for chunk in C.fit_transform()])
assert len(C.fit_transform()) == 12
C = Chunker({"box": self.BOX3d},
chunks={
"lon": 1,
"lat": 1
},
chunksize={"dpt": 10})
chunks = C.fit_transform()
assert all([is_box(chunk) for chunk in chunks])
assert chunks[0][5] - chunks[0][4] == 10
C = Chunker({"box": self.BOX3d}, chunks={"lon": 4, "lat": 2, "dpt": 1})
assert all([is_box(chunk) for chunk in C.fit_transform()])
assert len(C.fit_transform()) == 2 * 4
C = Chunker({"box": self.BOX3d}, chunks={"lon": 2, "lat": 3, "dpt": 4})
assert all([is_box(chunk) for chunk in C.fit_transform()])
assert len(C.fit_transform()) == 2 * 3 * 4
with pytest.raises(ValueError):
Chunker({"box": self.BOX3d}, chunks=["lon", 1])
C = Chunker({"box": self.BOX3d})
assert isinstance(C.this_chunker, types.FunctionType) or isinstance(
C.this_chunker, types.MethodType)