Beispiel #1
0
 def test_box_notok(self):
     for box in [[], list(range(0, 12))]:
         with pytest.raises(ValueError):
             is_box(box)
         with pytest.raises(ValueError):
             is_box(box, errors="raise")
         assert not is_box(box, errors="ignore")
Beispiel #2
0
    def init(self, box: list, **kw):
        """ Create Argo data loader

            Parameters
            ----------
            box : list(float, float, float, float, float, float, str, str)
                The box domain to load all Argo data for:
                    box = [lon_min, lon_max, lat_min, lat_max, pres_min, pres_max]
                    or:
                    box = [lon_min, lon_max, lat_min, lat_max, pres_min, pres_max, datim_min, datim_max]
        """
        # if len(box) == 6:
        # Select the last months of data:
        # end = pd.to_datetime('now')
        # start = end - pd.DateOffset(months=1)
        # box.append(start.strftime('%Y-%m-%d'))
        # box.append(end.strftime('%Y-%m-%d'))
        is_box(box, errors="raise")
        self.BOX = box

        if self.dataset_id == "phy":
            self.definition = "Ifremer erddap Argo data fetcher for a space/time region"
        elif self.dataset_id == "ref":
            self.definition = (
                "Ifremer erddap Argo REFERENCE data fetcher for a space/time region"
            )

        return self
Beispiel #3
0
    def init(self, box: list):
        """ Create Argo data loader

        Parameters
        ----------
        box : list()
            The box domain to load all Argo data for, with one of the following convention:

                - box = [lon_min, lon_max, lat_min, lat_max, pres_min, pres_max]
                - box = [lon_min, lon_max, lat_min, lat_max, pres_min, pres_max, datim_min, datim_max]
        """
        # We use a full domain definition (x, y, z, t) as argument for compatibility with the other fetchers
        # but at this point, we internally work only with x, y and t.
        is_box(box)
        self.BOX = box
        self.indexBOX = [box[ii] for ii in [0, 1, 2, 3]]
        if len(box) == 8:
            self.indexBOX = [box[ii] for ii in [0, 1, 2, 3, 6, 7]]

        # if len(box) == 6:
        #     # Select the last months of data:
        #     end = pd.to_datetime('now')
        #     start = end - pd.DateOffset(months=1)
        #     self.BOX.append(start.strftime('%Y-%m-%d'))
        #     self.BOX.append(end.strftime('%Y-%m-%d'))

        self.fs_index = indexstore(self.cache,
                                   self.cachedir,
                                   os.path.sep.join([self.local_ftp, "ar_index_global_prof.txt"]))
Beispiel #4
0
 def test_box_invalid_str(self):
     for i in [6, 7]:
         box = self.BOX4d
         box[i] = "str"
         with pytest.raises(ValueError):
             is_box(box)
         with pytest.raises(ValueError):
             is_box(box, errors="raise")
         assert not is_box(box, errors="ignore")
Beispiel #5
0
 def test_box_invalid_range(self):
     for i in [0, 1, 2, 3, 4, 5]:
         box = self.BOX3d
         box[i] = -1000
         with pytest.raises(ValueError):
             is_box(box)
         with pytest.raises(ValueError):
             is_box(box, errors="raise")
         assert not is_box(box, errors="ignore")
Beispiel #6
0
    def init(self, box: list):
        """ Create Argo data loader

            Parameters
            ----------
            box : list(float, float, float, float, float, float, str, str)
                The box domain to load all Argo data for:
                box = [lon_min, lon_max, lat_min, lat_max, pres_min, pres_max]
                or:
                box = [lon_min, lon_max, lat_min, lat_max, pres_min, pres_max, datim_min, datim_max]
        """
        if len(box) == 6:
            # Select the last months of data:
            end = pd.to_datetime("now")
            start = end - pd.DateOffset(months=1)
            box.append(start.strftime("%Y-%m-%d"))
            box.append(end.strftime("%Y-%m-%d"))
        is_box(box)
        self.BOX = box

        self.definition = "?"
        if self.dataset_id == "phy":
            self.definition = "Argovis Argo data fetcher for a space/time region"
        return self
Beispiel #7
0
 def test_box_ok(self):
     assert is_box(self.BOX3d)
     assert is_box(self.BOX4d)
Beispiel #8
0
    def test_chunk_box4d(self):
        C = Chunker({"box": self.BOX4d})
        assert all([is_box(chunk) for chunk in C.fit_transform()])

        C = Chunker({"box": self.BOX4d}, chunks="auto")
        assert all([is_box(chunk) for chunk in C.fit_transform()])

        C = Chunker({"box": self.BOX4d},
                    chunks={
                        "lon": 2,
                        "lat": 1,
                        "dpt": 1,
                        "time": 1
                    })
        assert all([is_box(chunk) for chunk in C.fit_transform()])
        assert len(C.fit_transform()) == 2

        C = Chunker(
            {"box": self.BOX4d},
            chunks={
                "lat": 1,
                "dpt": 1,
                "time": 1
            },
            chunksize={"lon": 10},
        )
        chunks = C.fit_transform()
        assert all([is_box(chunk) for chunk in chunks])
        assert chunks[0][1] - chunks[0][0] == 10

        C = Chunker({"box": self.BOX4d},
                    chunks={
                        "lon": 1,
                        "lat": 2,
                        "dpt": 1,
                        "time": 1
                    })
        assert all([is_box(chunk) for chunk in C.fit_transform()])
        assert len(C.fit_transform()) == 2

        C = Chunker(
            {"box": self.BOX4d},
            chunks={
                "lon": 1,
                "dpt": 1,
                "time": 1
            },
            chunksize={"lat": 10},
        )
        chunks = C.fit_transform()
        assert all([is_box(chunk) for chunk in chunks])
        assert chunks[0][3] - chunks[0][2] == 10

        C = Chunker({"box": self.BOX4d},
                    chunks={
                        "lon": 1,
                        "lat": 1,
                        "dpt": 2,
                        "time": 1
                    })
        assert all([is_box(chunk) for chunk in C.fit_transform()])
        assert len(C.fit_transform()) == 2

        C = Chunker(
            {"box": self.BOX4d},
            chunks={
                "lon": 1,
                "lat": 1,
                "time": 1
            },
            chunksize={"dpt": 10},
        )
        chunks = C.fit_transform()
        assert all([is_box(chunk) for chunk in chunks])
        assert chunks[0][5] - chunks[0][4] == 10

        C = Chunker({"box": self.BOX4d},
                    chunks={
                        "lon": 1,
                        "lat": 1,
                        "dpt": 1,
                        "time": 2
                    })
        assert all([is_box(chunk) for chunk in C.fit_transform()])
        assert len(C.fit_transform()) == 2

        C = Chunker(
            {"box": self.BOX4d},
            chunks={
                "lon": 1,
                "lat": 1,
                "dpt": 1
            },
            chunksize={"time": 5},
        )
        chunks = C.fit_transform()
        assert all([is_box(chunk) for chunk in chunks])
        assert np.timedelta64(
            pd.to_datetime(chunks[0][7]) - pd.to_datetime(chunks[0][6]),
            "D") <= np.timedelta64(5, "D")

        with pytest.raises(ValueError):
            Chunker({"box": self.BOX4d}, chunks=["lon", 1])

        C = Chunker({"box": self.BOX4d})
        assert isinstance(C.this_chunker, types.FunctionType) or isinstance(
            C.this_chunker, types.MethodType)
Beispiel #9
0
    def test_chunk_box3d(self):
        C = Chunker({"box": self.BOX3d})
        assert all([is_box(chunk) for chunk in C.fit_transform()])

        C = Chunker({"box": self.BOX3d}, chunks="auto")
        assert all([is_box(chunk) for chunk in C.fit_transform()])

        C = Chunker({"box": self.BOX3d},
                    chunks={
                        "lon": 12,
                        "lat": 1,
                        "dpt": 1
                    })
        assert all([is_box(chunk) for chunk in C.fit_transform()])
        assert len(C.fit_transform()) == 12

        C = Chunker({"box": self.BOX3d},
                    chunks={
                        "lat": 1,
                        "dpt": 1
                    },
                    chunksize={"lon": 10})
        chunks = C.fit_transform()
        assert all([is_box(chunk) for chunk in chunks])
        assert chunks[0][1] - chunks[0][0] == 10

        C = Chunker({"box": self.BOX3d},
                    chunks={
                        "lon": 1,
                        "lat": 12,
                        "dpt": 1
                    })
        assert all([is_box(chunk) for chunk in C.fit_transform()])
        assert len(C.fit_transform()) == 12

        C = Chunker({"box": self.BOX3d},
                    chunks={
                        "lon": 1,
                        "dpt": 1
                    },
                    chunksize={"lat": 10})
        chunks = C.fit_transform()
        assert all([is_box(chunk) for chunk in chunks])
        assert chunks[0][3] - chunks[0][2] == 10

        C = Chunker({"box": self.BOX3d},
                    chunks={
                        "lon": 1,
                        "lat": 1,
                        "dpt": 12
                    })
        assert all([is_box(chunk) for chunk in C.fit_transform()])
        assert len(C.fit_transform()) == 12

        C = Chunker({"box": self.BOX3d},
                    chunks={
                        "lon": 1,
                        "lat": 1
                    },
                    chunksize={"dpt": 10})
        chunks = C.fit_transform()
        assert all([is_box(chunk) for chunk in chunks])
        assert chunks[0][5] - chunks[0][4] == 10

        C = Chunker({"box": self.BOX3d}, chunks={"lon": 4, "lat": 2, "dpt": 1})
        assert all([is_box(chunk) for chunk in C.fit_transform()])
        assert len(C.fit_transform()) == 2 * 4

        C = Chunker({"box": self.BOX3d}, chunks={"lon": 2, "lat": 3, "dpt": 4})
        assert all([is_box(chunk) for chunk in C.fit_transform()])
        assert len(C.fit_transform()) == 2 * 3 * 4

        with pytest.raises(ValueError):
            Chunker({"box": self.BOX3d}, chunks=["lon", 1])

        C = Chunker({"box": self.BOX3d})
        assert isinstance(C.this_chunker, types.FunctionType) or isinstance(
            C.this_chunker, types.MethodType)