Beispiel #1
0
class Test_ISMN_Interface_CeopUnzipped(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        super(Test_ISMN_Interface_CeopUnzipped, cls).setUpClass()

        testdata = os.path.join(testdata_root,
                                "Data_seperate_files_20170810_20180809")
        metadata_path = os.path.join(testdata, "python_metadata")

        cleanup(metadata_path)
        ds = ISMN_Interface(testdata, network=[], parallel=True)
        assert ds.networks == OrderedDict()
        cls.testdata = testdata

    def setUp(self) -> None:
        self.ds = ISMN_Interface(self.testdata, network=["COSMOS"])

    def tearDown(self) -> None:
        self.ds.close_files()
        logging.shutdown()

    def test_list(self):
        with pytest.deprecated_call():
            assert len(self.ds.list_networks()) == 1
            assert len(self.ds.list_stations()) == len(
                self.ds.list_stations("COSMOS")) == 2
            assert len(self.ds.list_sensors()) == 2
            assert len(self.ds.list_sensors(station="Barrow-ARM")) == 1

    def test_network_for_station(self):
        assert self.ds.network_for_station("Barrow-ARM") == "COSMOS"
        assert self.ds.network_for_station("ARM-1") == "COSMOS"

    def test_stations_that_measure(self):
        for s in self.ds.stations_that_measure("soil_moisture"):
            assert s.name in ["ARM-1", "Barrow-ARM"]

        for s in self.ds.stations_that_measure("nonexisting"):
            raise AssertionError("Found var that doesnt exist")

    def test_get_dataset_ids(self):
        ids = self.ds.get_dataset_ids("soil_moisture",
                                      max_depth=100,
                                      groupby="network")
        assert list(ids.keys()) == ["COSMOS"]
        assert ids["COSMOS"] == [0, 1]

        ids = self.ds.get_dataset_ids("soil_moisture", max_depth=0.19)
        assert ids == [0]

        ids = self.ds.get_dataset_ids(
            ["soil_moisture"],
            max_depth=99,
            filter_meta_dict={
                "lc_2010": 210,
                "network": "COSMOS",
                "station": "Barrow-ARM",
            },
        )
        assert ids == [1]

        ids = self.ds.get_dataset_ids("novar")
        assert len(ids) == 0

        ids = self.ds.get_dataset_ids(["soil_moisture", "shouldhavenoeffect"],
                                      0.0, 0.19)  # should get 1
        assert len(ids) == 1

        ids = self.ds.get_dataset_ids("soil_moisture", 0.0,
                                      1.0)  # should get 2
        assert len(ids) == 2

        ids = self.ds.get_dataset_ids("soil_moisture",
                                      0.0,
                                      1.0,
                                      filter_meta_dict={"lc_2010":
                                                        210})  # should get 1
        assert len(ids) == 1

        ids = self.ds.get_dataset_ids("nonexisting")  # should get 0
        assert len(ids) == 0

    def test_read_ts(self):
        data1 = self.ds.read(0)
        assert not data1.empty

        data2, meta = self.ds.read_ts(1, return_meta=True)
        assert not data2.empty

    def test_read_metadata(self):
        data2, meta = self.ds.read_ts(1, return_meta=True)
        assert all(meta == self.ds.read_metadata(1, format="pandas"))
        assert self.ds.read_metadata(1, format="dict") is not None
        assert self.ds.read_metadata([1], format="obj") is not None

        assert not self.ds.metadata.empty
        assert self.ds.metadata.loc[1]['station']['val'] \
               == self.ds.read_metadata([0,1]).loc[1, ('station', 'val')]

    def test_find_nearest_station(self):
        should_lon, should_lat = -156.62870, 71.32980

        station = self.ds.find_nearest_station(should_lon, should_lat)

        assert station.lon == should_lon
        assert station.lat == should_lat

    def test_plot_station_locations(self):
        with TemporaryDirectory() as out_dir:
            outpath = os.path.join(out_dir, "plot.png")
            self.ds.plot_station_locations(["soil_moisture", 'precipitation'],
                                           markersize=5,
                                           filename=outpath)
            assert len(os.listdir(out_dir)) == 1

    def test_get_min_max_obs_timestamps(self):
        tmin, tmax = self.ds.get_min_max_obs_timestamps("soil_moisture",
                                                        max_depth=0.19)
        assert tmin == datetime(2017, 8, 10, 0)
        assert tmax == datetime(2018, 8, 9, 23)

    def test_get_min_max_obs_timestamps_for_station(self):
        station = self.ds.collection.networks["COSMOS"].stations["ARM-1"]
        tmin, tmax = station.get_min_max_obs_timestamp("soil_moisture", 0,
                                                       0.19)
        assert tmin == datetime(2017, 8, 10, 0)
        assert tmax == datetime(2018, 8, 9, 23)

    def test_get_static_var_val(self):
        vals = self.ds.get_static_var_vals("soil_moisture", max_depth=0.19)
        assert vals == {130: "Grassland"}

        vals = self.ds.get_landcover_types("soil_moisture", max_depth=100)
        assert len(vals) == 2
        assert vals[130] == "Grassland"
        assert vals[210] == "Water"
        self.ds.print_landcover_dict()

        vals = self.ds.get_climate_types("soil_moisture",
                                         max_depth=100,
                                         climate="climate_KG")
        assert len(vals) == 2
        assert vals["ET"] == "Polar Tundra"
        assert vals["Cfa"] == "Temperate Without Dry Season, Hot Summer"
        self.ds.print_climate_dict()

    def test_get_var(self):
        vars = self.ds.get_variables()
        assert vars == ["soil_moisture"]

    def test_get_sensors(self):
        i = 0
        for nw, station in self.ds.collection.iter_stations(
                filter_meta_dict={"network": "COSMOS"}):
            for se in station.iter_sensors():
                data = se.read_data()
                # check if the networks is COSMOS or station in [ARM, Barrow-ARM]
                assert not data.empty
                # check something for that one station
                i += 1
        assert i == 2

        i = 0
        for se in self.ds.networks["COSMOS"].stations[
                "Barrow-ARM"].iter_sensors():
            data = se.read_data()
            assert not data.empty
            # check something for that one station
            i += 1
        assert i == 1

        i = 0
        for net, stat, sens in self.ds.collection.iter_sensors(
                depth=Depth(0, 1),
                filter_meta_dict={"station": ["Barrow-ARM", "ARM-1"]},
        ):
            data = sens.read_data()
            assert not data.empty
            i += 1
        assert i == 2

        for nw, station in self.ds.collection.iter_stations():
            for se in station.iter_sensors(variable="nonexisting"):
                raise ValueError("Found sensor, although none should exist")

    def test_get_nearest_station(self):
        should_lon, should_lat = -156.62870, 71.32980

        station, dist = self.ds.collection.get_nearest_station(
            should_lon, should_lat)
        assert dist == 0
        assert station.lon == should_lon
        assert station.lat == should_lat
        gpi, dist = self.ds.collection.grid.find_nearest_gpi(
            int(should_lon), int(should_lat))
        assert dist != 0
        for net in self.ds.collection.iter_networks():
            if station.name in net.stations.keys():
                assert net.stations[station.name].lon == should_lon
                assert net.stations[station.name].lat == should_lat

        station, dist = self.ds.find_nearest_station(0,
                                                     0,
                                                     return_distance=True,
                                                     max_dist=100)
        assert station == dist == None

    def test_citation(self):
        with TemporaryDirectory() as out_dir:
            out_file = os.path.join(out_dir, 'citation.txt')
            refs = self.ds.collection.export_citations(out_file=out_file)
            assert all([
                net in refs.keys()
                for net in list(self.ds.collection.networks.keys())
            ])
            assert os.path.exists(out_file)
            with open(out_file, mode='r') as f:
                lines = f.readlines()
                assert len(lines) > 0
Beispiel #2
0
class Test_ISMN_Interface_CeopUnzipped(unittest.TestCase):
    
    @classmethod
    def setUpClass(cls):
        super(Test_ISMN_Interface_CeopUnzipped, cls).setUpClass()

        testdata = os.path.join(testdata_root,
            'Data_seperate_files_20170810_20180809')
        metadata_path = os.path.join(testdata, 'python_metadata')

        cleanup(metadata_path)
        ds = ISMN_Interface(testdata, network=[])
        assert ds.networks == OrderedDict()
        cls.testdata = testdata

    def setUp(self) -> None:
        self.ds = ISMN_Interface(self.testdata, network=['COSMOS'])

    def tearDown(self) -> None:
        self.ds.close_files()
        logging.shutdown()
            
    def test_list(self):
        assert len(self.ds.list_networks()) == 1
        assert len(self.ds.list_stations()) == len(self.ds.list_stations('COSMOS')) == 2
        assert len(self.ds.list_sensors()) == 2
        assert len(self.ds.list_sensors(station='Barrow-ARM')) == 1

    def test_network_for_station(self):
        assert self.ds.network_for_station('Barrow-ARM') == 'COSMOS'
        assert self.ds.network_for_station('ARM-1') == 'COSMOS'

    def test_stations_that_measure(self):
        for s in self.ds.stations_that_measure('soil_moisture'):
            assert s.name in ['ARM-1', 'Barrow-ARM']

        for s in self.ds.stations_that_measure('nonexisting'):
            raise AssertionError("Found var that doesnt exist")

    def test_get_dataset_ids(self):
        ids = self.ds.get_dataset_ids('soil_moisture', max_depth=100, groupby='network')
        assert list(ids.keys()) == ['COSMOS']
        assert ids['COSMOS'] == [0,1]

        ids = self.ds.get_dataset_ids('soil_moisture', max_depth=0.19)
        assert ids == [0]

        ids = self.ds.get_dataset_ids('soil_moisture', max_depth=99,
                                      filter_meta_dict={'lc_2010': 210,
                                                        'network': 'COSMOS',
                                                        'station': 'Barrow-ARM'})
        assert ids == [1]

        ids = self.ds.get_dataset_ids('novar')
        assert len(ids) == 0

        ids = self.ds.get_dataset_ids('soil_moisture', 0., 0.19) # should get 1
        assert len(ids) == 1

        ids = self.ds.get_dataset_ids('soil_moisture', 0., 1.) # should get 2
        assert len(ids) == 2

        ids = self.ds.get_dataset_ids('soil_moisture', 0., 1.,
                                      filter_meta_dict={'lc_2010': 210}) # should get 1
        assert len(ids) == 1

        ids = self.ds.get_dataset_ids('nonexisting') # should get 0
        assert len(ids) == 0

    def test_read_ts(self):
        data1 = self.ds.read(0)
        assert not data1.empty

        data2 = self.ds.read_ts(1)
        assert not data2.empty

        assert len(data1.index) != len(data2.index) # make sure they are not same

    def test_find_nearest_station(self):
        should_lon, should_lat = -156.62870, 71.32980

        station = self.ds.find_nearest_station(should_lon, should_lat)

        assert station.lon == should_lon
        assert station.lat == should_lat

    def test_plot_station_locations(self):
        with TemporaryDirectory() as out_dir:
            outpath = os.path.join(out_dir, 'plot.png')
            self.ds.plot_station_locations('soil_moisture', markersize=5,
                                           filename=outpath)
            assert len(os.listdir(out_dir)) == 1

    def test_get_min_max_obs_timestamps(self):
        tmin, tmax = self.ds.get_min_max_obs_timestamps('soil_moisture', max_depth=0.19)
        assert tmin == datetime(2017, 8, 10, 0)
        assert tmax == datetime(2018, 8, 9, 23)

    def test_get_min_max_obs_timestamps_for_station(self):
        station = self.ds.collection.networks['COSMOS'].stations['ARM-1']
        tmin, tmax = station.get_min_max_obs_timestamp('soil_moisture', 0, 0.19)
        assert tmin == datetime(2017, 8, 10, 0)
        assert tmax == datetime(2018, 8, 9, 23)

    def test_get_static_var_val(self):
        vals = self.ds.get_static_var_vals('soil_moisture', max_depth=0.19)
        assert vals == {130: 'Grassland'}

        vals = self.ds.get_landcover_types('soil_moisture', max_depth=100)
        assert len(vals) == 2
        assert vals[130] == 'Grassland'
        assert vals[210] == 'Water'
        self.ds.print_landcover_dict()

        vals = self.ds.get_climate_types('soil_moisture', max_depth=100,
                                         climate='climate_KG')
        assert len(vals) == 2
        assert vals['ET'] == 'Polar Tundra'
        assert vals['Cfa'] == 'Temperate Without Dry Season, Hot Summer'
        self.ds.print_climate_dict()

    def test_get_var(self):
        vars = self.ds.get_variables()
        assert vars == ['soil_moisture']


    def test_get_sensors(self):
        i = 0
        for nw, station in self.ds.collection.iter_stations(
                filter_meta_dict={'network': 'COSMOS'}):
            for se in station.iter_sensors():
                data = se.read_data()
                # check if the networks is COSMOS or station in [ARM, Barrow-ARM]
                assert not data.empty
                # check something for that one station
                i += 1
        assert i == 2

        i = 0
        for se in self.ds.networks['COSMOS'].stations['Barrow-ARM'].iter_sensors():
            data = se.read_data()
            assert not data.empty
            # check something for that one station
            i += 1
        assert i == 1

        i = 0
        for net, stat, sens in self.ds.collection.iter_sensors(
                depth=Depth(0,1),
                filter_meta_dict={'station': ['Barrow-ARM', 'ARM-1']}):
            data = sens.read_data()
            assert not data.empty
            i +=1
        assert i == 2


        for nw, station in self.ds.collection.iter_stations():
            for se in station.iter_sensors(variable='nonexisting'):
                raise ValueError("Found sensor, although none should exist")

    def test_get_nearest_station(self):
        should_lon, should_lat = -156.62870, 71.32980

        station, dist = self.ds.collection.get_nearest_station(should_lon, should_lat)
        assert dist == 0
        assert station.lon == should_lon
        assert station.lat == should_lat
        gpi, dist = self.ds.collection.grid.find_nearest_gpi(int(should_lon),int(should_lat))
        assert dist != 0
        for net in self.ds.collection.iter_networks():
            if station.name in net.stations.keys():
                assert net.stations[station.name].lon == should_lon
                assert net.stations[station.name].lat == should_lat

        station, dist = self.ds.find_nearest_station(0, 0, return_distance=True,
                                                     max_dist=100)
        assert station == dist == None