def test_metadata(self):
""" test reading the loaded metadata for a file (incl. static meta) """
assert self.file.check_metadata()
assert not self.file.check_metadata(
filter_meta_dict={'network': 'WRONGNAME'})
flag = self.file.check_metadata(self.variable,
allowed_depth=Depth(0, 0.21),
filter_meta_dict={
'longitude': self.longitude,
'variable': self.variable
})
assert flag == True
flag = self.file.check_metadata('nonexistingvar', Depth(0, 0.1))
assert flag == False
flag = self.file.check_metadata(self.variable, Depth(98., 99.))
assert flag == False
assert self.file.metadata['station'].val == self.station_name
assert self.file.metadata['network'].val == self.network_name
assert self.file.metadata['instrument'].depth.start == self.depth_from
assert self.file.metadata['instrument'].depth.end == self.depth_to
assert self.file.metadata['instrument'].val == self.instrument
assert self.file.metadata['longitude'].val == self.longitude
assert self.file.metadata['latitude'].val == self.latitude
assert self.file.metadata['variable'].val == self.variable
self.file.check_metadata(
filter_meta_dict={
'timerange_from': datetime(2017, 8, 10, 0),
'timerange_to': datetime(2018, 8, 9, 8)
})
def setUp(self):
"""
Setup test data.
"""
self.d = Depth(0, 0.05)
assert str(self.d) == "0.0 to 0.05 [m]"
assert tuple(self.d) == (0.0, 0.05)
def test_perc_overlap(self):
other = Depth(0.05, 0.1)
assert other.across0 == False
assert self.d.overlap(other) == True
assert self.d.perc_overlap(other) == 0.0
other = Depth(0.03, 0.05)
assert other.across0 == False
assert self.d.overlap(other) == True
assert self.d.perc_overlap(other) == round(0.02 / 0.05, 7)
other = Depth(0, 0.05)
assert other.across0 == False
assert self.d.overlap(other) == True
assert self.d.perc_overlap(other) == 1.0
other = Depth(-0.01, -0.05)
assert other.across0 == False
assert self.d.overlap(other) == False
assert self.d.perc_overlap(other) == -1
other = Depth(-0.01, 0.01)
assert other.across0 == True
assert self.d.overlap(other) == True
assert self.d.perc_overlap(other) == round(0.01 / 0.06, 7)
def test_invalid(self):
try:
Depth(0.5, 0.1)
raise AssertionError
except DepthError:
pass
try:
Depth(-0.5, -0.1)
raise AssertionError
except DepthError:
pass
def test_neg_depth(self):
other = Depth(0.0, -0.05)
assert str(other) == "0.0 to -0.05 [m]"
assert self.d != other
assert other.encloses(self.d) == False
assert other.enclosed(self.d) == False
assert self.d.across0 == other.across0 == False
assert self.d.is_profile == other.is_profile == True
assert other.perc_overlap(self.d) == 0.0
def test_get_sensors(self):
i = 0
for nw, station in self.ds.collection.iter_stations(
filter_meta_dict={"network": "COSMOS"}):
for se in station.iter_sensors():
data = se.read_data()
# check if the networks is COSMOS or station in [ARM, Barrow-ARM]
assert not data.empty
# check something for that one station
i += 1
assert i == 2
i = 0
for se in self.ds.networks["COSMOS"].stations[
"Barrow-ARM"].iter_sensors():
data = se.read_data()
assert not data.empty
# check something for that one station
i += 1
assert i == 1
i = 0
for net, stat, sens in self.ds.collection.iter_sensors(
depth=Depth(0, 1),
filter_meta_dict={"station": ["Barrow-ARM", "ARM-1"]},
):
data = sens.read_data()
assert not data.empty
i += 1
assert i == 2
for nw, station in self.ds.collection.iter_stations():
for se in station.iter_sensors(variable="nonexisting"):
raise ValueError("Found sensor, although none should exist")
def test_get_sensors(self):
i = 0
for nw, station in self.ds.collection.iter_stations(
filter_meta_dict={'network': 'COSMOS'}):
for se in station.iter_sensors():
data = se.read_data()
# check if the networks is COSMOS or station in [ARM, Barrow-ARM]
assert not data.empty
# check something for that one station
i += 1
assert i == 2
i = 0
for se in self.ds.networks['COSMOS'].stations['Barrow-ARM'].iter_sensors():
data = se.read_data()
assert not data.empty
# check something for that one station
i += 1
assert i == 1
i = 0
for net, stat, sens in self.ds.collection.iter_sensors(
depth=Depth(0,1),
filter_meta_dict={'station': ['Barrow-ARM', 'ARM-1']}):
data = sens.read_data()
assert not data.empty
i +=1
assert i == 2
for nw, station in self.ds.collection.iter_stations():
for se in station.iter_sensors(variable='nonexisting'):
raise ValueError("Found sensor, although none should exist")
def test_metadata(self):
"""test reading the loaded metadata for a file (incl. static meta)"""
assert self.file.check_metadata()
assert not self.file.check_metadata(filter_meta_dict={"network": "WRONGNAME"})
flag = self.file.check_metadata(
self.variable,
allowed_depth=Depth(0, 0.21),
filter_meta_dict={
"longitude": self.longitude,
"variable": self.variable,
},
)
assert flag == True
flag = self.file.check_metadata("nonexistingvar", Depth(0, 0.1))
assert flag == False
flag = self.file.check_metadata(self.variable, Depth(98.0, 99.0))
assert flag == False
assert self.file.metadata["station"].val == self.station_name
assert self.file.metadata["network"].val == self.network_name
assert self.file.metadata["instrument"].depth.start == self.depth_from
assert self.file.metadata["instrument"].depth.end == self.depth_to
assert self.file.metadata["instrument"].val == self.instrument
assert self.file.metadata["longitude"].val == self.longitude
assert self.file.metadata["latitude"].val == self.latitude
assert self.file.metadata["variable"].val == self.variable
self.file.check_metadata(
filter_meta_dict={
"timerange_from": datetime(2017, 8, 10, 0),
"timerange_to": datetime(2018, 8, 9, 8),
}
)
def get_min_max_obs_timestamp(self,
variable="soil moisture",
min_depth=None,
max_depth=None):
"""
Goes through the sensors associated with this station
and checks the metadata to get and approximate time coverage of the station.
This is just an overview. If holes have to be detected the
complete file must be read.
Parameters
----------
variable: str, optional (default: 'soil_moisture')
name of the variable, only sensors measuring that variable are used.
min_depth : float, optional (default: None)
depth_from of variable has to be >= min_depth in order to be
included.
max_depth : float, optional (default: None)
depth_to of variable has to be <= max_depth in order to be
included.
Returns
-------
start_date: datetime.datetime
Earliest date observed by any sensor at the station after filtering
for the passed requirements.
end_date: datetime.datetime
Latest date observed by any sensor at the station after filtering
for the passed requirements.
"""
depth = Depth(
-np.inf if min_depth is None else min_depth,
np.inf if max_depth is None else max_depth,
)
min_from, max_to = None, None
for sensor in self.iter_sensors(variable=variable, depth=depth):
time_from = sensor.metadata["timerange_from"].val
time_to = sensor.metadata["timerange_to"].val
if (min_from is None) or (time_from < min_from):
min_from = time_from
if (max_to is None) or (time_to > max_to):
max_to = time_to
min_from = min_from.to_pydatetime() if min_from is not None else None
max_to = max_to.to_pydatetime() if max_to is not None else None
return min_from, max_to
def test_enclose(self):
"""
Test if other depth encloses depth.
"""
other = Depth(0, 0.05)
assert self.d.encloses(other)
assert other.enclosed(self.d)
other = Depth(0, 0.1)
assert not self.d.encloses(other)
assert not other.enclosed(self.d)
assert other.across0 == False
other = Depth(-0.1, -0.2)
assert not self.d.encloses(other)
assert not self.d.enclosed(other)
assert other.across0 == False
def get_sensors(self, variable, depth_from, depth_to):
"""
get the sensors at which the variable was measured at the
given depth
Parameters
----------
variable : string
variable abbreviation
depth_from : float
shallower depth of layer the variable was measured at
depth_to : float
deeper depth of layer the variable was measured at
Returns
-------
sensors : numpy.array
array of sensors found for the given combination of variable and depths
"""
return np.array([
s for s in self.iter_sensors(variable=variable,
depth=Depth(depth_from, depth_to))
])
def _read_station_dir(
root: Union[IsmnRoot, Path, str],
stat_dir: Union[Path, str],
temp_root: Path,
) -> (dict, list):
"""
Parallelizable function to read metadata for files in station dir
"""
infos = []
if not isinstance(root, IsmnRoot):
proc_root = True
root = IsmnRoot(root)
else:
proc_root = False
csv = root.find_files(stat_dir, "*.csv")
try:
if len(csv) == 0:
raise IsmnFileError(
"Expected 1 csv file for station, found 0. "
"Use empty static metadata."
)
else:
if len(csv) > 1:
infos.append(
f"Expected 1 csv file for station, found {len(csv)}. "
f"Use first file in dir."
)
static_meta_file = StaticMetaFile(
root, csv[0], load_metadata=True, temp_root=temp_root
)
station_meta = static_meta_file.metadata
except IsmnFileError as e:
infos.append(f"Error loading static meta for station: {e}")
station_meta = MetaData([MetaVar(k, v) for k, v in CSV_META_TEMPLATE.items()])
data_files = root.find_files(stat_dir, "*.stm")
filelist = []
for file_path in data_files:
try:
f = DataFile(root, file_path, temp_root=temp_root)
except IOError as e:
infos.append(f"Error loading ismn file: {e}")
continue
f.metadata.merge(station_meta, inplace=True)
f.metadata = f.metadata.best_meta_for_depth(
Depth(
f.metadata["instrument"].depth.start,
f.metadata["instrument"].depth.end,
)
)
network = f.metadata["network"].val
station = f.metadata["station"].val
filelist.append((network, station, f))
infos.append(f"Processed file {file_path}")
if proc_root:
root.close()
return filelist, infos
def eval(self,
variable=None,
depth=None,
filter_meta_dict=None,
check_only_sensor_depth_from=False):
"""
Evaluate whether the sensor complies with the passed metadata
requirements.
Parameters
----------
variable : str, optional (default: None)
Check if the variable name matches, e.g. soil_moisture
depth : Depth or list or tuple, optional (default: None)
Check if the passed depth encloses the sensor depth.
A list/tuple must contain 2 values where the first is the depth start
and the second is the end. Start must be closer to 0 than end (or equal).
A negative depth range is above the surface.
filter_meta_dict : dict, optional (default: None)
Additional metadata keys and values for which the file list is filtered
e.g. {'lc_2010': [10, 130]} or
{'climate_KG': 'Dwa', 'lc_2010': [10, 130] }
to filter for a multiple landcover classes and a climate class.
check_only_sensor_depth_from : bool, optional (default: False)
Ignores the sensors depth_to value and only checks if depth_from of
the sensor is in the passed depth (e.g. for cosmic ray probes).
Returns
-------
flag : bool
Indicates weather metadata for this Sensor matches with the passed
requirements.
"""
if isinstance(depth, (list, tuple)):
depth = Depth(depth[0], depth[1])
if depth is None:
depth = Depth(-np.inf, np.inf)
flag = False
if check_only_sensor_depth_from:
d = Depth(self.depth.start, self.depth.start)
else:
d = self.depth
if (variable in [None, self.variable]) and depth.encloses(d):
flag = True
if flag and filter_meta_dict:
if self.filehandler is None:
warnings.warn("No filehandle found, can't filter by metadata.")
else:
# checks also if the metadata in file matches
flag = self.filehandler.check_metadata(
variable,
allowed_depth=depth,
filter_meta_dict=filter_meta_dict,
check_only_sensor_depth_from=check_only_sensor_depth_from)
return flag
class DepthTest(unittest.TestCase):
def setUp(self):
"""
Setup test data.
"""
self.d = Depth(0, 0.05)
assert str(self.d) == "0.0 to 0.05 [m]"
assert tuple(self.d) == (0.0, 0.05)
def test_neg_depth(self):
other = Depth(0.0, -0.05)
assert str(other) == "0.0 to -0.05 [m]"
assert self.d != other
assert other.encloses(self.d) == False
assert other.enclosed(self.d) == False
assert self.d.across0 == other.across0 == False
assert self.d.is_profile == other.is_profile == True
assert other.perc_overlap(self.d) == 0.0
def test_attributes(self):
"""
Test depth attributes.
"""
assert self.d.start == 0
assert self.d.end == 0.05
def test_is_profile(self):
"""
Test if depth represents a profile.
"""
assert self.d.is_profile
def test_equal(self):
"""
Test depth equality.
"""
other = Depth(0, 0.05)
assert self.d == other
def test_invalid(self):
try:
Depth(0.5, 0.1)
raise AssertionError
except DepthError:
pass
try:
Depth(-0.5, -0.1)
raise AssertionError
except DepthError:
pass
def test_enclose(self):
"""
Test if other depth encloses depth.
"""
other = Depth(0, 0.05)
assert self.d.encloses(other)
assert other.enclosed(self.d)
other = Depth(0, 0.1)
assert not self.d.encloses(other)
assert not other.enclosed(self.d)
assert other.across0 == False
other = Depth(-0.1, -0.2)
assert not self.d.encloses(other)
assert not self.d.enclosed(other)
assert other.across0 == False
def test_perc_overlap(self):
other = Depth(0.05, 0.1)
assert other.across0 == False
assert self.d.overlap(other) == True
assert self.d.perc_overlap(other) == 0.0
other = Depth(0.03, 0.05)
assert other.across0 == False
assert self.d.overlap(other) == True
assert self.d.perc_overlap(other) == round(0.02 / 0.05, 7)
other = Depth(0, 0.05)
assert other.across0 == False
assert self.d.overlap(other) == True
assert self.d.perc_overlap(other) == 1.0
other = Depth(-0.01, -0.05)
assert other.across0 == False
assert self.d.overlap(other) == False
assert self.d.perc_overlap(other) == -1
other = Depth(-0.01, 0.01)
assert other.across0 == True
assert self.d.overlap(other) == True
assert self.d.perc_overlap(other) == round(0.01 / 0.06, 7)
def test_equal(self):
"""
Test depth equality.
"""
other = Depth(0, 0.05)
assert self.d == other