def test_closest_daterange():
dateranges = [
"2012-01-01-00:00:00+00:00_2014-01-01-00:00:00+00:00",
"2014-01-02-00:00:00+00:00_2015-01-01-00:00:00+00:00",
"2016-01-02-00:00:00+00:00_2017-01-01-00:00:00+00:00",
"2019-01-02-00:00:00+00:00_2021-01-01-00:00:00+00:00",
]
to_comp = create_daterange_str(start="2011-01-09", end="2011-09-09")
closest = closest_daterange(to_compare=to_comp, dateranges=dateranges)
assert closest == "2012-01-01-00:00:00+00:00_2014-01-01-00:00:00+00:00"
to_comp = create_daterange_str(start="2015-01-09", end="2015-10-09")
closest = closest_daterange(to_compare=to_comp, dateranges=dateranges)
assert closest == "2014-01-02-00:00:00+00:00_2015-01-01-00:00:00+00:00"
to_comp = create_daterange_str(start="2021-01-09", end="2022-10-09")
closest = closest_daterange(to_compare=to_comp, dateranges=dateranges)
assert closest == "2019-01-02-00:00:00+00:00_2021-01-01-00:00:00+00:00"
with pytest.raises(ValueError):
to_comp = create_daterange_str(start="2019-01-09", end="2021-10-09")
closest = closest_daterange(to_compare=to_comp, dateranges=dateranges)
def test_daterange_contains():
container = create_daterange_str(start="2004-05-09", end="2013-01-01")
contained = create_daterange_str(start="2007-05-09", end="2010-01-01")
res = daterange_contains(container=container, contained=contained)
assert res
container = create_daterange_str(start="2009-05-09", end="2013-01-01")
contained = create_daterange_str(start="2007-05-09", end="2010-01-01")
res = daterange_contains(container=container, contained=contained)
assert not res
def test_create_daterange_str():
start = Timestamp("2019-1-1-15:33:12", tz="UTC")
end = Timestamp("2020-1-1-18:55:12", tz="UTC")
s = create_daterange_str(start=start, end=end)
assert s == "2019-01-01-15:33:12+00:00_2020-01-01-18:55:12+00:00"
def get_rank(self: T, uuid: str, start_date: Timestamp, end_date: Timestamp) -> Dict:
"""Get the rank for the given Datasource for a given date range
Args:
uuid: UUID of Datasource
start_date: Start date
end_date: End date
Returns:
dict: Dictionary of rank and daterange covered by that rank
"""
from openghg.util import daterange_overlap, create_daterange_str
from collections import defaultdict
if uuid not in self._rank_data:
return {}
search_daterange = create_daterange_str(start=start_date, end=end_date)
rank_data = self._rank_data[uuid]
ranked = defaultdict(list)
# Check if this Datasource is ranked for the dates passed
for daterange, rank in rank_data.items():
if daterange_overlap(daterange_a=search_daterange, daterange_b=daterange):
ranked[rank].append(daterange)
return ranked
def test_in_daterange(data):
metadata = data["ch4"]["metadata"]
data = data["ch4"]["data"]
d = Datasource()
d._uuid = "test-id-123"
d.add_data(metadata=metadata, data=data, data_type="timeseries")
d.save()
expected_keys = [
"data/uuid/test-id-123/v1/2014-01-30-11:12:30+00:00_2014-11-30-11:23:30+00:00",
"data/uuid/test-id-123/v1/2015-01-30-11:12:30+00:00_2015-11-30-11:23:30+00:00",
"data/uuid/test-id-123/v1/2016-04-02-06:52:30+00:00_2016-11-02-12:54:30+00:00",
"data/uuid/test-id-123/v1/2017-02-18-06:36:30+00:00_2017-12-18-15:41:30+00:00",
"data/uuid/test-id-123/v1/2018-02-18-15:42:30+00:00_2018-12-18-15:42:30+00:00",
"data/uuid/test-id-123/v1/2019-02-03-17:38:30+00:00_2019-12-09-10:47:30+00:00",
"data/uuid/test-id-123/v1/2020-02-01-18:08:30+00:00_2020-12-01-22:31:30+00:00",
]
assert d.data_keys() == expected_keys
start = pd.Timestamp("2014-1-1")
end = pd.Timestamp("2014-2-1")
daterange = create_daterange_str(start=start, end=end)
dated_keys = d.keys_in_daterange_str(daterange=daterange)
assert dated_keys[0].split(
"/")[-1] == "2014-01-30-11:12:30+00:00_2014-11-30-11:23:30+00:00"
def test_daterange_overlap():
start_date_a = "2001-01-01"
end_date_a = "2001-06-30"
start_date_b = "2001-02-01"
end_date_b = "2001-09-01"
daterange_a = create_daterange_str(start=start_date_a, end=end_date_a)
daterange_b = create_daterange_str(start=start_date_b, end=end_date_b)
assert daterange_overlap(daterange_a=daterange_a,
daterange_b=daterange_b) is True
start_date_b = "2001-07-01"
end_date_b = "2001-11-01"
daterange_b = create_daterange_str(start=start_date_b, end=end_date_b)
assert daterange_overlap(daterange_a=daterange_a,
daterange_b=daterange_b) is False
def test_split_encompassed_daterange():
container = create_daterange_str(start="2004-05-09", end="2013-01-01")
contained = create_daterange_str(start="2007-05-09", end="2010-01-01")
result = split_encompassed_daterange(container=container,
contained=contained)
expected = {
"container_start":
"2004-05-09-00:00:00+00:00_2007-05-08-23:59:59+00:00",
"contained": "2007-05-09-00:00:00+00:00_2009-12-31-23:59:59+00:00",
"container_end": "2010-01-01-00:00:01+00:00_2013-01-01-00:00:00+00:00",
}
assert result == expected
container = create_daterange_str(start="1995-05-09", end="1997-01-01")
with pytest.raises(ValueError):
split_encompassed_daterange(container=container, contained=contained)
def test_set_rank():
o = ObsSurface.load()
o._rank_data.clear()
test_uid = "test-uid-123"
daterange_str = create_daterange_str(start="2001-01-01", end="2005-01-01")
o.set_rank(uuid=test_uid, rank=1, date_range=daterange_str)
assert o._rank_data == {
"test-uid-123": {
"2001-01-01-00:00:00+00:00_2005-01-01-00:00:00+00:00": 1
}
}
daterange_str = create_daterange_str(start="2007-01-01", end="2009-01-01")
o.set_rank(uuid=test_uid, rank=1, date_range=daterange_str)
assert o._rank_data["test-uid-123"] == {
"2001-01-01-00:00:00+00:00_2005-01-01-00:00:00+00:00": 1,
"2007-01-01-00:00:00+00:00_2009-01-01-00:00:00+00:00": 1,
}
# Make sure we can't set another rank for the same daterange
with pytest.raises(ValueError):
o.set_rank(uuid=test_uid, rank=2, date_range=daterange_str)
daterange_str = create_daterange_str(start="2008-01-01", end="2009-01-01")
with pytest.raises(ValueError):
o.set_rank(uuid=test_uid, rank=3, date_range=daterange_str)
daterange_str = create_daterange_str(start="2007-01-01", end="2015-01-01")
o.set_rank(uuid=test_uid, rank=1, date_range=daterange_str)
assert o._rank_data["test-uid-123"] == {
"2001-01-01-00:00:00+00:00_2005-01-01-00:00:00+00:00": 1,
"2007-01-01-00:00:00+00:00_2015-01-01-00:00:00+00:00": 1,
}
def test_set_rank_overwrite():
o = ObsSurface.load()
o._rank_data.clear()
test_uid = "test-uid-123"
daterange_str = create_daterange_str(start="2007-01-01", end="2015-01-01")
o.set_rank(uuid=test_uid, rank=1, date_range=daterange_str)
assert o._rank_data["test-uid-123"] == {
"2007-01-01-00:00:00+00:00_2015-01-01-00:00:00+00:00": 1
}
daterange_str = create_daterange_str(start="2008-01-01", end="2009-01-01")
o.set_rank(uuid=test_uid, rank=2, date_range=daterange_str, overwrite=True)
expected_ranking = {
"2007-01-01-00:00:00+00:00_2007-12-31-23:59:59+00:00": 1,
"2008-01-01-00:00:00+00:00_2008-12-31-23:59:59+00:00": 2,
"2009-01-01-00:00:01+00:00_2015-01-01-00:00:00+00:00": 1,
}
assert o._rank_data["test-uid-123"] == expected_ranking
daterange_str = create_daterange_str(start="1994-01-01", end="2023-01-01")
o.set_rank(uuid=test_uid, rank=2, date_range=daterange_str, overwrite=True)
assert o._rank_data["test-uid-123"] == {
"1994-01-01-00:00:00+00:00_2023-01-01-00:00:00+00:00": 2
}
o._rank_data.clear()
daterange_str = create_daterange_str(start="2001-01-01", end="2021-01-01")
o.set_rank(uuid=test_uid, rank=1, date_range=daterange_str)
assert o._rank_data["test-uid-123"] == {
"2001-01-01-00:00:00+00:00_2021-01-01-00:00:00+00:00": 1
}
daterange_str = create_daterange_str(start="2007-01-01", end="2009-01-01")
o.set_rank(uuid=test_uid, rank=2, date_range=daterange_str, overwrite=True)
daterange_str = create_daterange_str(start="2015-01-01", end="2016-01-01")
o.set_rank(uuid=test_uid, rank=2, date_range=daterange_str, overwrite=True)
expected = {
"2001-01-01-00:00:00+00:00_2006-12-31-23:59:59+00:00": 1,
"2007-01-01-00:00:00+00:00_2008-12-31-23:59:59+00:00": 2,
"2009-01-01-00:00:01+00:00_2014-12-31-23:59:59+00:00": 1,
"2015-01-01-00:00:00+00:00_2015-12-31-23:59:59+00:00": 2,
"2016-01-01-00:00:01+00:00_2021-01-01-00:00:00+00:00": 1,
}
assert o._rank_data["test-uid-123"] == expected
def _set_rank_local(
self,
inlet: str,
rank: Union[int, str],
start_date: str,
end_date: str,
overwrite: bool = False,
) -> None:
obs = ObsSurface.load()
inlet = inlet.lower()
uuid = self._key_lookup[inlet]
daterange = create_daterange_str(start=start_date, end=end_date)
obs.set_rank(uuid=uuid, rank=rank, date_range=daterange, overwrite=overwrite)
self._needs_update = True
def test_trim_daterange():
trim_me = create_daterange_str(start="2011-01-01", end="2021-09-01")
overlapping = create_daterange_str(start="2004-05-09", end="2013-01-01")
trimmed = trim_daterange(to_trim=trim_me, overlapping=overlapping)
assert trimmed == "2013-01-01-00:00:01+00:00_2021-09-01-00:00:00+00:00"
trim_me = create_daterange_str(start="2001-01-01", end="2005-09-01")
overlapping = create_daterange_str(start="2004-05-09", end="2013-01-01")
trimmed = trim_daterange(to_trim=trim_me, overlapping=overlapping)
assert trimmed == "2001-01-01-00:00:00+00:00_2004-05-08-23:59:59+00:00"
trim_me = create_daterange_str(start="2000-01-01", end="2005-09-01")
overlapping = create_daterange_str(start="2007-05-09", end="2013-01-01")
with pytest.raises(ValueError):
trimmed = trim_daterange(to_trim=trim_me, overlapping=overlapping)