def test_timestamp(self):
d = Pendulum(1970, 1, 1, 0, 0, 0)
self.assertEqual(0, d.timestamp())
self.assertEqual(0, d.timestamp)
self.assertEqual(60.123456,
d.add(minutes=1, microseconds=123456).timestamp())
self.assertEqual(60, d.add(minutes=1, microseconds=123456).timestamp)
def test_intersect_excluded(self):
start = Pendulum(2016, 8, 7)
end = start.add(weeks=1)
p1 = Period(start, end)
intersection = p1.intersect(
Period(start.add(days=-2), start.add(days=-1)))
self.assertIsNone(intersection)
def test_intersect_included(self):
start = Pendulum(2016, 8, 7)
end = start.add(weeks=1)
p1 = Period(start, end)
intersection = p1.intersect(
Period(start.add(days=2), start.add(days=4)))
self.assertPendulum(intersection.start, 2016, 8, 9)
self.assertPendulum(intersection.end, 2016, 8, 11)
def test_intersect_multiple(self):
start = Pendulum(2016, 8, 7)
end = start.add(weeks=1)
p1 = Period(start, end)
intersection = p1.intersect(
Period(start.add(days=-2), start.add(days=2)),
Period(start.add(days=1), start.add(days=2)))
self.assertPendulum(intersection.start, 2016, 8, 8)
self.assertPendulum(intersection.end, 2016, 8, 9)
def test_day_of_year(self):
f = AlternativeFormatter()
d = Pendulum(2016, 8, 28)
self.assertEqual('241', f.format(d, 'DDDD'))
self.assertEqual('241', f.format(d, 'DDD'))
self.assertEqual('001', f.format(d.start_of('year'), 'DDDD'))
self.assertEqual('1', f.format(d.start_of('year'), 'DDD'))
self.assertEqual('241st', f.format(d, 'DDDo'))
self.assertEqual('244th', f.format(d.add(days=3), 'DDDo'))
self.assertEqual('241e', f.format(d, 'DDDo', locale='fr'))
self.assertEqual('244e', f.format(d.add(days=3), 'DDDo', locale='fr'))
def test_floor_divide(self):
dt1 = Pendulum(2016, 8, 7, 12, 34, 56)
dt2 = dt1.add(days=2, seconds=34)
it = Period(dt1, dt2)
mul = it // 2
self.assertIsInstanceOfInterval(mul)
self.assertInterval(mul, 0, 1, 0, 0, 17)
dt1 = Pendulum(2016, 8, 7, 12, 34, 56)
dt2 = dt1.add(days=2, seconds=35)
it = Period(dt1, dt2)
mul = it // 3
self.assertIsInstanceOfInterval(mul)
self.assertInterval(mul, 0, 0, 16, 0, 11)
def test_multiply(self):
dt1 = Pendulum(2016, 8, 7, 12, 34, 56)
dt2 = dt1.add(days=6, seconds=34)
it = Period(dt1, dt2)
mul = it * 2
self.assertIsInstanceOfInterval(mul)
self.assertInterval(mul, 1, 5, 0, 1, 8)
dt1 = Pendulum(2016, 8, 7, 12, 34, 56)
dt2 = dt1.add(days=6, seconds=34)
it = Period(dt1, dt2)
mul = it * 2
self.assertIsInstanceOfInterval(mul)
self.assertInterval(mul, 1, 5, 0, 1, 8)
def test_time_as_datetime(self, mock_now):
set_time = time(12, 00)
# This is a monday
now = Pendulum(year=2016,
month=9,
day=19,
hour=set_time.hour,
minute=set_time.minute,
tzinfo=None)
for current_day in range(7):
now = now.add(days=1)
mock_now.return_value = now
for weekday_config in WEEKDAYS:
day_in_week = weekday_config[0]
period = OpeningPeriod(day=day_in_week,
time=set_time,
duration=time(1, 00),
store=self.store)
as_datetime = period.weekday_as_datetime(weekday=period.day,
time=period.time,
store=period.store)
if day_in_week == now.isoweekday():
self.assertEqual(as_datetime, now)
else:
self.assertGreater(as_datetime, now)
def get_year_month_to_period_map(self, start_year, end_year):
"""
generate mapping (year, month) --> period
:param start_year:
:param end_year:
:return:
"""
res = {}
for y in range(start_year, end_year + 1):
start = Pendulum(y, self.start_month, 1)
end = start.add(months=self.nmonths).subtract(microseconds=1)
if end.year > end_year:
continue
print(start, end)
p = Period(start, end)
for s in p.range("months"):
res[(s.year, s.month)] = p
return res
async def test_retrieve_device(running_backend, alice_remote_devices_manager,
bob):
remote_devices_manager = alice_remote_devices_manager
d1 = Pendulum(2000, 1, 1)
with freeze_time(d1):
# Offline with no cache
with pytest.raises(RemoteDevicesManagerBackendOfflineError):
with running_backend.offline():
await remote_devices_manager.get_device(bob.device_id)
# Online
device = await remote_devices_manager.get_device(bob.device_id)
assert device.device_id == bob.device_id
assert device.verify_key == bob.verify_key
# Offline with cache
with running_backend.offline():
device2 = await remote_devices_manager.get_device(bob.device_id)
assert device2 is device
d2 = d1.add(remote_devices_manager.cache_validity + 1)
with freeze_time(d2):
# Offline with cache expired
with pytest.raises(RemoteDevicesManagerBackendOfflineError):
with running_backend.offline():
await remote_devices_manager.get_device(bob.device_id)
# Online with cache expired
device = await remote_devices_manager.get_device(bob.device_id)
assert device.device_id == bob.device_id
assert device.verify_key == bob.verify_key
def get_mean_number_of_hles_days(self, start_year: int, end_year: int, season_to_months: dict, hles_vname: str):
result = defaultdict(dict)
cache_dir = Path(self.base_folder) / "cache"
cache_dir.mkdir(exist_ok=True)
seasons_str = "-".join(season_to_months)
cache_file = cache_dir / f"get_mean_number_of_hles_days_{start_year}-{end_year}_m{seasons_str}_{hles_vname}.bin"
if cache_file.exists():
return pickle.load(cache_file.open("rb"))
for season, months in season_to_months.items():
for y in range(start_year, end_year + 1):
d1 = Pendulum(y, months[0], 1)
d2 = d1.add(months=len(months)).subtract(seconds=1)
if d2.year > end_year:
continue
current_period = Period(d1, d2)
print("calculating mean for [{}, {}]".format(current_period.start, current_period.end))
data = self.read_data_for_period(current_period, hles_vname)
# calculate number of hles days
data_daily = data.resample(t="1D", keep_attrs=True).mean(dim="t")
result[season][y] = (data_daily.values >= 0.1).sum(axis=0)
pickle.dump(result, cache_file.open("wb"))
return result
async def test_retrieve_user(running_backend, alice_remote_devices_manager,
bob):
remote_devices_manager = alice_remote_devices_manager
d1 = Pendulum(2000, 1, 1)
with freeze_time(d1):
# Offline with no cache
with pytest.raises(RemoteDevicesManagerBackendOfflineError):
with running_backend.offline():
await remote_devices_manager.get_user(bob.user_id)
# Online
user, revoked_user = await remote_devices_manager.get_user(bob.user_id)
assert user.user_id == bob.user_id
assert user.public_key == bob.public_key
assert revoked_user is None
# Offline with cache
with running_backend.offline():
user2, revoked_user2 = await remote_devices_manager.get_user(
bob.user_id)
assert user2 is user
assert revoked_user2 is None
d2 = d1.add(remote_devices_manager.cache_validity + 1)
with freeze_time(d2):
# Offline with cache expired
with pytest.raises(RemoteDevicesManagerBackendOfflineError):
with running_backend.offline():
await remote_devices_manager.get_user(bob.user_id)
# Online with cache expired
user, revoked_user = await remote_devices_manager.get_user(bob.user_id)
assert user.user_id == bob.user_id
assert user.public_key == bob.public_key
assert revoked_user is None
def get_seasonal_maxima(self, start_year: int, end_year: int, season_to_months: dict, varname_internal: str):
"""
returns a dictionary {season:{year: field of maxima}}
:param start_year:
:param end_year:
:param season_to_months:
(order of months in the list of months is important, i.e. for DJF the order should be [12, 1, 2])
"""
result = defaultdict(dict)
for season, months in season_to_months.items():
for y in range(start_year, end_year + 1):
d1 = Pendulum(y, months[0], 1)
d2 = d1.add(months=len(months)).subtract(seconds=1)
if d2.year > end_year:
continue
current_period = Period(d1, d2)
print("calculating mean for [{}, {}]".format(current_period.start, current_period.end))
data = self.read_data_for_period(current_period, varname_internal)
if varname_internal == LAKE_ICE_FRACTION:
result[season][y] = np.ma.masked_where(data.values > 1, data.values).max(axis=0)
else:
result[season][y] = data.max(dim="t").values
return result
def test_intersect_same(self):
start = Pendulum(2016, 8, 7)
end = start.add(weeks=1)
p1 = Period(start, end)
intersection = p1.intersect(Period(start.copy(), end.copy()))
self.assertPendulum(intersection.start, 2016, 8, 7)
self.assertPendulum(intersection.end, 2016, 8, 14)
def list_days_between(start: pendulum.Pendulum, end: pendulum.Pendulum):
start = start.start_of("day")
end = end.start_of("day")
days = []
while start <= end:
days.append(start)
start = start.add(days=1)
return days
def test_range_months_overflow(self):
dt1 = Pendulum(2016, 1, 30, tzinfo='America/Sao_Paulo')
dt2 = dt1.add(months=4)
p = Period(dt1, dt2)
r = p.range('months')
self.assertPendulum(r[0], 2016, 1, 30, 0, 0, 0)
self.assertPendulum(r[-1], 2016, 5, 30, 0, 0, 0)
def test_range_with_dst(self):
dt1 = Pendulum(2016, 10, 14, tzinfo='America/Sao_Paulo')
dt2 = dt1.add(weeks=1)
p = Period(dt1, dt2)
r = p.range('days')
self.assertPendulum(r[0], 2016, 10, 14, 0, 0, 0)
self.assertPendulum(r[2], 2016, 10, 16, 1, 0, 0)
self.assertPendulum(r[-1], 2016, 10, 21, 0, 0, 0)
def format(self):
d = Pendulum(2000, 1, 1, 12, 45, 31)
self.assertEqual('samedi', d.format('%A'))
self.assertEqual('sam', d.format('%a'))
self.assertEqual('janvier', d.format('%B'))
self.assertEqual('janv', d.format('%b'))
self.assertEqual('', d.format('%p'))
self.assertEqual('er', d.format('%_t'))
self.assertEqual('e', d.add(days=1).format('%_t'))
async def populate_paths(self, manifest_cache, entry_id: EntryID,
early: Pendulum, late: Pendulum):
# TODO : Use future manifest source field to follow files and directories
async with trio.open_service_nursery() as child_nursery:
child_nursery.start_soon(self.populate_source_path, manifest_cache,
entry_id, early.add(microseconds=-1))
child_nursery.start_soon(self.populate_destination_path,
manifest_cache, entry_id, late)
child_nursery.start_soon(self.populate_current_path,
manifest_cache, entry_id, early)
def hourly_fn(execution_date: Pendulum) -> Pendulum:
"""
default function for sensors of daily DAGs that depend on hourly DAGs
assuming the daily task runs at 05:25
:param execution_date: the DAG execution date
:return: Pendulum
"""
execution_date_transformed: Pendulum = execution_date.add(days=1)
return execution_date_transformed
def test_range_amount(self):
dt1 = Pendulum(2016, 10, 14, tzinfo='America/Sao_Paulo')
dt2 = dt1.add(weeks=1)
p = Period(dt1, dt2)
r = p.range('days', 2)
self.assertEqual(len(r), 4)
self.assertPendulum(r[0], 2016, 10, 14, 0, 0, 0)
self.assertPendulum(r[1], 2016, 10, 16, 1, 0, 0)
self.assertPendulum(r[2], 2016, 10, 18, 0, 0, 0)
self.assertPendulum(r[3], 2016, 10, 20, 0, 0, 0)
def ignore_past(execution_date: Pendulum) -> Pendulum:
"""
Decide what execution_date the sensor should look for
If execution date is more then 30 days ago, return execution date of 30 days ago
otherwise return execution date
:param execution_date:
:return: Pendulum
"""
execution_diff = execution_date.diff().in_days()
if execution_diff > 30:
return execution_date.add(days=(execution_diff - 30))
return execution_date
async def test_user_revoke_certify_too_old(backend, alice_backend_sock, alice,
bob):
now = Pendulum(2000, 1, 1)
revoked_user_certificate = RevokedUserCertificateContent(
author=alice.device_id, timestamp=now,
user_id=bob.user_id).dump_and_sign(alice.signing_key)
with freeze_time(now.add(seconds=INVITATION_VALIDITY + 1)):
rep = await user_revoke(
alice_backend_sock,
revoked_user_certificate=revoked_user_certificate)
assert rep == {
"status": "invalid_certification",
"reason": "Invalid timestamp in certification.",
}
async def test_create_bad_timestamp(alice, alice_backend_sock, realm):
blob = b"Initial commit."
d1 = Pendulum(2000, 1, 1)
with freeze_time(d1):
d2 = d1.add(seconds=3600)
rep = await vlob_create(alice_backend_sock,
realm,
VLOB_ID,
blob,
timestamp=d2,
check_rep=False)
assert rep == {
"status": "bad_timestamp",
"reason": "Timestamp is out of date."
}
def get_season_periods(self, start_year, end_year) -> list:
plist = []
for y in range(start_year, end_year + 1):
start = Pendulum(y, self.start_month, 1)
end = start.add(months=self.nmonths).subtract(microseconds=1)
if end.year > end_year:
continue
# print(start, end)
plist.append(Period(start, end))
return plist
def get_season_periods(self, start_year, end_year) -> list:
plist = []
for y in range(start_year, end_year + 1):
start = Pendulum(y, self.start_month, 1)
end = start.add(months=self.nmonths).subtract(microseconds=1)
if end.year > end_year:
continue
# print(start, end)
plist.append(
Period(start, end)
)
return plist
def get_year_month_to_period_map(self, start_year, end_year):
"""
generate mapping (year, month) --> period
:param start_year:
:param end_year:
:return:
"""
res = {}
for y in range(start_year, end_year + 1):
start = Pendulum(y, self.start_month, 1)
end = start.add(months=self.nmonths).subtract(microseconds=1)
if end.year > end_year:
continue
print(start, end)
p = Period(start, end)
for s in p.range("months"):
res[(s.year, s.month)] = p
return res
def get_seasonal_means(self, start_year: int, end_year: int, season_to_months: dict, varname_internal: str):
"""
returns a dictionary {season:{year: mean_field}}
:param start_year:
:param end_year:
:param season_to_months:
(order of months in the list of months is important, i.e. for DJF the order should be [12, 1, 2])
"""
result = defaultdict(dict)
cache_dir = Path(self.base_folder) / "cache"
cache_dir.mkdir(exist_ok=True)
seasons_str = "-".join(season_to_months)
cache_file = cache_dir / f"get_seasonal_means_{start_year}-{end_year}_m{seasons_str}_{varname_internal}.bin"
if cache_file.exists():
return pickle.load(cache_file.open("rb"))
for season, months in season_to_months.items():
for y in range(start_year, end_year + 1):
d1 = Pendulum(y, months[0], 1)
d2 = d1.add(months=len(months)).subtract(seconds=1)
if d2.year > end_year:
continue
current_period = Period(d1, d2)
print("calculating mean for [{}, {}]".format(current_period.start, current_period.end))
data = self.read_data_for_period(current_period, varname_internal)
result[season][y] = data.mean(dim="t").values
pickle.dump(result, cache_file.open("wb"))
return result
def main():
direction_file_path = Path("/RECH2/huziy/BC-MH/bc_mh_044deg/Samples/bc_mh_044deg_198001/pm1980010100_00000000p")
sim_label = "mh_0.44"
start_year = 1981
end_year = 2010
streamflow_internal_name = "streamflow"
selected_staion_ids = constants.selected_station_ids_for_streamflow_validation
# ======================================================
day = timedelta(days=1)
t0 = datetime(2001, 1, 1)
stamp_dates = [t0 + i * day for i in range(365)]
print("stamp dates range {} ... {}".format(stamp_dates[0], stamp_dates[-1]))
lake_fraction = None
# establish the correspondence between the stations and model grid points
with RPN(str(direction_file_path)) as r:
assert isinstance(r, RPN)
fldir = r.get_first_record_for_name("FLDR")
flow_acc_area = r.get_first_record_for_name("FAA")
lons, lats = r.get_longitudes_and_latitudes_for_the_last_read_rec()
# lake_fraction = r.get_first_record_for_name("LF1")
cell_manager = CellManager(fldir, lons2d=lons, lats2d=lats, accumulation_area_km2=flow_acc_area)
stations = stfl_stations.load_stations_from_csv(selected_ids=selected_staion_ids)
station_to_model_point = cell_manager.get_model_points_for_stations(station_list=stations, lake_fraction=lake_fraction,
nneighbours=8)
# Update the end year if required
max_year_st = -1
for station in station_to_model_point:
y = max(station.get_list_of_complete_years())
if y >= max_year_st:
max_year_st = y
if end_year > max_year_st:
print("Updated end_year to {}, because no obs data after...".format(max_year_st))
end_year = max_year_st
# read model data
mod_data_manager = DataManager(
store_config={
"varname_mapping": {streamflow_internal_name: "STFA"},
"base_folder": str(direction_file_path.parent.parent),
"data_source_type": data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT,
"level_mapping": {streamflow_internal_name: VerticalLevel(-1, level_type=level_kinds.ARBITRARY)},
"offset_mapping": vname_to_offset_CRCM5,
"filename_prefix_mapping": {streamflow_internal_name: "pm"}
})
station_to_model_data = defaultdict(list)
for year in range(start_year, end_year + 1):
start = Pendulum(year, 1, 1)
p_test = Period(start, start.add(years=1).subtract(microseconds=1))
stfl_mod = mod_data_manager.read_data_for_period(p_test, streamflow_internal_name)
# convert to daily
stfl_mod = stfl_mod.resample("D", "t", how="mean", closed="left", keep_attrs=True)
assert isinstance(stfl_mod, xr.DataArray)
for station, model_point in station_to_model_point.items():
assert isinstance(model_point, ModelPoint)
ts1 = stfl_mod[:, model_point.ix, model_point.jy].to_series()
station_to_model_data[station].append(pd.Series(index=stfl_mod.t.values, data=ts1))
# concatenate the timeseries for each point, if required
if end_year - start_year + 1 > 1:
for station in station_to_model_data:
station_to_model_data[station] = pd.concat(station_to_model_data[station])
else:
for station in station_to_model_data:
station_to_model_data[station] = station_to_model_data[station][0]
# calculate observed climatology
station_to_climatology = OrderedDict()
for s in sorted(station_to_model_point, key=lambda st: st.latitude, reverse=True):
assert isinstance(s, Station)
print(s.id, len(s.get_list_of_complete_years()))
# Check if there are continuous years for the selected period
common_years = set(s.get_list_of_complete_years()).intersection(set(range(start_year, end_year + 1)))
if len(common_years) > 0:
_, station_to_climatology[s] = s.get_daily_climatology_for_complete_years_with_pandas(stamp_dates=stamp_dates,
years=common_years)
_, station_to_model_data[s] = pandas_utils.get_daily_climatology_from_pandas_series(station_to_model_data[s],
stamp_dates,
years_of_interest=common_years)
else:
print("Skipping {}, since it does not have enough data during the period of interest".format(s.id))
# ---- Do the plotting ----
ncols = 4
nrows = len(station_to_climatology) // ncols
nrows += int(not (len(station_to_climatology) % ncols == 0))
axes_list = []
plot_utils.apply_plot_params(width_cm=8 * ncols, height_cm=8 * nrows, font_size=8)
fig = plt.figure()
gs = GridSpec(nrows=nrows, ncols=ncols)
for i, (s, clim) in enumerate(station_to_climatology.items()):
assert isinstance(s, Station)
row = i // ncols
col = i % ncols
print(row, col, nrows, ncols)
# normalize by the drainage area
if s.drainage_km2 is not None:
station_to_model_data[s] *= s.drainage_km2 / station_to_model_point[s].accumulation_area
if s.id in constants.stations_to_greyout:
ax = fig.add_subplot(gs[row, col], facecolor="0.45")
else:
ax = fig.add_subplot(gs[row, col])
assert isinstance(ax, Axes)
ax.plot(stamp_dates, clim, color="k", lw=2, label="Obs.")
ax.plot(stamp_dates, station_to_model_data[s], color="r", lw=2, label="Mod.")
ax.xaxis.set_major_formatter(FuncFormatter(format_month_label))
ax.xaxis.set_major_locator(MonthLocator(bymonthday=15))
ax.xaxis.set_minor_locator(MonthLocator(bymonthday=1))
ax.grid()
ax.annotate(s.get_pp_name(), xy=(1.02, 1), xycoords="axes fraction",
horizontalalignment="left", verticalalignment="top", fontsize=8, rotation=-90)
last_date = stamp_dates[-1]
last_date = last_date.replace(day=calendar.monthrange(last_date.year, last_date.month)[1])
ax.set_xlim(stamp_dates[0].replace(day=1), last_date)
ymin, ymax = ax.get_ylim()
ax.set_ylim(0, ymax)
if s.drainage_km2 is not None:
ax.set_title("{}: ({:.1f}$^\circ$E, {:.1f}$^\circ$N, DA={:.0f} km$^2$)".format(s.id, s.longitude, s.latitude, s.drainage_km2))
else:
ax.set_title(
"{}: ({:.1f}$^\circ$E, {:.1f}$^\circ$N, DA not used)".format(s.id, s.longitude, s.latitude))
axes_list.append(ax)
# plot the legend
axes_list[-1].legend()
if not img_folder.exists():
img_folder.mkdir()
fig.tight_layout()
img_file = img_folder / "{}_{}-{}_{}.png".format(sim_label, start_year, end_year, "-".join(sorted(s.id for s in station_to_climatology)))
print("Saving {}".format(img_file))
fig.savefig(str(img_file), bbox_inches="tight", dpi=300)
def _get_period_for_ym(year, month):
start = Pendulum(year, month, 1)
end = start.add(months=1).subtract(microseconds=1)
return Period(start, end)
def get_mean_number_of_cao_days(self, start_year: int, end_year: int, season_to_months: dict,
temperature_vname: str, min_cao_width_cells=5):
"""
calculate mean number of CAO days for each season and year {season: {year: field}}
Calculation following Wheeler et al 2011
:param self:
:param start_year:
:param end_year:
:param season_to_months:
:param temperature_vname:
"""
season_to_year_to_std = defaultdict(dict)
season_to_year_to_data = defaultdict(dict)
season_to_year_to_rolling_mean = defaultdict(dict)
season_to_year_to_n_cao_days = defaultdict(dict)
cache_dir = Path(self.base_folder) / "cache"
cache_dir.mkdir(exist_ok=True)
seasons_str = "-".join(season_to_months)
cache_file = cache_dir / f"get_mean_number_of_cao_days_{start_year}-{end_year}_m{seasons_str}_{temperature_vname}.bin"
if cache_file.exists() and False:
return pickle.load(cache_file.open("rb"))
for season, months in season_to_months.items():
for y in range(start_year, end_year + 1):
d1 = Pendulum(y, months[0], 1)
d2 = d1.add(months=len(months)).subtract(seconds=1)
if d2.year > end_year:
continue
current_period = Period(d1, d2)
print("calculating mean for [{}, {}]".format(current_period.start, current_period.end))
data = self.read_data_for_period(current_period, temperature_vname)
# calculate daily means
data_daily = data.resample(t="1D", keep_attrs=True).mean(dim="t").dropna(dim="t")
assert isinstance(data_daily, xarray.DataArray)
# save the data for reuse below
season_to_year_to_data[season][y] = data_daily.values
season_to_year_to_std[season][y] = data_daily.std(dim="t").values
season_to_year_to_rolling_mean[season][y] = data_daily.rolling(center=True, t=31).mean(dim="t").values
# Calculate climatological std and rolling mean
season_to_std_clim = {
s: np.mean([f for f in y_to_std.values()], axis=0) for s, y_to_std in season_to_year_to_std.items()
}
season_to_rolling_clim = {
s: np.mean([f for f in y_to_rolling.values()], axis=0) for s, y_to_rolling in season_to_year_to_rolling_mean.items()
}
# calculate number of CAO days
for season, std_clim in season_to_std_clim.items():
for y in range(start_year, end_year + 1):
t31_rolling = season_to_rolling_clim[season]
cao_suspect = (np.array(season_to_year_to_data[season][y]) <= t31_rolling - 1.5 * std_clim) & (std_clim > 2)
n_cao_days = cao_suspect.sum(axis=0)
season_to_year_to_n_cao_days[season][y] = n_cao_days
pickle.dump(season_to_year_to_n_cao_days, cache_file.open("wb"))
return season_to_year_to_n_cao_days
def foo(execution_date: Pendulum,
last_post_date: Pendulum):
empty_slots = 0
should_post = empty_slots == 0
diff_passed = execution_date.subtract(minutes=_last_post_diff) >= last_post_date
# if enough time passed since the last post,
# or it's the first time it ran, save the current time
return should_post and diff_passed
now = Pendulum(year=2020, month=4, day=22, minute=25)
for i in range(5):
p = now.add(minutes=i * 5)
r = foo(p, now)
print("{} -> {}".format(p, r))
print("")
for i in range(23):
p = now.add(hours=i)
r = mariadb_fn(p)
print("{} -> {}".format(p, r))
print("")
for i in range(23):
p = now.add(hours=i)
r = other_fn(p)
def test_timestamp(self):
d = Pendulum(1970, 1, 1, 0, 0, 0)
self.assertEqual(0, d.timestamp)
self.assertEqual(60, d.add(minutes=1).timestamp)
def test_timestamp(self):
f = AlternativeFormatter()
d = Pendulum(1970, 1, 1)
self.assertEqual('0', f.format(d, 'X'))
self.assertEqual('86400', f.format(d.add(days=1), 'X'))
def main():
direction_file_path = Path(
"/RECH2/huziy/BC-MH/bc_mh_044deg/Samples/bc_mh_044deg_198001/pm1980010100_00000000p"
)
sim_label = "mh_0.44"
start_year = 1981
end_year = 2010
streamflow_internal_name = "streamflow"
selected_staion_ids = constants.selected_station_ids_for_streamflow_validation
# ======================================================
day = timedelta(days=1)
t0 = datetime(2001, 1, 1)
stamp_dates = [t0 + i * day for i in range(365)]
print("stamp dates range {} ... {}".format(stamp_dates[0],
stamp_dates[-1]))
lake_fraction = None
# establish the correspondence between the stations and model grid points
with RPN(str(direction_file_path)) as r:
assert isinstance(r, RPN)
fldir = r.get_first_record_for_name("FLDR")
flow_acc_area = r.get_first_record_for_name("FAA")
lons, lats = r.get_longitudes_and_latitudes_for_the_last_read_rec()
# lake_fraction = r.get_first_record_for_name("LF1")
cell_manager = CellManager(fldir,
lons2d=lons,
lats2d=lats,
accumulation_area_km2=flow_acc_area)
stations = stfl_stations.load_stations_from_csv(
selected_ids=selected_staion_ids)
station_to_model_point = cell_manager.get_model_points_for_stations(
station_list=stations, lake_fraction=lake_fraction, nneighbours=8)
# Update the end year if required
max_year_st = -1
for station in station_to_model_point:
y = max(station.get_list_of_complete_years())
if y >= max_year_st:
max_year_st = y
if end_year > max_year_st:
print("Updated end_year to {}, because no obs data after...".format(
max_year_st))
end_year = max_year_st
# read model data
mod_data_manager = DataManager(
store_config={
"varname_mapping": {
streamflow_internal_name: "STFA"
},
"base_folder": str(direction_file_path.parent.parent),
"data_source_type":
data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT,
"level_mapping": {
streamflow_internal_name:
VerticalLevel(-1, level_type=level_kinds.ARBITRARY)
},
"offset_mapping": vname_to_offset_CRCM5,
"filename_prefix_mapping": {
streamflow_internal_name: "pm"
}
})
station_to_model_data = defaultdict(list)
for year in range(start_year, end_year + 1):
start = Pendulum(year, 1, 1)
p_test = Period(start, start.add(years=1).subtract(microseconds=1))
stfl_mod = mod_data_manager.read_data_for_period(
p_test, streamflow_internal_name)
# convert to daily
stfl_mod = stfl_mod.resample("D",
"t",
how="mean",
closed="left",
keep_attrs=True)
assert isinstance(stfl_mod, xr.DataArray)
for station, model_point in station_to_model_point.items():
assert isinstance(model_point, ModelPoint)
ts1 = stfl_mod[:, model_point.ix, model_point.jy].to_series()
station_to_model_data[station].append(
pd.Series(index=stfl_mod.t.values, data=ts1))
# concatenate the timeseries for each point, if required
if end_year - start_year + 1 > 1:
for station in station_to_model_data:
station_to_model_data[station] = pd.concat(
station_to_model_data[station])
else:
for station in station_to_model_data:
station_to_model_data[station] = station_to_model_data[station][0]
# calculate observed climatology
station_to_climatology = OrderedDict()
for s in sorted(station_to_model_point,
key=lambda st: st.latitude,
reverse=True):
assert isinstance(s, Station)
print(s.id, len(s.get_list_of_complete_years()))
# Check if there are continuous years for the selected period
common_years = set(s.get_list_of_complete_years()).intersection(
set(range(start_year, end_year + 1)))
if len(common_years) > 0:
_, station_to_climatology[
s] = s.get_daily_climatology_for_complete_years_with_pandas(
stamp_dates=stamp_dates, years=common_years)
_, station_to_model_data[
s] = pandas_utils.get_daily_climatology_from_pandas_series(
station_to_model_data[s],
stamp_dates,
years_of_interest=common_years)
else:
print(
"Skipping {}, since it does not have enough data during the period of interest"
.format(s.id))
# ---- Do the plotting ----
ncols = 4
nrows = len(station_to_climatology) // ncols
nrows += int(not (len(station_to_climatology) % ncols == 0))
axes_list = []
plot_utils.apply_plot_params(width_cm=8 * ncols,
height_cm=8 * nrows,
font_size=8)
fig = plt.figure()
gs = GridSpec(nrows=nrows, ncols=ncols)
for i, (s, clim) in enumerate(station_to_climatology.items()):
assert isinstance(s, Station)
row = i // ncols
col = i % ncols
print(row, col, nrows, ncols)
# normalize by the drainage area
if s.drainage_km2 is not None:
station_to_model_data[
s] *= s.drainage_km2 / station_to_model_point[
s].accumulation_area
if s.id in constants.stations_to_greyout:
ax = fig.add_subplot(gs[row, col], facecolor="0.45")
else:
ax = fig.add_subplot(gs[row, col])
assert isinstance(ax, Axes)
ax.plot(stamp_dates, clim, color="k", lw=2, label="Obs.")
ax.plot(stamp_dates,
station_to_model_data[s],
color="r",
lw=2,
label="Mod.")
ax.xaxis.set_major_formatter(FuncFormatter(format_month_label))
ax.xaxis.set_major_locator(MonthLocator(bymonthday=15))
ax.xaxis.set_minor_locator(MonthLocator(bymonthday=1))
ax.grid()
ax.annotate(s.get_pp_name(),
xy=(1.02, 1),
xycoords="axes fraction",
horizontalalignment="left",
verticalalignment="top",
fontsize=8,
rotation=-90)
last_date = stamp_dates[-1]
last_date = last_date.replace(
day=calendar.monthrange(last_date.year, last_date.month)[1])
ax.set_xlim(stamp_dates[0].replace(day=1), last_date)
ymin, ymax = ax.get_ylim()
ax.set_ylim(0, ymax)
if s.drainage_km2 is not None:
ax.set_title(
"{}: ({:.1f}$^\circ$E, {:.1f}$^\circ$N, DA={:.0f} km$^2$)".
format(s.id, s.longitude, s.latitude, s.drainage_km2))
else:
ax.set_title(
"{}: ({:.1f}$^\circ$E, {:.1f}$^\circ$N, DA not used)".format(
s.id, s.longitude, s.latitude))
axes_list.append(ax)
# plot the legend
axes_list[-1].legend()
if not img_folder.exists():
img_folder.mkdir()
fig.tight_layout()
img_file = img_folder / "{}_{}-{}_{}.png".format(
sim_label, start_year, end_year, "-".join(
sorted(s.id for s in station_to_climatology)))
print("Saving {}".format(img_file))
fig.savefig(str(img_file), bbox_inches="tight", dpi=300)
'name': 'page_events',
'task': page_events
}, {
'name': 'session',
'task': session
}, {
'name': 'enrichment',
'task': enrichment
}]
if __name__ == "__main__":
start = Pendulum(2021, 1, 17, 5)
end = Pendulum(2021, 1, 18, 4)
hours_ahead = 24
while start < end:
next_hour = start.add(hours=hours_ahead)
start_fmt = start.format('%Y-%m-%d-%H')
next_fmt = next_hour.format('%Y-%m-%d-%H')
print(f'echo -e "{start_fmt}, {next_fmt}"')
for task in tasks:
task_name = task['name']
task_cmd = task['task']
if task_name in ['session', 'enrichment']:
if task_name == 'enrichment':
print(copy.deepcopy(task_cmd) % (start_fmt, next_fmt))
else:
for h in range(0, hours_ahead + 1):
print(
copy.deepcopy(task_cmd) %
start.add(hours=h).format('%Y-%m-%d-%H'))
else:
