def test_timestamp(self): d = Pendulum(1970, 1, 1, 0, 0, 0) self.assertEqual(0, d.timestamp()) self.assertEqual(0, d.timestamp) self.assertEqual(60.123456, d.add(minutes=1, microseconds=123456).timestamp()) self.assertEqual(60, d.add(minutes=1, microseconds=123456).timestamp)
def test_intersect_excluded(self): start = Pendulum(2016, 8, 7) end = start.add(weeks=1) p1 = Period(start, end) intersection = p1.intersect( Period(start.add(days=-2), start.add(days=-1))) self.assertIsNone(intersection)
def test_intersect_included(self): start = Pendulum(2016, 8, 7) end = start.add(weeks=1) p1 = Period(start, end) intersection = p1.intersect( Period(start.add(days=2), start.add(days=4))) self.assertPendulum(intersection.start, 2016, 8, 9) self.assertPendulum(intersection.end, 2016, 8, 11)
def test_intersect_multiple(self): start = Pendulum(2016, 8, 7) end = start.add(weeks=1) p1 = Period(start, end) intersection = p1.intersect( Period(start.add(days=-2), start.add(days=2)), Period(start.add(days=1), start.add(days=2))) self.assertPendulum(intersection.start, 2016, 8, 8) self.assertPendulum(intersection.end, 2016, 8, 9)
def test_day_of_year(self): f = AlternativeFormatter() d = Pendulum(2016, 8, 28) self.assertEqual('241', f.format(d, 'DDDD')) self.assertEqual('241', f.format(d, 'DDD')) self.assertEqual('001', f.format(d.start_of('year'), 'DDDD')) self.assertEqual('1', f.format(d.start_of('year'), 'DDD')) self.assertEqual('241st', f.format(d, 'DDDo')) self.assertEqual('244th', f.format(d.add(days=3), 'DDDo')) self.assertEqual('241e', f.format(d, 'DDDo', locale='fr')) self.assertEqual('244e', f.format(d.add(days=3), 'DDDo', locale='fr'))
def test_floor_divide(self): dt1 = Pendulum(2016, 8, 7, 12, 34, 56) dt2 = dt1.add(days=2, seconds=34) it = Period(dt1, dt2) mul = it // 2 self.assertIsInstanceOfInterval(mul) self.assertInterval(mul, 0, 1, 0, 0, 17) dt1 = Pendulum(2016, 8, 7, 12, 34, 56) dt2 = dt1.add(days=2, seconds=35) it = Period(dt1, dt2) mul = it // 3 self.assertIsInstanceOfInterval(mul) self.assertInterval(mul, 0, 0, 16, 0, 11)
def test_multiply(self): dt1 = Pendulum(2016, 8, 7, 12, 34, 56) dt2 = dt1.add(days=6, seconds=34) it = Period(dt1, dt2) mul = it * 2 self.assertIsInstanceOfInterval(mul) self.assertInterval(mul, 1, 5, 0, 1, 8) dt1 = Pendulum(2016, 8, 7, 12, 34, 56) dt2 = dt1.add(days=6, seconds=34) it = Period(dt1, dt2) mul = it * 2 self.assertIsInstanceOfInterval(mul) self.assertInterval(mul, 1, 5, 0, 1, 8)
def test_time_as_datetime(self, mock_now): set_time = time(12, 00) # This is a monday now = Pendulum(year=2016, month=9, day=19, hour=set_time.hour, minute=set_time.minute, tzinfo=None) for current_day in range(7): now = now.add(days=1) mock_now.return_value = now for weekday_config in WEEKDAYS: day_in_week = weekday_config[0] period = OpeningPeriod(day=day_in_week, time=set_time, duration=time(1, 00), store=self.store) as_datetime = period.weekday_as_datetime(weekday=period.day, time=period.time, store=period.store) if day_in_week == now.isoweekday(): self.assertEqual(as_datetime, now) else: self.assertGreater(as_datetime, now)
def get_year_month_to_period_map(self, start_year, end_year): """ generate mapping (year, month) --> period :param start_year: :param end_year: :return: """ res = {} for y in range(start_year, end_year + 1): start = Pendulum(y, self.start_month, 1) end = start.add(months=self.nmonths).subtract(microseconds=1) if end.year > end_year: continue print(start, end) p = Period(start, end) for s in p.range("months"): res[(s.year, s.month)] = p return res
async def test_retrieve_device(running_backend, alice_remote_devices_manager, bob): remote_devices_manager = alice_remote_devices_manager d1 = Pendulum(2000, 1, 1) with freeze_time(d1): # Offline with no cache with pytest.raises(RemoteDevicesManagerBackendOfflineError): with running_backend.offline(): await remote_devices_manager.get_device(bob.device_id) # Online device = await remote_devices_manager.get_device(bob.device_id) assert device.device_id == bob.device_id assert device.verify_key == bob.verify_key # Offline with cache with running_backend.offline(): device2 = await remote_devices_manager.get_device(bob.device_id) assert device2 is device d2 = d1.add(remote_devices_manager.cache_validity + 1) with freeze_time(d2): # Offline with cache expired with pytest.raises(RemoteDevicesManagerBackendOfflineError): with running_backend.offline(): await remote_devices_manager.get_device(bob.device_id) # Online with cache expired device = await remote_devices_manager.get_device(bob.device_id) assert device.device_id == bob.device_id assert device.verify_key == bob.verify_key
def get_mean_number_of_hles_days(self, start_year: int, end_year: int, season_to_months: dict, hles_vname: str): result = defaultdict(dict) cache_dir = Path(self.base_folder) / "cache" cache_dir.mkdir(exist_ok=True) seasons_str = "-".join(season_to_months) cache_file = cache_dir / f"get_mean_number_of_hles_days_{start_year}-{end_year}_m{seasons_str}_{hles_vname}.bin" if cache_file.exists(): return pickle.load(cache_file.open("rb")) for season, months in season_to_months.items(): for y in range(start_year, end_year + 1): d1 = Pendulum(y, months[0], 1) d2 = d1.add(months=len(months)).subtract(seconds=1) if d2.year > end_year: continue current_period = Period(d1, d2) print("calculating mean for [{}, {}]".format(current_period.start, current_period.end)) data = self.read_data_for_period(current_period, hles_vname) # calculate number of hles days data_daily = data.resample(t="1D", keep_attrs=True).mean(dim="t") result[season][y] = (data_daily.values >= 0.1).sum(axis=0) pickle.dump(result, cache_file.open("wb")) return result
async def test_retrieve_user(running_backend, alice_remote_devices_manager, bob): remote_devices_manager = alice_remote_devices_manager d1 = Pendulum(2000, 1, 1) with freeze_time(d1): # Offline with no cache with pytest.raises(RemoteDevicesManagerBackendOfflineError): with running_backend.offline(): await remote_devices_manager.get_user(bob.user_id) # Online user, revoked_user = await remote_devices_manager.get_user(bob.user_id) assert user.user_id == bob.user_id assert user.public_key == bob.public_key assert revoked_user is None # Offline with cache with running_backend.offline(): user2, revoked_user2 = await remote_devices_manager.get_user( bob.user_id) assert user2 is user assert revoked_user2 is None d2 = d1.add(remote_devices_manager.cache_validity + 1) with freeze_time(d2): # Offline with cache expired with pytest.raises(RemoteDevicesManagerBackendOfflineError): with running_backend.offline(): await remote_devices_manager.get_user(bob.user_id) # Online with cache expired user, revoked_user = await remote_devices_manager.get_user(bob.user_id) assert user.user_id == bob.user_id assert user.public_key == bob.public_key assert revoked_user is None
def get_seasonal_maxima(self, start_year: int, end_year: int, season_to_months: dict, varname_internal: str): """ returns a dictionary {season:{year: field of maxima}} :param start_year: :param end_year: :param season_to_months: (order of months in the list of months is important, i.e. for DJF the order should be [12, 1, 2]) """ result = defaultdict(dict) for season, months in season_to_months.items(): for y in range(start_year, end_year + 1): d1 = Pendulum(y, months[0], 1) d2 = d1.add(months=len(months)).subtract(seconds=1) if d2.year > end_year: continue current_period = Period(d1, d2) print("calculating mean for [{}, {}]".format(current_period.start, current_period.end)) data = self.read_data_for_period(current_period, varname_internal) if varname_internal == LAKE_ICE_FRACTION: result[season][y] = np.ma.masked_where(data.values > 1, data.values).max(axis=0) else: result[season][y] = data.max(dim="t").values return result
def test_intersect_same(self): start = Pendulum(2016, 8, 7) end = start.add(weeks=1) p1 = Period(start, end) intersection = p1.intersect(Period(start.copy(), end.copy())) self.assertPendulum(intersection.start, 2016, 8, 7) self.assertPendulum(intersection.end, 2016, 8, 14)
def list_days_between(start: pendulum.Pendulum, end: pendulum.Pendulum): start = start.start_of("day") end = end.start_of("day") days = [] while start <= end: days.append(start) start = start.add(days=1) return days
def test_range_months_overflow(self): dt1 = Pendulum(2016, 1, 30, tzinfo='America/Sao_Paulo') dt2 = dt1.add(months=4) p = Period(dt1, dt2) r = p.range('months') self.assertPendulum(r[0], 2016, 1, 30, 0, 0, 0) self.assertPendulum(r[-1], 2016, 5, 30, 0, 0, 0)
def test_range_with_dst(self): dt1 = Pendulum(2016, 10, 14, tzinfo='America/Sao_Paulo') dt2 = dt1.add(weeks=1) p = Period(dt1, dt2) r = p.range('days') self.assertPendulum(r[0], 2016, 10, 14, 0, 0, 0) self.assertPendulum(r[2], 2016, 10, 16, 1, 0, 0) self.assertPendulum(r[-1], 2016, 10, 21, 0, 0, 0)
def format(self): d = Pendulum(2000, 1, 1, 12, 45, 31) self.assertEqual('samedi', d.format('%A')) self.assertEqual('sam', d.format('%a')) self.assertEqual('janvier', d.format('%B')) self.assertEqual('janv', d.format('%b')) self.assertEqual('', d.format('%p')) self.assertEqual('er', d.format('%_t')) self.assertEqual('e', d.add(days=1).format('%_t'))
async def populate_paths(self, manifest_cache, entry_id: EntryID, early: Pendulum, late: Pendulum): # TODO : Use future manifest source field to follow files and directories async with trio.open_service_nursery() as child_nursery: child_nursery.start_soon(self.populate_source_path, manifest_cache, entry_id, early.add(microseconds=-1)) child_nursery.start_soon(self.populate_destination_path, manifest_cache, entry_id, late) child_nursery.start_soon(self.populate_current_path, manifest_cache, entry_id, early)
def hourly_fn(execution_date: Pendulum) -> Pendulum: """ default function for sensors of daily DAGs that depend on hourly DAGs assuming the daily task runs at 05:25 :param execution_date: the DAG execution date :return: Pendulum """ execution_date_transformed: Pendulum = execution_date.add(days=1) return execution_date_transformed
def test_range_amount(self): dt1 = Pendulum(2016, 10, 14, tzinfo='America/Sao_Paulo') dt2 = dt1.add(weeks=1) p = Period(dt1, dt2) r = p.range('days', 2) self.assertEqual(len(r), 4) self.assertPendulum(r[0], 2016, 10, 14, 0, 0, 0) self.assertPendulum(r[1], 2016, 10, 16, 1, 0, 0) self.assertPendulum(r[2], 2016, 10, 18, 0, 0, 0) self.assertPendulum(r[3], 2016, 10, 20, 0, 0, 0)
def ignore_past(execution_date: Pendulum) -> Pendulum: """ Decide what execution_date the sensor should look for If execution date is more then 30 days ago, return execution date of 30 days ago otherwise return execution date :param execution_date: :return: Pendulum """ execution_diff = execution_date.diff().in_days() if execution_diff > 30: return execution_date.add(days=(execution_diff - 30)) return execution_date
async def test_user_revoke_certify_too_old(backend, alice_backend_sock, alice, bob): now = Pendulum(2000, 1, 1) revoked_user_certificate = RevokedUserCertificateContent( author=alice.device_id, timestamp=now, user_id=bob.user_id).dump_and_sign(alice.signing_key) with freeze_time(now.add(seconds=INVITATION_VALIDITY + 1)): rep = await user_revoke( alice_backend_sock, revoked_user_certificate=revoked_user_certificate) assert rep == { "status": "invalid_certification", "reason": "Invalid timestamp in certification.", }
async def test_create_bad_timestamp(alice, alice_backend_sock, realm): blob = b"Initial commit." d1 = Pendulum(2000, 1, 1) with freeze_time(d1): d2 = d1.add(seconds=3600) rep = await vlob_create(alice_backend_sock, realm, VLOB_ID, blob, timestamp=d2, check_rep=False) assert rep == { "status": "bad_timestamp", "reason": "Timestamp is out of date." }
def get_season_periods(self, start_year, end_year) -> list: plist = [] for y in range(start_year, end_year + 1): start = Pendulum(y, self.start_month, 1) end = start.add(months=self.nmonths).subtract(microseconds=1) if end.year > end_year: continue # print(start, end) plist.append(Period(start, end)) return plist
def get_season_periods(self, start_year, end_year) -> list: plist = [] for y in range(start_year, end_year + 1): start = Pendulum(y, self.start_month, 1) end = start.add(months=self.nmonths).subtract(microseconds=1) if end.year > end_year: continue # print(start, end) plist.append( Period(start, end) ) return plist
def get_seasonal_means(self, start_year: int, end_year: int, season_to_months: dict, varname_internal: str): """ returns a dictionary {season:{year: mean_field}} :param start_year: :param end_year: :param season_to_months: (order of months in the list of months is important, i.e. for DJF the order should be [12, 1, 2]) """ result = defaultdict(dict) cache_dir = Path(self.base_folder) / "cache" cache_dir.mkdir(exist_ok=True) seasons_str = "-".join(season_to_months) cache_file = cache_dir / f"get_seasonal_means_{start_year}-{end_year}_m{seasons_str}_{varname_internal}.bin" if cache_file.exists(): return pickle.load(cache_file.open("rb")) for season, months in season_to_months.items(): for y in range(start_year, end_year + 1): d1 = Pendulum(y, months[0], 1) d2 = d1.add(months=len(months)).subtract(seconds=1) if d2.year > end_year: continue current_period = Period(d1, d2) print("calculating mean for [{}, {}]".format(current_period.start, current_period.end)) data = self.read_data_for_period(current_period, varname_internal) result[season][y] = data.mean(dim="t").values pickle.dump(result, cache_file.open("wb")) return result
def main(): direction_file_path = Path("/RECH2/huziy/BC-MH/bc_mh_044deg/Samples/bc_mh_044deg_198001/pm1980010100_00000000p") sim_label = "mh_0.44" start_year = 1981 end_year = 2010 streamflow_internal_name = "streamflow" selected_staion_ids = constants.selected_station_ids_for_streamflow_validation # ====================================================== day = timedelta(days=1) t0 = datetime(2001, 1, 1) stamp_dates = [t0 + i * day for i in range(365)] print("stamp dates range {} ... {}".format(stamp_dates[0], stamp_dates[-1])) lake_fraction = None # establish the correspondence between the stations and model grid points with RPN(str(direction_file_path)) as r: assert isinstance(r, RPN) fldir = r.get_first_record_for_name("FLDR") flow_acc_area = r.get_first_record_for_name("FAA") lons, lats = r.get_longitudes_and_latitudes_for_the_last_read_rec() # lake_fraction = r.get_first_record_for_name("LF1") cell_manager = CellManager(fldir, lons2d=lons, lats2d=lats, accumulation_area_km2=flow_acc_area) stations = stfl_stations.load_stations_from_csv(selected_ids=selected_staion_ids) station_to_model_point = cell_manager.get_model_points_for_stations(station_list=stations, lake_fraction=lake_fraction, nneighbours=8) # Update the end year if required max_year_st = -1 for station in station_to_model_point: y = max(station.get_list_of_complete_years()) if y >= max_year_st: max_year_st = y if end_year > max_year_st: print("Updated end_year to {}, because no obs data after...".format(max_year_st)) end_year = max_year_st # read model data mod_data_manager = DataManager( store_config={ "varname_mapping": {streamflow_internal_name: "STFA"}, "base_folder": str(direction_file_path.parent.parent), "data_source_type": data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT, "level_mapping": {streamflow_internal_name: VerticalLevel(-1, level_type=level_kinds.ARBITRARY)}, "offset_mapping": vname_to_offset_CRCM5, "filename_prefix_mapping": {streamflow_internal_name: "pm"} }) station_to_model_data = defaultdict(list) for year in range(start_year, end_year + 1): start = Pendulum(year, 1, 1) p_test = Period(start, start.add(years=1).subtract(microseconds=1)) stfl_mod = mod_data_manager.read_data_for_period(p_test, streamflow_internal_name) # convert to daily stfl_mod = stfl_mod.resample("D", "t", how="mean", closed="left", keep_attrs=True) assert isinstance(stfl_mod, xr.DataArray) for station, model_point in station_to_model_point.items(): assert isinstance(model_point, ModelPoint) ts1 = stfl_mod[:, model_point.ix, model_point.jy].to_series() station_to_model_data[station].append(pd.Series(index=stfl_mod.t.values, data=ts1)) # concatenate the timeseries for each point, if required if end_year - start_year + 1 > 1: for station in station_to_model_data: station_to_model_data[station] = pd.concat(station_to_model_data[station]) else: for station in station_to_model_data: station_to_model_data[station] = station_to_model_data[station][0] # calculate observed climatology station_to_climatology = OrderedDict() for s in sorted(station_to_model_point, key=lambda st: st.latitude, reverse=True): assert isinstance(s, Station) print(s.id, len(s.get_list_of_complete_years())) # Check if there are continuous years for the selected period common_years = set(s.get_list_of_complete_years()).intersection(set(range(start_year, end_year + 1))) if len(common_years) > 0: _, station_to_climatology[s] = s.get_daily_climatology_for_complete_years_with_pandas(stamp_dates=stamp_dates, years=common_years) _, station_to_model_data[s] = pandas_utils.get_daily_climatology_from_pandas_series(station_to_model_data[s], stamp_dates, years_of_interest=common_years) else: print("Skipping {}, since it does not have enough data during the period of interest".format(s.id)) # ---- Do the plotting ---- ncols = 4 nrows = len(station_to_climatology) // ncols nrows += int(not (len(station_to_climatology) % ncols == 0)) axes_list = [] plot_utils.apply_plot_params(width_cm=8 * ncols, height_cm=8 * nrows, font_size=8) fig = plt.figure() gs = GridSpec(nrows=nrows, ncols=ncols) for i, (s, clim) in enumerate(station_to_climatology.items()): assert isinstance(s, Station) row = i // ncols col = i % ncols print(row, col, nrows, ncols) # normalize by the drainage area if s.drainage_km2 is not None: station_to_model_data[s] *= s.drainage_km2 / station_to_model_point[s].accumulation_area if s.id in constants.stations_to_greyout: ax = fig.add_subplot(gs[row, col], facecolor="0.45") else: ax = fig.add_subplot(gs[row, col]) assert isinstance(ax, Axes) ax.plot(stamp_dates, clim, color="k", lw=2, label="Obs.") ax.plot(stamp_dates, station_to_model_data[s], color="r", lw=2, label="Mod.") ax.xaxis.set_major_formatter(FuncFormatter(format_month_label)) ax.xaxis.set_major_locator(MonthLocator(bymonthday=15)) ax.xaxis.set_minor_locator(MonthLocator(bymonthday=1)) ax.grid() ax.annotate(s.get_pp_name(), xy=(1.02, 1), xycoords="axes fraction", horizontalalignment="left", verticalalignment="top", fontsize=8, rotation=-90) last_date = stamp_dates[-1] last_date = last_date.replace(day=calendar.monthrange(last_date.year, last_date.month)[1]) ax.set_xlim(stamp_dates[0].replace(day=1), last_date) ymin, ymax = ax.get_ylim() ax.set_ylim(0, ymax) if s.drainage_km2 is not None: ax.set_title("{}: ({:.1f}$^\circ$E, {:.1f}$^\circ$N, DA={:.0f} km$^2$)".format(s.id, s.longitude, s.latitude, s.drainage_km2)) else: ax.set_title( "{}: ({:.1f}$^\circ$E, {:.1f}$^\circ$N, DA not used)".format(s.id, s.longitude, s.latitude)) axes_list.append(ax) # plot the legend axes_list[-1].legend() if not img_folder.exists(): img_folder.mkdir() fig.tight_layout() img_file = img_folder / "{}_{}-{}_{}.png".format(sim_label, start_year, end_year, "-".join(sorted(s.id for s in station_to_climatology))) print("Saving {}".format(img_file)) fig.savefig(str(img_file), bbox_inches="tight", dpi=300)
def _get_period_for_ym(year, month): start = Pendulum(year, month, 1) end = start.add(months=1).subtract(microseconds=1) return Period(start, end)
def get_mean_number_of_cao_days(self, start_year: int, end_year: int, season_to_months: dict, temperature_vname: str, min_cao_width_cells=5): """ calculate mean number of CAO days for each season and year {season: {year: field}} Calculation following Wheeler et al 2011 :param self: :param start_year: :param end_year: :param season_to_months: :param temperature_vname: """ season_to_year_to_std = defaultdict(dict) season_to_year_to_data = defaultdict(dict) season_to_year_to_rolling_mean = defaultdict(dict) season_to_year_to_n_cao_days = defaultdict(dict) cache_dir = Path(self.base_folder) / "cache" cache_dir.mkdir(exist_ok=True) seasons_str = "-".join(season_to_months) cache_file = cache_dir / f"get_mean_number_of_cao_days_{start_year}-{end_year}_m{seasons_str}_{temperature_vname}.bin" if cache_file.exists() and False: return pickle.load(cache_file.open("rb")) for season, months in season_to_months.items(): for y in range(start_year, end_year + 1): d1 = Pendulum(y, months[0], 1) d2 = d1.add(months=len(months)).subtract(seconds=1) if d2.year > end_year: continue current_period = Period(d1, d2) print("calculating mean for [{}, {}]".format(current_period.start, current_period.end)) data = self.read_data_for_period(current_period, temperature_vname) # calculate daily means data_daily = data.resample(t="1D", keep_attrs=True).mean(dim="t").dropna(dim="t") assert isinstance(data_daily, xarray.DataArray) # save the data for reuse below season_to_year_to_data[season][y] = data_daily.values season_to_year_to_std[season][y] = data_daily.std(dim="t").values season_to_year_to_rolling_mean[season][y] = data_daily.rolling(center=True, t=31).mean(dim="t").values # Calculate climatological std and rolling mean season_to_std_clim = { s: np.mean([f for f in y_to_std.values()], axis=0) for s, y_to_std in season_to_year_to_std.items() } season_to_rolling_clim = { s: np.mean([f for f in y_to_rolling.values()], axis=0) for s, y_to_rolling in season_to_year_to_rolling_mean.items() } # calculate number of CAO days for season, std_clim in season_to_std_clim.items(): for y in range(start_year, end_year + 1): t31_rolling = season_to_rolling_clim[season] cao_suspect = (np.array(season_to_year_to_data[season][y]) <= t31_rolling - 1.5 * std_clim) & (std_clim > 2) n_cao_days = cao_suspect.sum(axis=0) season_to_year_to_n_cao_days[season][y] = n_cao_days pickle.dump(season_to_year_to_n_cao_days, cache_file.open("wb")) return season_to_year_to_n_cao_days
def foo(execution_date: Pendulum, last_post_date: Pendulum): empty_slots = 0 should_post = empty_slots == 0 diff_passed = execution_date.subtract(minutes=_last_post_diff) >= last_post_date # if enough time passed since the last post, # or it's the first time it ran, save the current time return should_post and diff_passed now = Pendulum(year=2020, month=4, day=22, minute=25) for i in range(5): p = now.add(minutes=i * 5) r = foo(p, now) print("{} -> {}".format(p, r)) print("") for i in range(23): p = now.add(hours=i) r = mariadb_fn(p) print("{} -> {}".format(p, r)) print("") for i in range(23): p = now.add(hours=i) r = other_fn(p)
def test_timestamp(self): d = Pendulum(1970, 1, 1, 0, 0, 0) self.assertEqual(0, d.timestamp) self.assertEqual(60, d.add(minutes=1).timestamp)
def test_timestamp(self): f = AlternativeFormatter() d = Pendulum(1970, 1, 1) self.assertEqual('0', f.format(d, 'X')) self.assertEqual('86400', f.format(d.add(days=1), 'X'))
def main(): direction_file_path = Path( "/RECH2/huziy/BC-MH/bc_mh_044deg/Samples/bc_mh_044deg_198001/pm1980010100_00000000p" ) sim_label = "mh_0.44" start_year = 1981 end_year = 2010 streamflow_internal_name = "streamflow" selected_staion_ids = constants.selected_station_ids_for_streamflow_validation # ====================================================== day = timedelta(days=1) t0 = datetime(2001, 1, 1) stamp_dates = [t0 + i * day for i in range(365)] print("stamp dates range {} ... {}".format(stamp_dates[0], stamp_dates[-1])) lake_fraction = None # establish the correspondence between the stations and model grid points with RPN(str(direction_file_path)) as r: assert isinstance(r, RPN) fldir = r.get_first_record_for_name("FLDR") flow_acc_area = r.get_first_record_for_name("FAA") lons, lats = r.get_longitudes_and_latitudes_for_the_last_read_rec() # lake_fraction = r.get_first_record_for_name("LF1") cell_manager = CellManager(fldir, lons2d=lons, lats2d=lats, accumulation_area_km2=flow_acc_area) stations = stfl_stations.load_stations_from_csv( selected_ids=selected_staion_ids) station_to_model_point = cell_manager.get_model_points_for_stations( station_list=stations, lake_fraction=lake_fraction, nneighbours=8) # Update the end year if required max_year_st = -1 for station in station_to_model_point: y = max(station.get_list_of_complete_years()) if y >= max_year_st: max_year_st = y if end_year > max_year_st: print("Updated end_year to {}, because no obs data after...".format( max_year_st)) end_year = max_year_st # read model data mod_data_manager = DataManager( store_config={ "varname_mapping": { streamflow_internal_name: "STFA" }, "base_folder": str(direction_file_path.parent.parent), "data_source_type": data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT, "level_mapping": { streamflow_internal_name: VerticalLevel(-1, level_type=level_kinds.ARBITRARY) }, "offset_mapping": vname_to_offset_CRCM5, "filename_prefix_mapping": { streamflow_internal_name: "pm" } }) station_to_model_data = defaultdict(list) for year in range(start_year, end_year + 1): start = Pendulum(year, 1, 1) p_test = Period(start, start.add(years=1).subtract(microseconds=1)) stfl_mod = mod_data_manager.read_data_for_period( p_test, streamflow_internal_name) # convert to daily stfl_mod = stfl_mod.resample("D", "t", how="mean", closed="left", keep_attrs=True) assert isinstance(stfl_mod, xr.DataArray) for station, model_point in station_to_model_point.items(): assert isinstance(model_point, ModelPoint) ts1 = stfl_mod[:, model_point.ix, model_point.jy].to_series() station_to_model_data[station].append( pd.Series(index=stfl_mod.t.values, data=ts1)) # concatenate the timeseries for each point, if required if end_year - start_year + 1 > 1: for station in station_to_model_data: station_to_model_data[station] = pd.concat( station_to_model_data[station]) else: for station in station_to_model_data: station_to_model_data[station] = station_to_model_data[station][0] # calculate observed climatology station_to_climatology = OrderedDict() for s in sorted(station_to_model_point, key=lambda st: st.latitude, reverse=True): assert isinstance(s, Station) print(s.id, len(s.get_list_of_complete_years())) # Check if there are continuous years for the selected period common_years = set(s.get_list_of_complete_years()).intersection( set(range(start_year, end_year + 1))) if len(common_years) > 0: _, station_to_climatology[ s] = s.get_daily_climatology_for_complete_years_with_pandas( stamp_dates=stamp_dates, years=common_years) _, station_to_model_data[ s] = pandas_utils.get_daily_climatology_from_pandas_series( station_to_model_data[s], stamp_dates, years_of_interest=common_years) else: print( "Skipping {}, since it does not have enough data during the period of interest" .format(s.id)) # ---- Do the plotting ---- ncols = 4 nrows = len(station_to_climatology) // ncols nrows += int(not (len(station_to_climatology) % ncols == 0)) axes_list = [] plot_utils.apply_plot_params(width_cm=8 * ncols, height_cm=8 * nrows, font_size=8) fig = plt.figure() gs = GridSpec(nrows=nrows, ncols=ncols) for i, (s, clim) in enumerate(station_to_climatology.items()): assert isinstance(s, Station) row = i // ncols col = i % ncols print(row, col, nrows, ncols) # normalize by the drainage area if s.drainage_km2 is not None: station_to_model_data[ s] *= s.drainage_km2 / station_to_model_point[ s].accumulation_area if s.id in constants.stations_to_greyout: ax = fig.add_subplot(gs[row, col], facecolor="0.45") else: ax = fig.add_subplot(gs[row, col]) assert isinstance(ax, Axes) ax.plot(stamp_dates, clim, color="k", lw=2, label="Obs.") ax.plot(stamp_dates, station_to_model_data[s], color="r", lw=2, label="Mod.") ax.xaxis.set_major_formatter(FuncFormatter(format_month_label)) ax.xaxis.set_major_locator(MonthLocator(bymonthday=15)) ax.xaxis.set_minor_locator(MonthLocator(bymonthday=1)) ax.grid() ax.annotate(s.get_pp_name(), xy=(1.02, 1), xycoords="axes fraction", horizontalalignment="left", verticalalignment="top", fontsize=8, rotation=-90) last_date = stamp_dates[-1] last_date = last_date.replace( day=calendar.monthrange(last_date.year, last_date.month)[1]) ax.set_xlim(stamp_dates[0].replace(day=1), last_date) ymin, ymax = ax.get_ylim() ax.set_ylim(0, ymax) if s.drainage_km2 is not None: ax.set_title( "{}: ({:.1f}$^\circ$E, {:.1f}$^\circ$N, DA={:.0f} km$^2$)". format(s.id, s.longitude, s.latitude, s.drainage_km2)) else: ax.set_title( "{}: ({:.1f}$^\circ$E, {:.1f}$^\circ$N, DA not used)".format( s.id, s.longitude, s.latitude)) axes_list.append(ax) # plot the legend axes_list[-1].legend() if not img_folder.exists(): img_folder.mkdir() fig.tight_layout() img_file = img_folder / "{}_{}-{}_{}.png".format( sim_label, start_year, end_year, "-".join( sorted(s.id for s in station_to_climatology))) print("Saving {}".format(img_file)) fig.savefig(str(img_file), bbox_inches="tight", dpi=300)
'name': 'page_events', 'task': page_events }, { 'name': 'session', 'task': session }, { 'name': 'enrichment', 'task': enrichment }] if __name__ == "__main__": start = Pendulum(2021, 1, 17, 5) end = Pendulum(2021, 1, 18, 4) hours_ahead = 24 while start < end: next_hour = start.add(hours=hours_ahead) start_fmt = start.format('%Y-%m-%d-%H') next_fmt = next_hour.format('%Y-%m-%d-%H') print(f'echo -e "{start_fmt}, {next_fmt}"') for task in tasks: task_name = task['name'] task_cmd = task['task'] if task_name in ['session', 'enrichment']: if task_name == 'enrichment': print(copy.deepcopy(task_cmd) % (start_fmt, next_fmt)) else: for h in range(0, hours_ahead + 1): print( copy.deepcopy(task_cmd) % start.add(hours=h).format('%Y-%m-%d-%H')) else: