def _nwp_issue_time_generator(fx, gap_start, gap_end): # max_run_time is the forecast issue time that will generate forecast # that end before gap_end max_run_time = utils.find_next_issue_time_from_last_forecast( fx, gap_end - pd.Timedelta('1ns')) # next_issue_time is the forecast issue time that will generate forecast # values at gap_start next_issue_time = utils.find_next_issue_time_from_last_forecast( fx, gap_start) while next_issue_time < max_run_time: yield next_issue_time next_issue_time = utils.get_next_issue_time( fx, next_issue_time + pd.Timedelta('1ns'))
def generate_reference_persistence_forecast_parameters(session, forecasts, observations, max_run_time): """Sort through all *forecasts* to find those that should be generated by the Arbiter from persisting Observation values. The forecast must have ``'is_reference_persistence_forecast': true`` and an observation_id in Forecast.extra_parameters (formatted as a JSON string). A boolean value for "index_persistence" in Forecast.extra_parameters controls whether the persistence forecast should be made adjusting for clear-sky/AC power index or not. Parameters ---------- session : solarforecastarbiter.io.api.APISession forecasts : list of datamodel.Forecasts The forecasts that should be filtered to find references. observations : list of datamodel.Observations Observations that will are available to use to fetch values and make persistence forecasts. max_run_time : pandas.Timestamp The maximum run time/issue time for any forecasts. Usually now. Returns ------- generator of (Forecast, Observation, next_issue_time, index) """ user_info = session.get_user_info() observation_dict = {obs.observation_id: obs for obs in observations} for fx in forecasts: if not _is_reference_persistence_forecast(fx.extra_parameters): logger.debug( 'Forecast %s is not labeled as a reference ' 'persistence forecast', fx.forecast_id) continue if not fx.provider == user_info['organization']: logger.debug("Forecast %s is not in user's organization", fx.forecast_id) continue try: extra_parameters = json.loads(fx.extra_parameters) except json.JSONDecodeError: logger.warning( 'Failed to decode extra_parameters for %s: %s as JSON', fx.name, fx.forecast_id) continue try: observation_id = extra_parameters['observation_id'] except KeyError: logger.error( 'Forecast, %s: %s, has no observation_id to base forecasts' ' off of. Cannot make persistence forecast.', fx.name, fx.forecast_id) continue if observation_id not in observation_dict: logger.error( 'Observation %s not in set of given observations.' ' Cannot generate persistence forecast for %s: %s.', observation_id, fx.name, fx.forecast_id) continue observation = observation_dict[observation_id] index = extra_parameters.get('index_persistence', False) obs_mint, obs_maxt = session.get_observation_time_range(observation_id) if pd.isna(obs_maxt): # no observations to use anyway logger.info( 'No observation values to use for %s: %s from observation %s', fx.name, fx.forecast_id, observation_id) continue fx_mint, fx_maxt = session.get_forecast_time_range(fx.forecast_id) # find the next issue time for the forecast based on the last value # in the forecast series if pd.isna(fx_maxt): # if there is no forecast yet, go back a bit from the last # observation. Don't use the start of observations, since it # could really stress the workers if we have a few years of # data before deciding to make a persistence fx next_issue_time = utils.get_next_issue_time( fx, obs_maxt - fx.run_length) else: next_issue_time = utils.find_next_issue_time_from_last_forecast( fx, fx_maxt) # now find all the run times that can be made based on the # last observation timestamp while next_issue_time <= max_run_time: data_start, data_end = utils.get_data_start_end( observation, fx, next_issue_time) if data_end > obs_maxt: break if data_start > obs_mint: yield (fx, observation, next_issue_time, index) next_issue_time = utils.get_next_issue_time( fx, next_issue_time + pd.Timedelta('1ns'))
def test_find_next_issue_time_from_last_forecast(fxargs, last_time, expected, site_metadata): fx = default_forecast(site_metadata, **fxargs) out = utils.find_next_issue_time_from_last_forecast(fx, last_time) assert out == expected
def generate_reference_persistence_forecast_parameters(session, forecasts, observations, max_run_time): """Sort through all *forecasts* to find those that should be generated by the Arbiter from persisting Observation values. The forecast must have ``'is_reference_persistence_forecast': true`` and an observation_id in Forecast.extra_parameters (formatted as a JSON string). A boolean value for "index_persistence" in Forecast.extra_parameters controls whether the persistence forecast should be made adjusting for clear-sky/AC power index or not. Parameters ---------- session : solarforecastarbiter.io.api.APISession forecasts : list of datamodel.Forecasts The forecasts that should be filtered to find references. observations : list of datamodel.Observations Observations that will are available to use to fetch values and make persistence forecasts. max_run_time : pandas.Timestamp The maximum run time/issue time for any forecasts. Usually now. Returns ------- generator of (Forecast, Observation, index, data_start, issue_times) """ user_info = session.get_user_info() observation_dict = {obs.observation_id: obs for obs in observations} out = namedtuple( 'PersistenceParameters', ['forecast', 'observation', 'index', 'data_start', 'issue_times']) for fx in forecasts: obs_ind_mint_maxt = _ref_persistence_check(fx, observation_dict, user_info, session) if obs_ind_mint_maxt is None: continue observation, index, obs_mint, obs_maxt = obs_ind_mint_maxt # probably split this out to generate issues times for only gaps vs # latest if isinstance(fx, datamodel.ProbabilisticForecast): fx_mint, fx_maxt = \ session.get_probabilistic_forecast_constant_value_time_range( fx.constant_values[0].forecast_id) else: fx_mint, fx_maxt = session.get_forecast_time_range(fx.forecast_id) # find the next issue time for the forecast based on the last value # in the forecast series if pd.isna(fx_maxt): # if there is no forecast yet, go back a bit from the last # observation. Don't use the start of observations, since it # could really stress the workers if we have a few years of # data before deciding to make a persistence fx next_issue_time = utils.get_next_issue_time( fx, obs_maxt - fx.run_length) else: next_issue_time = utils.find_next_issue_time_from_last_forecast( fx, fx_maxt) data_start, _ = utils.get_data_start_end(observation, fx, next_issue_time, next_issue_time) issue_times = tuple( _issue_time_generator(observation, fx, obs_mint, obs_maxt, next_issue_time, max_run_time)) if len(issue_times) == 0: continue yield out(fx, observation, index, data_start, issue_times)