def validate_dc_power(observation, values): """ Run a number of validation checks on a daily timeseries of DC power. Parameters ---------- observation : solarforecastarbiter.datamodel.Observation Observation object that the data is associated with values : pandas.Series Series of observation values Returns ------- timestamp_flag : pandas.Series Bitmask from :py:func:`.validator.check_timestamp_spacing` night_flag : pandas.Series Bitmask from :py:func:`.validator.check_day_night` or :py:func:`.validator.check_day_night_interval` limit_flag : pandas.Series Bitmask from :py:func:`.validator.check_dc_power_limits` """ solar_position, dni_extra, timestamp_flag, night_flag = _solpos_dni_extra( observation, values) day_night = \ ~quality_mapping.convert_mask_into_dataframe(night_flag)['NIGHTTIME'] dc_limit_flag = validator.check_dc_power_limits( values, day_night, observation.site.modeling_parameters.dc_capacity, _return_mask=True) return timestamp_flag, night_flag, dc_limit_flag
def generate_observation_figure(observation, data, limit=pd.Timedelta('3d')): """ Creates a bokeh figure from API responses for an observation Parameters ---------- observation : datamodel.Observation The Observation that is being plotted data : pandas.DataFrame The observation data to be plotted with datetime index and ('value', 'quality_flag') columns limit : pandas.Timedelta or None The time limit from the last datapoint to plot. If None, all data is plotted. Returns ------- None When the data is empty script, div : str When return_components = True, return the <script> and <div> components for the Bokeh plot. bokeh components from gridplot When return_components = False """ logger.info('Starting observation forecast generation...') if len(data.index) == 0: return None data = plot_utils.align_index(data, observation.interval_length, limit) quality_flag = data.pop('quality_flag').dropna().astype(int) bool_flags = quality_mapping.convert_mask_into_dataframe(quality_flag) active_flags = quality_mapping.convert_flag_frame_to_strings(bool_flags) active_flags.name = 'active_flags' flags = bool_flags.mask(~bool_flags).reindex(data.index) # add missing flags['MISSING'] = pd.Series(1.0, index=data.index)[pd.isna(data['value'])] # need to fill as line needs more than a single point to show up if observation.interval_label == 'ending': flags.bfill(axis=0, limit=1, inplace=True) else: # for interval beginning and instantaneous flags.ffill(axis=0, limit=1, inplace=True) cds = ColumnDataSource(pd.concat([data, flags, active_flags], axis=1)) figs = [ make_basic_timeseries(cds, observation.name, observation.variable, observation.interval_label, PLOT_WIDTH) ] figs.extend(make_quality_bars(cds, PLOT_WIDTH, figs[0].x_range)) layout = _make_layout(figs) logger.info('Figure generated succesfully') return layout
def test_convert_mask_into_dataframe(): flags = (pd.Series([0, 0, 1, 1 << 12, 1 << 9 | 1 << 7 | 1 << 5]) | quality_mapping.LATEST_VERSION_FLAG) expected = pd.DataFrame([[0] * 12, [0] * 12, [1] + [0] * 11, [0] * 9 + [1, 0, 0], [0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0]], columns=DESCRIPTIONS + ['NOT VALIDATED'], dtype=bool) out = quality_mapping.convert_mask_into_dataframe(flags) assert_frame_equal(out, expected)
def test_convert_mask_into_dataframe_w_unvalidated(): flags = (pd.Series([0, 0, 1, 1 << 12, 1 << 9 | 1 << 7 | 1 << 5]) | quality_mapping.LATEST_VERSION_FLAG) flags.iloc[0] = 0 columns = DESCRIPTIONS + ['NOT VALIDATED'] + DERIVED_DESCRIPTIONS expected = pd.DataFrame( [[0] * 12 + [1, 0, 0, 0], [0] * 13 + [1, 0, 0], [1] + [0] * 12 + [1, 0, 0], [0] * 9 + [1, 0, 0, 0, 1, 0, 0], [0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0]], columns=columns, dtype=bool) out = quality_mapping.convert_mask_into_dataframe(flags) assert_frame_equal(out, expected, check_like=True)
def _resample_event_obs(obs, fx, obs_data, quality_flags): """Resample the event observation. Parameters ---------- obs : datamodel.Observation The Observation being resampled. fx : datamodel.EventForecast The corresponding Forecast. obs_data : pd.Series Timeseries data of the event observation. quality_flags : tuple of solarforecastarbiter.datamodel.QualityFlagFilter Flags to process and apply as filters during resampling. Returns ------- obs_resampled : pandas.Series Timeseries data of the Observation resampled to match the Forecast. counts : dict Dict where keys are quality_flag.quality_flags and values are integers indicating the number of points filtered for the given flag. Raises ------ ValueError If the Forecast and Observation do not have the same interval length. """ if fx.interval_length != obs.interval_length: raise ValueError("Event observation and forecast time-series " "must have matching interval length.") # bools w/ has columns like NIGHTTIME, CLEARSKY EXCEEDED, but many of # these are not valid for event obs! Arguably only USER FLAGGED and # NIGHTTIME are valid for event obs. obs_flags = quality_mapping.convert_mask_into_dataframe( obs_data['quality_flag']) obs_flags['ISNAN'] = obs_data['value'].isna() # determine the points that should never contribute # combine unique elements of tuple of tuples discard_before_resample_flags = set(['ISNAN']) for f in filter(lambda x: x.discard_before_resample, quality_flags): discard_before_resample_flags |= set(f.quality_flags) discard_before_resample = obs_flags[discard_before_resample_flags] counts = discard_before_resample.astype(int).sum(axis=0).to_dict() to_discard_before_resample = discard_before_resample.any(axis=1) obs_resampled = obs_data[~to_discard_before_resample] return obs_resampled, counts
def apply_validation(obs_df, qfilter, handle_func): """ Apply validation steps based on provided filters to the data. Parameters ---------- obs_df : pandas.DataFrame The observation data with 'value' and 'quality_flag' columns qfilter : solarforecastarbiter.datamodel.QualityFlagFilter handle_func : function Function that handles how `quality_flags` will be used. See solarforecastarbiter.metrics.preprocessing.exclude as an example. Returns ------- validated_obs : pandas.Series The validated timeseries data as pandas.Series. counts : dict Dict where keys are qfilter.quality_flags and values are integers indicating the number of points filtered for the given flag. """ # List of flags from filter if not isinstance(qfilter, datamodel.QualityFlagFilter): raise TypeError(f"{qfilter} not a QualityFlagFilter") filters = qfilter.quality_flags if obs_df.empty: return obs_df.value, {f: 0 for f in filters} else: validation_df = quality_mapping.convert_mask_into_dataframe( obs_df['quality_flag']) validation_df = validation_df[list(filters)] validated_obs = handle_func(obs_df.value, validation_df) counts = validation_df.astype(int).sum(axis=0).to_dict() return validated_obs, counts
def test_convert_mask_into_dataframe_all_unvalidated(): flags = pd.Series([0, 0, 1, 1, 0]) columns = ['NOT VALIDATED'] expected = pd.DataFrame([[1]] * 5, columns=columns, dtype=bool) out = quality_mapping.convert_mask_into_dataframe(flags) assert_frame_equal(out, expected, check_like=True)
def _resample_obs(obs, fx, obs_data, quality_flags): """Resample observations. Parameters ---------- obs : datamodel.Observation The Observation being resampled. fx : datamodel.Forecast The corresponding Forecast. obs_data : pandas.DataFrame Timeseries of values and quality flags of the observation/aggregate data. quality_flags : tuple of solarforecastarbiter.datamodel.QualityFlagFilter Flags to process and apply as filters during resampling. Returns ------- obs_resampled : pandas.Series The observation time series resampled to match the forecast interval_length. Time series will have missing labels where values failed validation. counts : dict Dict where keys are quality_flag.quality_flags and values are integers indicating the number of points filtered for the given flag. Raises ------ ValueError If fx.interval_length < obs.interval_length """ if fx.interval_length < obs.interval_length: # typically impossible to reach this because ForecastObservation init # prevents it raise ValueError( 'Cannot resample observation to match forecast because ' 'fx.interval_length < obs.interval_length.') if obs_data.empty: return obs_data['value'], [] # label convention when resampling closed = datamodel.CLOSED_MAPPING[obs.interval_label] # bools w/ has columns like NIGHTTIME, CLEARSKY EXCEEDED obs_flags = quality_mapping.convert_mask_into_dataframe( obs_data['quality_flag']) obs_flags['ISNAN'] = obs_data['value'].isna() # determine the points that should be discarded before resampling. to_discard_before_resample, val_results = _calc_discard_before_resample( obs_flags, quality_flags, fx.interval_length, closed) # resample using all of the data except for what was flagged by the # discard before resample process. resampled_values = \ obs_data.loc[~to_discard_before_resample, 'value'].resample( fx.interval_length, closed=closed, label=closed).mean() # determine the intervals that have too many flagged points to_discard_after_resample, after_resample_val_results = \ _calc_discard_after_resample( obs_flags, quality_flags, to_discard_before_resample, fx.interval_length, obs.interval_length, closed ) # discard the intervals with too many flagged sub-interval points. # resampled_values.index does not contain labels for intervals for # which all points were discarded, so care is needed in the next # indexing operation. good_labels = to_discard_after_resample.index[~to_discard_after_resample] obs_resampled = resampled_values.loc[resampled_values.index.intersection( good_labels)] # merge the val_results lists val_results += after_resample_val_results return obs_resampled, val_results