def test_energy_from_power_single_value_instantaneous():
power = pd.Series([1], pd.date_range('2019-01-01', periods=1, freq='15T'))
power.index.freq = None
match = (
"power_type='instantaneous' is incompatible with single element power. "
"Use power_type='right-labeled'")
with pytest.raises(ValueError, match=match):
energy_from_power(power, power_type='instantaneous')
def test_energy_from_power_max_timedelta_edge_case():
times = pd.date_range('2020-01-01 12:00', periods=4, freq='15T')
power = pd.Series(1, index=times)
power = power.drop(power.index[2])
result = energy_from_power(power,
'30T',
max_timedelta=pd.to_timedelta('20 minutes'))
assert result.isnull().all()
def test_energy_from_power_downsample(power):
expected = power.resample('20T').asfreq()
expected = expected.iloc[1:]
expected = pd.Series([0.75, 0.833333333, 0.416666667],
index=expected.index)
expected.name = 'energy_Wh'
result = energy_from_power(power, target_frequency='20T')
pd.testing.assert_series_equal(result, expected)
def test_energy_from_power_max_timedelta_inference(power):
expected = power.iloc[1:] * 0.25
expected.name = 'energy_Wh'
expected.iloc[:2] = np.nan
match = 'Fraction of excluded data (.*) exceeded threshold'
with pytest.warns(UserWarning, match=match):
result = energy_from_power(power.drop(power.index[1]))
pd.testing.assert_series_equal(result, expected)
def test_energy_from_power_leading_nans():
# GH 244
power = pd.Series(1, pd.date_range('2019-01-01', freq='15min', periods=5))
power.iloc[:2] = np.nan
expected_result = pd.Series([np.nan, np.nan, 0.25, 0.25],
index=power.index[1:],
name='energy_Wh')
result = energy_from_power(power)
pd.testing.assert_series_equal(result, expected_result)
def test_energy_from_power_downsample():
times = pd.date_range('2018-04-01 12:00', '2018-04-01 13:00', freq='15T')
time_series = pd.Series(data=[1.0, 2.0, 3.0, 4.0, 5.0], index=times)
expected_energy_series = pd.Series(index=[pd.to_datetime('2018-04-01 13:00:00')],
data=3.0, name='energy_Wh')
expected_energy_series.index.freq = '60T'
result = energy_from_power(time_series, '60T')
pd.testing.assert_series_equal(result, expected_energy_series)
def test_energy_from_power_downsample_max_timedelta_not_exceeded():
times = pd.date_range('2018-04-01 12:00', '2018-04-01 13:00', freq='15T')
time_series = pd.Series(data=[1.0, 2.0, 3.0, 4.0, 5.0], index=times)
expected_energy_series = pd.Series(index=[pd.to_datetime('2018-04-01 13:00:00')],
data=3.0, name='energy_Wh')
expected_energy_series.index.freq = '60T'
result = energy_from_power(time_series.drop(time_series.index[2]), '60T', pd.to_timedelta('60 minutes'))
pd.testing.assert_series_equal(result, expected_energy_series)
def test_energy_from_power_calculation():
power_times = pd.date_range('2018-04-01 12:00', '2018-04-01 13:00', freq='15T')
result_times = power_times[1:]
power_series = pd.Series(data=4.0, index=power_times)
expected_energy_series = pd.Series(data=1.0, index=result_times)
expected_energy_series.name = 'energy_Wh'
result = energy_from_power(power_series, max_timedelta=pd.to_timedelta('15 minutes'))
pd.testing.assert_series_equal(result, expected_energy_series)
def test_energy_from_power_for_issue_107():
times = pd.date_range('2018-04-01 12:00', '2018-04-01 16:00', freq='15T')
dc_power = pd.Series(index=times, data=1.0)
dc_power = dc_power.drop(dc_power.index[5:12])
expected_times = pd.date_range('2018-04-01 13:00', '2018-04-01 16:00', freq='60T')
expected_energy_series = pd.Series(index=expected_times,
data=[1.0, np.nan, np.nan, 1.0],
name='energy_Wh')
result = energy_from_power(dc_power, '60T')
pd.testing.assert_series_equal(result, expected_energy_series)
def test_energy_from_power_upsample_maxtimedelta_exceeded():
times = pd.date_range('2018-04-01 12:00', '2018-04-01 13:30', freq='30T')
time_series = pd.Series(data=[1.0, 3.0, 5.0, 6.0], index=times)
expected_result_times = pd.date_range('2018-04-01 12:15', '2018-04-01 13:30', freq='15T')
expected_energy_series = pd.Series(index=expected_result_times,
data=[np.nan, np.nan, np.nan, np.nan, 1.3125, 1.4375],
name='energy_Wh')
result = energy_from_power(time_series.drop(time_series.index[1]), '15T', pd.to_timedelta('30 minutes'))
pd.testing.assert_series_equal(result, expected_energy_series)
def test_energy_from_power_max_interval():
power_times = pd.date_range('2018-04-01 12:00', '2018-04-01 13:00', freq='15T')
result_times = power_times[1:]
power_series = pd.Series(data=4.0, index=power_times)
expected_energy_series = pd.Series(data=np.nan, index=result_times)
expected_energy_series.name = 'energy_Wh'
result = energy_from_power(power_series, max_timedelta=pd.to_timedelta('5 minutes'))
# We expect series of NaNs, because max_interval_hours is smaller than the
# time step of the power time series
pd.testing.assert_series_equal(result, expected_energy_series)
def test_energy_from_power_single_argument():
power_times = pd.date_range('2018-04-01 12:00', '2018-04-01 15:00', freq='15T')
result_times = power_times[1:]
power_series = pd.Series(data=4.0, index=power_times)
missing = pd.to_datetime('2018-04-01 13:00:00')
power_series = power_series.drop(missing)
expected_energy_series = pd.Series(data=1.0, index=result_times)
expected_nan = [missing]
expected_nan.append(pd.to_datetime('2018-04-01 13:15:00'))
expected_energy_series.loc[expected_nan] = np.nan
expected_energy_series.name = 'energy_Wh'
# Test that the result has the expected missing timestamp based on median timestep
result = energy_from_power(power_series)
pd.testing.assert_series_equal(result, expected_energy_series)
def _combine_losses(self, rollup_period='M'):
"""
Combine subsystem and system losses.
Sets the `loss_total` and `results` attributes.
Parameters
----------
rollup_period : pandas offset string, default 'M'
The period on which to roll up losses and calculate availability.
"""
if ((self.loss_system > 0) & (self.loss_subsystem > 0)).any():
msg = (
'Loss detected simultaneously at both system and subsystem '
'levels. This is unexpected and could indicate a problem with '
'the input time series data.')
warnings.warn(msg, UserWarning)
self.loss_total = self.loss_system + self.loss_subsystem
# calculate actual production based on corrected cumulative meter
cumulative_energy = self.energy_cumulative_corrected
resampled_cumulative = cumulative_energy.resample(rollup_period)
actual_production = (resampled_cumulative.last() -
resampled_cumulative.first())
lost_production = rdtools.energy_from_power(self.loss_total)
df = pd.DataFrame({
'lost_production':
lost_production.resample(rollup_period).sum(),
'actual_production':
actual_production,
})
loss_plus_actual = df['lost_production'] + df['actual_production']
df['availability'] = 1 - df['lost_production'] / loss_plus_actual
self.results = df
def test_energy_from_power_max_timedelta_inference(power):
expected = power.iloc[1:] * 0.25
expected.name = 'energy_Wh'
expected.iloc[:2] = np.nan
result = energy_from_power(power.drop(power.index[1]))
pd.testing.assert_series_equal(result, expected)
def _calc_error_distributions(self, quantiles):
"""
Calculate the error distributions of Section II-A in [1]_.
Sets the `power_expected_rescaled`, `energy_expected_rescaled`,
`error_info`, `interp_lower`, and `interp_upper` attributes.
Parameters
----------
quantiles : 2-element tuple, default (0.01, 0.99)
The quantiles of the error distribution used for the expected
energy confidence interval. The lower bound is used to classify
outages as either (1) a simple communication interruption with
no production loss or (2) a power outage with an associated
production loss estimate.
"""
df = pd.DataFrame({
'Meter_kW': self.power_system,
'Expected Power': self.power_expected,
'Meter_kWh': self.energy_cumulative,
})
system_performing_normally = ((self.loss_subsystem == 0) &
(self.power_system > 0))
# filter out nighttime as well, since night intervals shouldn't count
subset = system_performing_normally & (df['Expected Power'] > 0)
# rescale expected energy to better match actual production.
# this shifts the error distributions so that as interval length
# increases, error -> 0
scaling_subset = df.loc[subset, ['Expected Power', 'Meter_kW']].sum()
scaling_factor = (scaling_subset['Expected Power'] /
scaling_subset['Meter_kW'])
df['Expected Power'] /= scaling_factor
self.power_expected_rescaled = df['Expected Power']
df['Expected Energy'] = rdtools.energy_from_power(df['Expected Power'])
self.energy_expected_rescaled = df['Expected Energy']
df['Meter_kWh_interval'] = rdtools.energy_from_power(df['Meter_kW'])
df_subset = df.loc[subset, :]
# window length is "number of daytime intervals".
# Note: these bounds are intended to provide good resolution
# across many dataset lengths
window_lengths = 2**np.arange(1, int(np.log2(len(df_subset))), 1)
results_list = []
for window_length in window_lengths:
rolling = df_subset.rolling(window=window_length, center=True)
window = rolling.sum()
actual = window['Meter_kWh_interval']
expected = window['Expected Energy']
# remove the nans at beginning and end because of minimum window
# length
actual = actual[~np.isnan(actual)]
expected = expected[~np.isnan(expected)]
temp = pd.DataFrame({
'actual': actual,
'expected': expected,
'window length': window_length
})
results_list.append(temp)
df_error = pd.concat(results_list)
df_error['error'] = df_error['actual'] / df_error['expected'] - 1
self.error_info = df_error
error = df_error.groupby('window length')['error']
lower = error.quantile(quantiles[0])
upper = error.quantile(quantiles[1])
# functions to predict the confidence interval for a given outage
# length. linear interp inside the range, nearest neighbor outside the
# range.
def interp(series):
return interp1d(series.index,
series.values,
fill_value=(series.values[0], series.values[-1]),
bounds_error=False)
# functions mapping number of intervals (outage length) to error bounds
def interp_lower(n_intervals):
return float(interp(lower)(n_intervals))
def interp_upper(n_intervals):
return float(interp(upper)(n_intervals))
self.interp_lower = interp_lower
self.interp_upper = interp_upper
def _calc_loss_system(self):
"""
Estimate total production loss from system downtime events.
See Section II-B in [1]_.
This implements the "expected energy" method from [1]_ of comparing
system production recovered from cumulative production data with
expected production from an energy model.
This function is useful for full system outages when no system data is
available at all. However, it does require cumulative production data
recorded at the device level and only reports estimated lost production
for entire outages rather than timeseries lost power.
Sets the `outage_info`, `energy_cumulative_corrected`, and
`loss_system` attributes.
"""
# Calculate boolean series to indicate full outages. Considerations:
# - Multi-day outages need to span across nights
# - Full outages don't always take out communications, so the
# cumulative meter can either drop out or stay constant depending on
# the case.
# During a full outage, no inverters will report production:
looks_offline = ~self.reporting_mask.any(axis=1)
# Now span across nights:
all_times = self.power_system.index
masked = looks_offline[self.power_expected > 0].reindex(all_times)
# Note: in Series, (nan | True) is False, but (True | nan) is True
full_outage = (masked.ffill().fillna(False)
| masked.bfill().fillna(False))
# Find expected production and associated uncertainty for each outage
diff = full_outage.astype(int).diff()
starts = all_times[diff == 1].tolist()
ends = all_times[diff.shift(-1) == -1].tolist()
steps = diff[~diff.isnull() & (diff != 0)]
if not steps.empty:
if steps[0] == -1:
# data starts in an outage
starts.insert(0, all_times[0])
if steps[-1] == 1:
# data ends in an outage
ends.append(all_times[-1])
outage_data = []
for start, end in zip(starts, ends):
outage_expected_power = self.power_expected_rescaled[start:end]
daylight_intervals = (outage_expected_power > 0).sum()
outage_expected_energy = self.energy_expected_rescaled[start:end]
# self.cumulative_energy[start] is the first value in the outage.
# so to get the starting energy, need to get previous value:
start_minus_one = all_times[all_times.get_loc(start) - 1]
data = {
'start': start,
'end': end,
'duration': end - start,
'intervals': len(outage_expected_power),
'daylight_intervals': daylight_intervals,
'error_lower': self.interp_lower(daylight_intervals),
'error_upper': self.interp_upper(daylight_intervals),
'energy_expected': outage_expected_energy.sum(),
'energy_start': self.energy_cumulative[start_minus_one],
'energy_end': self.energy_cumulative[end],
}
outage_data.append(data)
# specify columns in case no outages are found. Also specifies
# the order for pandas < 0.25.0
cols = [
'start', 'end', 'duration', 'intervals', 'daylight_intervals',
'error_lower', 'error_upper', 'energy_expected', 'energy_start',
'energy_end'
]
df_outages = pd.DataFrame(outage_data, columns=cols)
df_outages['energy_actual'] = (df_outages['energy_end'] -
df_outages['energy_start'])
# poor-quality cumulative meter data can create "negative production"
# outages. Set to nan so that negative value doesn't pollute other
# calcs. However, if using a net meter (instead of delivered), system
# consumption creates a legitimate decrease during some outages. Rule
# of thumb is that system consumption is about 0.5% of system
# production, but it'll be larger during winter. Choose 5% to be safer.
lower_limit = -0.05 * df_outages['energy_expected'] # Note the sign
below_limit = df_outages['energy_actual'] < lower_limit
df_outages.loc[below_limit, 'energy_actual'] = np.nan
df_outages['ci_lower'] = ((1 + df_outages['error_lower']) *
df_outages['energy_expected'])
df_outages['ci_upper'] = ((1 + df_outages['error_upper']) *
df_outages['energy_expected'])
df_outages['type'] = np.where(
df_outages['energy_actual'] < df_outages['ci_lower'], 'real',
'comms')
df_outages.loc[df_outages['energy_actual'].isnull(),
'type'] = 'unknown'
df_outages['loss'] = np.where(
df_outages['type'] == 'real',
df_outages['energy_expected'] - df_outages['energy_actual'], 0)
df_outages.loc[df_outages['type'] == 'unknown', 'loss'] = np.nan
self.outage_info = df_outages
# generate a best-guess timeseries loss for the full outages by
# scaling the expected power signal to match the actual
lost_power_full = pd.Series(0, index=self.loss_subsystem.index)
expected_power = self.power_expected
corrected_cumulative_energy = self.energy_cumulative.copy()
for i, row in self.outage_info.iterrows():
start = row['start']
end = row['end']
subset = expected_power.loc[start:end].copy()
subset_energy = rdtools.energy_from_power(subset)
loss_fill = subset * row['loss'] / subset_energy.sum()
lost_power_full.loc[subset.index] += loss_fill
# fill in the cumulative meter during the outages, again using
# the expected energy signal, but rescaled to match actual
# production this time:
production_fill = subset_energy.cumsum()
production_fill *= row['energy_actual'] / subset_energy.sum()
corrected_segment = row['energy_start'] + production_fill
corrected_cumulative_energy.loc[start:end] = corrected_segment
self.energy_cumulative_corrected = corrected_cumulative_energy
self.loss_system = lost_power_full
def test_energy_from_power_instantaneous(power):
expected = (0.25 * (power + power.shift()) / 2).dropna()
expected.name = 'energy_Wh'
result = energy_from_power(power, power_type='instantaneous')
pd.testing.assert_series_equal(result, expected)
def test_energy_from_power_single_arg(power):
expected = power.iloc[1:] * 0.25
expected.name = 'energy_Wh'
result = energy_from_power(power)
pd.testing.assert_series_equal(result, expected)
def test_energy_from_power_max_timedelta(power):
expected = power.iloc[1:] * 0.25
expected.name = 'energy_Wh'
result = energy_from_power(power.drop(power.index[1]),
max_timedelta=pd.to_timedelta('30 minutes'))
pd.testing.assert_series_equal(result, expected)
def test_energy_from_power_upsample(power):
expected = power.resample('10T').asfreq().interpolate() / 6
expected = expected.iloc[1:]
expected.name = 'energy_Wh'
result = energy_from_power(power, target_frequency='10T')
pd.testing.assert_series_equal(result, expected)
def test_energy_from_power_single_value_input():
times = pd.date_range('2019-01-01', freq='15T', periods=1)
power = pd.Series([100.], index=times)
expected_result = pd.Series([25.], index=times, name='energy_Wh')
result = energy_from_power(power)
pd.testing.assert_series_equal(result, expected_result)
def test_energy_from_power_validation():
power_series = pd.Series(data=[4.0] * 4)
with pytest.raises(ValueError):
energy_from_power(power_series, max_timedelta=pd.to_timedelta('15 minutes'))
def test_energy_from_power_single_value_input_no_freq():
power = pd.Series([1], pd.date_range('2019-01-01', periods=1, freq='15T'))
power.index.freq = None
match = "Could not determine period of input power"
with pytest.raises(ValueError, match=match):
energy_from_power(power)
