def table_annual_energy(self):
"""Creates a table summarizing annual energy fron generation, imports, and load.
Returns
-------
df: Dataframe
Energy table [TWh]
"""
# Power [MW]
load = NYISOData(dataset="load_5m",
year=self.year).df.tz_convert("US/Eastern")["NYCA"]
fuel_mix = NYISOData(dataset="fuel_mix_5m",
year=self.year).df.tz_convert("US/Eastern")
imports = NYISOData(dataset="interface_flows_5m",
year=self.year).df.tz_convert("US/Eastern")
imports.drop(
("External Flows", "SCH - HQ IMPORT EXPORT", "Flow (MW)"),
axis="columns",
inplace=True,
) # HQ Net is a subset of another external flow
imports = imports.loc[:, ("External Flows", slice(None), "Flow (MW)")]
# Energy Converstion [MWh] and Resampling By Summing Energy
load = (load * 1 / 12).sum(axis="index").sum() / (10**6
) # MW->MWh->TWh
fuel_mix = (fuel_mix * 1 / 12).sum(axis="index") / (10**6)
imports = (imports * 1 / 12).sum(axis="index").sum() / (10**6)
fuel_mix = fuel_mix.to_frame()
fuel_mix = fuel_mix.rename(columns={
0: f'Historic ({self.year})'
}).sort_values(f'Historic ({self.year})', ascending=False)
#reorder carbon free resources first
carbon_free_resources = [
'Hydro', 'Wind', 'Other Renewables', 'Nuclear'
]
df = fuel_mix.loc[carbon_free_resources]
df = pd.concat([
df,
fuel_mix.loc[[
ind for ind in fuel_mix.index
if ind not in carbon_free_resources
]],
])
df.loc["Total Generation"] = fuel_mix.sum()
df.loc["Total Renewable Generation"] = fuel_mix.loc[[
"Hydro", "Other Renewables", "Wind"
]].sum()
df.loc["Total Carbon-Free Generation"] = fuel_mix.loc[[
"Nuclear", "Hydro", "Other Renewables", "Wind"
]].sum()
df.loc["Imports"] = imports
df.loc["Total Generation + Imports"] = (df.loc["Imports"] +
df.loc["Total Generation"])
df.loc["Load"] = load
df[f"Historic ({self.year}) [% of Load]"] = df / load * 100
return df
def table_annual_energy(self):
"""Creates a table summarizing annual energy fron generation, imports, and load.
Returns
-------
df: Dataframe
Energy table [TWh]
"""
#Power [MW]
load = NYISOData(dataset='load_5m',
year=self.year).df.tz_convert('US/Eastern')['NYCA']
fuel_mix = NYISOData(dataset='fuel_mix_5m',
year=self.year).df.tz_convert('US/Eastern')
imports = NYISOData(dataset='interface_flows_5m',
year=self.year).df.tz_convert('US/Eastern')
imports.drop(
('External Flows', 'SCH - HQ IMPORT EXPORT', 'Flow (MW)'),
axis='columns',
inplace=True) #HQ Net is a subset of another external flow
imports = imports.loc[:, ('External Flows', slice(None), 'Flow (MW)')]
#Energy Converstion [MWh] and Resampling By Summing Energy
load = (load * 1 / 12).sum(axis='index').sum() / (10**6) #MW->MWh->TWh
fuel_mix = (fuel_mix * 1 / 12).sum(axis='index') / (10**6)
imports = (imports * 1 / 12).sum(axis='index').sum() / (10**6)
fuel_mix = fuel_mix.to_frame()
fuel_mix = fuel_mix.rename(columns={
0: f'Historic ({self.year})'
}).sort_values(f'Historic ({self.year})', ascending=False)
#reorder carbon free resources first
carbon_free_resources = [
'Nuclear', 'Hydro', 'Other Renewables', 'Wind'
]
df = fuel_mix.loc[carbon_free_resources]
df = pd.concat([
df, fuel_mix.loc[[
ind for ind in fuel_mix.index
if ind not in carbon_free_resources
]]
])
df.loc['Total Generation'] = fuel_mix.sum()
df.loc['Total Renewable Generation'] = fuel_mix.loc[[
'Hydro', 'Other Renewables', 'Wind'
]].sum()
df.loc['Total Carbon-Free Generation'] = fuel_mix.loc[[
'Nuclear', 'Hydro', 'Other Renewables', 'Wind'
]].sum()
df.loc['Imports'] = imports
df.loc['Total Generation + Imports'] = df.loc['Imports'] + df.loc[
'Total Generation']
df.loc['Load'] = load
df[f'Historic ({self.year}) [% of Load]'] = df / load * 100
return df
def table_hourly_dataset(dataset, year):
"""Convert datasets from power to energy and resamples at hourly frequency
Returns
-------
df: Dataframe
Energy table [GWh]
"""
df = NYISOData(dataset=dataset,
year=year).df.tz_convert("US/Eastern") # MW
if nd.f == "5T":
df = (df * 1 / 12).resample("H").sum() / 1000 # MW->MWh->GWh
elif nd.f == "H":
df = df.sum() / 1000 # MW=MWh->GWh
return df
def fig_demand_pdf(self, cumulative=False):
"""Creates a figure of the year's demand probability density function."""
df = NYISOData(dataset="load_5m", year=self.year).df.tz_convert('US/Eastern')["NYCA"]/1000 #MW->GW
df = pd.DataFrame(df)
ax_kwargs = {"title":f'State-wide Demand Probability Distribution ({self.year})',
"xlabel":"Demand [GW]", "ylabel":"Probability [%]",
"xlim":(df.values.min(),df.values.max()), "ylim":None}
HISTPLOT_KWARGS.update({"cumulative":cumulative})
if cumulative:
ax_kwargs.update({"title": f'State-wide Demand Cumulative Probability Distribution ({self.year})'})
fig = pdf(df.rename(columns={'NYCA':"Values"}), ax_kwargs, HISTPLOT_KWARGS)
# Save
if cumulative:
file = pl.Path(self.out_dir,f'{self.year}_demand_cumulative_pdf.png')
else:
file = pl.Path(self.out_dir,f'{self.year}_demand_pdf.png')
fig.savefig(file, bbox_inches='tight', transparent=True)
def tables_energy_generation(self, f='D'):
"""Gathers datasets (in US/Eastern) needed to produce a few figures.
Parameters
----------
f: str
Frequency of graph to generate (daily ('D') and monthly ('M') recommended)
Returns
-------
tables: Dictionary
Dictionary containing dataset names as keys and respective Dataframes
"""
if f in ['Y']:
raise ValueError('Frequency Not Supported!')
#Power [MW]
load = NYISOData(dataset='load_5m', year=self.year).df.tz_convert('US/Eastern')['NYCA']
fuel_mix = NYISOData(dataset='fuel_mix_5m', year=self.year).df.tz_convert('US/Eastern')
imports = NYISOData(dataset='interface_flows_5m', year=self.year).df.tz_convert('US/Eastern')
imports = imports.loc[:, ('External Flows', slice(None), 'Flow (MW)')]
imports.drop(('External Flows', 'SCH - HQ IMPORT EXPORT', 'Flow (MW)'),
axis='columns', inplace=True) #'SCH - HQ IMPORT EXPORT' is a subset of another external flow
imports = imports.sum(axis='columns')
tables = {'load':load, 'fuel_mix':fuel_mix, 'imports': imports} # group datasets into dictionary to apply
def power_to_energy(df):
"""Energy Converstion [MWh] and Resampling By Summing Energy"""
return (df * 1 / 12).resample(f).sum() / 1000 # MW->MWh->GWh
tables = {k: power_to_energy(v) for k, v in tables.items()}
tables["fuel_mix"]["Imports"] = tables["imports"] # add imports to fuel mix
del tables["imports"] # remove imports from the tables list
order = [
"Nuclear",
"Hydro",
"Other Renewables",
"Wind",
"Natural Gas",
"Dual Fuel",
"Other Fossil Fuels",
"Imports",
]
tables["fuel_mix"] = tables["fuel_mix"][order] # fix order
def month_adj_object(df):
"""Adjust index for months and make index objects to label correctly"""
if f == "M":
df.index = df.index.shift(-1, "M").shift(1, "D")
df = df / 1000 # GWh->TWh
df.index = df.index.astype("O")
return df
tables = {k: month_adj_object(v) for k, v in tables.items()}
return tables
def fig_demand_forecast_error(self, cumulative=False):
load = NYISOData(dataset="load_h", year=self.year).df.tz_convert('US/Eastern')["NYCA"] #MW
load_forecast = NYISOData(dataset="load_forecast_h", year=self.year).df.tz_convert('US/Eastern')["NYCA"] #MW
load_forecast = load_forecast.loc[load.index]
df = (load_forecast-load)/load *100 # error [%]
df = pd.DataFrame(df)
ax_kwargs = {"title":f'State-wide Demand Forecast Error Probability Distribution ({self.year})',
"xlabel":"Demand [MW]", "ylabel":"Probability [%]",
"xlim":(df.values.min(),df.values.max()), "ylim":None}
HISTPLOT_KWARGS.update({"cumulative":cumulative})
if cumulative:
ax_kwargs.update({"title": f'State-wide Demand Forecast Error Cumulative Probability Distribution ({self.year})'})
fig = pdf(df.rename(columns={'NYCA':"Values"}), ax_kwargs, HISTPLOT_KWARGS)
# Save
if cumulative:
file = pl.Path(self.out_dir,f'{self.year}_demand_forecast_error_cumulative_pdf.png')
else:
file = pl.Path(self.out_dir,f'{self.year}_demand_forecast_error_pdf.png')
fig.savefig(file, bbox_inches='tight', transparent=True)
def table_instate_flow(self):
"""Gets the flow between the upstate and downstate region consistent with NYISO's definition
Returns
-------
df: Dataframe
Energy table [GWh]
"""
df = NYISOData(dataset='interface_flows_5m',
year=self.year).df.tz_convert('US/Eastern')
df = df[('Internal Flows', 'TOTAL EAST', 'Flow (MW)')]
df = (df * 1 / 12).resample('H').sum() / 1000 #MW->MWh->GWh
return df
def table_instate_flow(self):
"""Gets the flow between the upstate and downstate region consistent with NYISO's definition
Returns
-------
df: Dataframe
Energy table [GWh]
"""
df = NYISOData(dataset="interface_flows_5m",
year=self.year).df.tz_convert("US/Eastern")
df = df[("Internal Flows", "TOTAL EAST", "Flow (MW)")]
df = (df * 1 / 12).resample("H").sum() / 1000 # MW->MWh->GWh
return df
def dfs_energy(year='2019', f='D'):
if f in ['Y']:
raise ValueError('Frequency Not Supported!')
#Power [MW]
load = NYISOData(dataset='load_5m',
year=year).df.tz_convert('US/Eastern')['NYCA']
fuel_mix = NYISOData(dataset='fuel_mix_5m',
year=year).df.tz_convert('US/Eastern')
imports = NYISOData(dataset='interface_flows_5m',
year=year).df.tz_convert('US/Eastern')
imports = imports.loc[:, ('External Flows', slice(None), 'Flow (MW)')]
imports.drop(
('External Flows', 'SCH - HQ_IMPORT_EXPORT', 'Flow (MW)'),
axis='columns',
inplace=True
) #'SCH - HQ_IMPORT_EXPORT' is a subset of another external flow
imports = imports.sum(axis='columns')
dfs = {
'load': load,
'fuel_mix': fuel_mix,
'imports': imports
} # group datasets into dictionary to apply
def power_to_energy(df):
"""Energy Converstion [MWh] and Resampling By Summing Energy"""
return (df * 1 / 12).resample(f).sum() / 1000 #MW->MWh->GWh
dfs = {k: power_to_energy(v) for k, v in dfs.items()}
dfs['fuel_mix']['Imports'] = dfs['imports'] #add imports to fuel mix
del dfs['imports'] #remove imports from the dfs list
order = [
'Nuclear', 'Hydro', 'Other Renewables', 'Wind', 'Natural Gas',
'Dual Fuel', 'Other Fossil Fuels', 'Imports'
]
dfs['fuel_mix'] = dfs['fuel_mix'][order] #fix order
def month_adj_object(df):
"""Adjust index for months and make index objects to label correctly"""
if f == 'M':
df.index = df.index.shift(-1, 'M').shift(1, 'D')
df = df / 1000 # GWh->TWh
df.index = df.index.astype('O')
return df
dfs = {k: month_adj_object(v) for k, v in dfs.items()}
return dfs
def __init__(self, year='2019', out_dir = pl.Path(c_dir,'visualizations'), **kwargs):
"""
Parameters
----------
year: str, int
Year of dataset(s) to use for graphs
out_dir: Pathlib Object
Directory to save the figure to
kwargs:
Additional parameters to be passed to NYISOData initialization
"""
self.year = year
self.out_dir = out_dir
#if redownload is passed, update existing databases
#looks to see which datasets are available and redownloads them
if kwargs.get("redownload", False) or kwargs.get("reconstruct", False):
existing_dbs = pl.Path(DATABASE_DIR).glob(f"{self.year}*.pkl")
existing_datasets = [db.name.replace(".pkl","").replace(f"{self.year}_","") for db in existing_dbs]
for dataset in existing_datasets:
NYISOData(dataset=dataset, year=self.year, **kwargs)
def table_instate_flow(year):
"""Flow between Upstate and Downstate"""
df = NYISOData('interface_flows_5m', year).df.tz_convert('US/Eastern')
df = df[('Internal Flows', 'TOTAL EAST', 'Flow (MW)')]
df = (df * 1 / 12).resample('H').sum() / 1000 #MW->MWh->GWh
return df
def table_hourly_dataset(dataset, year):
df = NYISOData(dataset=dataset,
year=year).df.tz_convert('US/Eastern') #MW
df = (df * 1 / 12).resample('H').sum() / 1000 #MW->MWh->GWh
return df