def create_ModelData(rts_gmlc_dir:str,
begin_time:Union[datetime,str], end_time:Union[datetime,str],
simulation:str="DAY_AHEAD", t0_state:Optional[dict] = None) -> md.ModelData:
"""
Create a ModelData object from the RTS-GMLC data.
Parameters
----------
rts_gmlc_dir : str
Path to directory holding csv files in RTS-GMLC format (bus.csv, gen.csv, etc).
begin_time : datetime.datetime or str
Beginning of time horizon. If str, date/time in "YYYY-MM-DD HH:MM:SS" or "YYYY-MM-DD" format,
the later of which assumes a midnight start.
end_time : datetime.datetime or str
End of time horizon. If str, date/time in "YYYY-MM-DD HH:MM:SS" or "YYYY-MM-DD" format,
the later of which assumes a midnight start.
simulation : str
Either "DAY_AHEAD" or "REAL_TIME", which specifies which time series the data is taken from,
default is "DAY_AHEAD".
t0_state : dict or None
Keys of this dict are thermal generator names, each element of which is another dictionary with
keys "initial_status", "initial_p_output", and "initial_q_output", which specify whether the
generator is on at t0, the real power output at t0, and the reactive power output at t0.
If t0_state is None, values are read from initial_status.csv in the rts_gmlc_dir.
If that file does not exist, no initial state data is set in the model.
Returns
-------
egret.model_data.ModelData
Returns a ModelData object with the timeseries data specified
"""
return md.ModelData(create_model_data_dict(rts_gmlc_dir, begin_time, end_time, simulation, t0_state))
def pysp_instance_creation_callback(scenario_name, node_names, cb_data):
#print("Building instance for scenario =", scenario_name)
scennum = sputils.extract_num(scenario_name)
uc_model_params = pdp.get_uc_model()
# Now using cb_data
##path = os.environ[UC_NUMSCENS_ENV_VAR] + "scenarios_r1"
path = cb_data["path"]
scenario_data = DataPortal(model=uc_model_params)
scenario_data.load(filename=path + os.sep + "RootNode.dat")
scenario_data.load(filename=path + os.sep + "Node" + str(scennum) + ".dat")
scenario_params = uc_model_params.create_instance(scenario_data,
report_timing=False,
name=scenario_name)
scenario_md = md.ModelData(
pdp.create_model_data_dict_params(scenario_params, keep_names=True))
## TODO: use the "power_balance_constraints" for now. In the future, networks should be
## handled with a custom callback -- also consider other base models
scenario_instance = uc.create_tight_unit_commitment_model(
scenario_md, network_constraints='power_balance_constraints')
# hold over string attribute from Egret,
# causes warning wth LShaped/Benders
del scenario_instance.objective
return scenario_instance
def pysp_instance_creation_callback(scenario_name,
path=None,
scenario_count=None):
"""
Notes:
- The uc_cylinders.py code has a `scenario_count` kwarg that gets passed to
the spokes, but it seems to be unused here...
"""
#print("Building instance for scenario =", scenario_name)
scennum = sputils.extract_num(scenario_name)
uc_model_params = pdp.get_uc_model()
scenario_data = DataPortal(model=uc_model_params)
scenario_data.load(filename=path + os.sep + "RootNode.dat")
scenario_data.load(filename=path + os.sep + "Node" + str(scennum) + ".dat")
scenario_params = uc_model_params.create_instance(scenario_data,
report_timing=False,
name=scenario_name)
scenario_md = md.ModelData(
pdp.create_model_data_dict_params(scenario_params, keep_names=True))
## TODO: use the "power_balance_constraints" for now. In the future, networks should be
## handled with a custom callback -- also consider other base models
scenario_instance = uc.create_tight_unit_commitment_model(
scenario_md, network_constraints='power_balance_constraints')
# hold over string attribute from Egret,
# causes warning wth LShaped/Benders
del scenario_instance.objective
return scenario_instance
def create_ModelData(rts_gmlc_dir,
begin_time,
end_time,
simulation="DAY_AHEAD",
t0_state=None):
"""
Create a ModelData object from the RTS-GMLC data.
Parameters
----------
rts_gmlc_dir : str
Path to RTS-GMLC directory
begin_time : datetime.datetime or str
Beginning of time horizon. If str, date/time in "YYYY-MM-DD HH:MM:SS" or "YYYY-MM-DD" format,
the later of which assumes a midnight start.
end_time : datetime.datetime or str
End of time horizon. If str, date/time in "YYYY-MM-DD HH:MM:SS" or "YYYY-MM-DD" format,
the later of which assumes a midnight start.
simulation : str
Either "DAY_AHEAD" or "REAL_TIME", which specifies which time series the data is taken from,
default is "DAY_AHEAD".
t0_state : dict or Nonetype
Keys of this dict are thermal generator names, each element of which is another dictionary with
keys "initial_status", "initial_p_output", and "initial_q_output", which specify whether the
generator is on at t0, the real power output at t0, and the reactive power output at t0.
If this is None, default values are loaded.
Returns
-------
egret.model_data.ModelData
Returns a ModelData object with the timeseries data specified
"""
return md.ModelData(
create_model_data_dict(rts_gmlc_dir, begin_time, end_time, simulation,
t0_state))
def create_ModelData(pglib_uc_filename):
"""
Parse a pglib-uc input file into a ModelData object containing
the model_data dictionary
Parameters
----------
pglib_uc_filename : str
Path and filename of the pglib-uc input file you wish to load
Returns
-------
ModelData
"""
data = create_model_data_dict(pglib_uc_filename)
return md.ModelData(data)
def create_ModelData(dat_file):
'''
Create a ModelData object from a prescient dat file
Parameters
----------
dat_file : str
Path to prescient *.dat file
Returns
-------
egret.model_data.ModelData
Returns a ModelData object with the dat file data specified
'''
return md.ModelData(create_model_data_dict(dat_file))
def create_ModelData(matpower_filename):
"""
Parse a MATPOWER input file into a ModelData object containing
the model_data dictionary
Parameters
----------
matpower_filename : str
Path and filename of the matpower inp file you wish to load
Returns
-------
ModelData
"""
data = create_model_data_dict(matpower_filename)
return md.ModelData(data)
def create_model_data_dict(rts_gmlc_dir,
begin_time,
end_time,
simulation="DAY_AHEAD",
t0_state=None):
"""
Create a model_data dictionary from the RTS-GMLC data.
Parameters
----------
rts_gmlc_dir : str
Path to RTS-GMLC directory
begin_time : datetime.datetime or str
Beginning of time horizon. If str, date/time in "YYYY-MM-DD HH:MM:SS" or "YYYY-MM-DD" format,
the later of which assumes a midnight start.
end_time : datetime.datetime or str
End of time horizon. If str, date/time in "YYYY-MM-DD HH:MM:SS" or "YYYY-MM-DD" format,
the later of which assumes a midnight start.
simulation : str
Either "DAY_AHEAD" or "REAL_TIME", which specifies which time series the data is taken from,
default is "DAY_AHEAD".
t0_state : dict or Nonetype
Keys of this dict are thermal generator names, each element of which is another dictionary with
keys "initial_status", "initial_p_output", and "initial_q_output", which specify whether the
generator is on at t0, the real power output at t0, and the reactive power output at t0.
If this is None, default values are loaded.
Returns
-------
dict : A dictionary in the format required for the ModelData object.
"""
simulation = simulation.upper()
if simulation not in ["DAY_AHEAD", "REAL_TIME"]:
raise ValueError('simulation must be "DAY_AHEAD" or "REAL_TIME"')
base_dir = os.path.join(rts_gmlc_dir, 'RTS_Data', 'SourceData')
begin_time, end_time = _get_datetimes(begin_time, end_time)
TimeSeriesPointer = namedtuple(
'TimeSeriesPointer', ['Object', 'Simulation', 'Parameter', 'DataFile'],
verbose=False)
DateTimeValue = namedtuple('DateTimeValue', ['DateTime', 'Value'],
verbose=False)
Load = namedtuple('Load', ['DateTime', 'Area1', 'Area2', 'Area3'],
verbose=False)
timeseries_pointer_df = pd.read_table(os.path.join(
base_dir, "timeseries_pointers.csv"),
header=0,
sep=',')
time_delta = end_time - begin_time
hours = 24 * time_delta.days + math.ceil(time_delta.seconds / 3600.)
model_data = _create_rtsgmlc_skeleton(rts_gmlc_dir)
## create an object for easy iterating
md_obj = md.ModelData(model_data)
system = md_obj.data["system"]
elements = md_obj.data["elements"]
if simulation == "DAY_AHEAD":
system["time_period_length_minutes"] = 60
else:
system["time_period_length_minutes"] = 5
# compute aggregate load per area, and then compute
# load participation factors from each bus from that data.
region_total_load = {}
areas = ["Area" + str(i) for i in range(1, 4)]
for this_region in areas:
this_region_total_load = 0.0
## loads have exactly one bus
for name, load in md_obj.elements("load"):
bus = elements["bus"][load["bus"]]
if bus["area"] == this_region:
this_region_total_load += load["p_load"]
region_total_load[this_region] = this_region_total_load
bus_load_participation_factor_dict = {}
bus_Ql_over_Pl_dict = {}
for name, load in md_obj.elements("load"):
bus = elements["bus"][load["bus"]]
bus_load_participation_factor_dict[
name] = load["p_load"] / region_total_load[bus["area"]]
bus_Ql_over_Pl_dict[name] = load["q_load"] / load["p_load"]
timeseries_pointer_dict = {}
for timeseries_pointer_index in timeseries_pointer_df.index.tolist():
this_timeseries_pointer_dict = timeseries_pointer_df.loc[
timeseries_pointer_index].to_dict()
new_timeseries_pointer = TimeSeriesPointer(
this_timeseries_pointer_dict["Object"],
this_timeseries_pointer_dict["Simulation"],
this_timeseries_pointer_dict["Parameter"],
os.path.join(base_dir, this_timeseries_pointer_dict["Data File"]))
timeseries_pointer_dict[(
new_timeseries_pointer.Object,
new_timeseries_pointer.Simulation)] = new_timeseries_pointer
filtered_timeseries = {}
for name, gen in md_obj.elements("generator", generator_type="renewable"):
if gen["fuel"] in ["Solar", "Wind", "Hydro"]:
if (name, simulation) not in timeseries_pointer_dict:
print(
"***WARNING - No timeseries pointer entry found for generator=%s"
% name)
else:
#print("Time series for generator=%s will be loaded from file=%s" % (name, timeseries_pointer_dict[(name,"DAY_AHEAD")].DataFile))
renewables_timeseries_df = _read_rts_gmlc_table(
timeseries_pointer_dict[(name, simulation)].DataFile,
simulation)
this_source_timeseries_df = renewables_timeseries_df.loc[:, [
"Year_Month_Day_Period", name
]]
this_source_timeseries_df = this_source_timeseries_df.rename(
columns={"Year_Month_Day_Period": "DateTime"})
start_mask = this_source_timeseries_df["DateTime"] >= begin_time
end_mask = this_source_timeseries_df["DateTime"] < end_time
this_source_masked_timeseries_df = this_source_timeseries_df[
start_mask & end_mask]
renewables_timeseries_dict = this_source_masked_timeseries_df.to_dict(
orient='split')
renewables_timeseries = []
for this_row in renewables_timeseries_dict["data"]:
renewables_timeseries.append(
DateTimeValue(this_row[0], float(this_row[1])))
filtered_timeseries[name] = renewables_timeseries
load_timeseries_spec = timeseries_pointer_dict[("Load", simulation)]
load_timeseries_df = _read_rts_gmlc_table(load_timeseries_spec.DataFile,
simulation)
load_timeseries_df = load_timeseries_df.rename(
columns={"Year_Month_Day_Period": "DateTime"})
start_mask = load_timeseries_df["DateTime"] >= begin_time
end_mask = load_timeseries_df["DateTime"] < end_time
masked_load_timeseries_df = load_timeseries_df[start_mask & end_mask]
load_dict = masked_load_timeseries_df.to_dict(orient='split')
load_timeseries = []
for load_row in load_dict["data"]:
load_timeseries.append(
Load(load_row[0], float(load_row[1]), float(load_row[2]),
float(load_row[3])))
reserves_dfs = {}
spin_reserve_categories = ["Spin_Up_R1", "Spin_Up_R2", "Spin_Up_R3"]
other_reserve_categories = [
"Reg_Down",
"Reg_Up",
]
## flexiramp products only in day-ahead simulation
if simulation == "DAY_AHEAD":
other_reserve_categories += [
"Flex_Down",
"Flex_Up",
]
for reserve in spin_reserve_categories:
reserves_dfs[reserve] = _read_rts_gmlc_table(
timeseries_pointer_dict[(reserve, simulation)].DataFile,
simulation)
reserves_dict = {}
for name, reserve_df in reserves_dfs.items():
reserve_df = reserve_df.rename(
columns={"Year_Month_Day_Period": "DateTime"})
start_mask = reserve_df["DateTime"] >= begin_time
end_mask = reserve_df["DateTime"] < end_time
reserve_df = reserve_df[start_mask & end_mask]
reserve_timeseries = []
for this_row in reserve_df.to_dict(orient='split')["data"]:
reserve_timeseries.append(
DateTimeValue(this_row[0], float(this_row[1])))
reserves_dict[name] = reserve_timeseries
for reserve in other_reserve_categories:
reserves_dict[reserve] = _read_rts_gmlc_reserve_table(
timeseries_pointer_dict[(reserve, simulation)].DataFile,
begin_time,
end_time,
simulation,
)
times = []
for load in load_timeseries:
times.append(str(load.DateTime))
system["time_indices"] = times
## load into grid_network object
## First, load Pl, Ql
for name, load in md_obj.elements("load"):
pl_dict, ql_dict = dict(), dict()
bus = elements["bus"][load["bus"]]
for load_time in load_timeseries:
area_load = getattr(load_time, bus["area"])
pl_dict[str(load_time.DateTime)] = round(
bus_load_participation_factor_dict[name] * area_load, 2)
ql_dict[str(load_time.DateTime)] = pl_dict[str(
load_time.DateTime)] * bus_Ql_over_Pl_dict[name]
load["p_load"] = _make_time_series_dict(pl_dict)
load["q_load"] = _make_time_series_dict(ql_dict)
## load in area reserve factors
area_spin_map = {
'Area1': 'Spin_Up_R1',
'Area2': 'Spin_Up_R2',
'Area3': 'Spin_Up_R3'
}
for name, area in md_obj.elements("area"):
spin_reserve_dict = dict()
for datetimevalue in reserves_dict[area_spin_map[name]]:
spin_reserve_dict[str(datetimevalue.DateTime)] = round(
datetimevalue.Value, 2)
area["spinning_reserve_requirement"] = _make_time_series_dict(
spin_reserve_dict)
## load in global reserve factors
rts_to_egret_reserve_map = {
"Flex_Down": "flexible_ramp_down_requirement",
"Flex_Up": "flexible_ramp_up_requirement",
"Reg_Down": "regulation_down_requirement",
"Reg_Up": "regulation_up_requirement",
}
for reserve in other_reserve_categories:
system[rts_to_egret_reserve_map[reserve]] = _make_time_series_dict(
reserves_dict[reserve])
## now load renewable generator stuff
for name, gen in md_obj.elements("generator", generator_type="renewable"):
if gen["fuel"] not in ["Solar", "Wind", "Hydro"]:
continue
renewables_timeseries = filtered_timeseries[name]
## for safety, curtailable renewables can go down to 0
gen["p_min"] = 0.
output_dict = dict()
for datetimevalue in renewables_timeseries:
output_dict[str(datetimevalue.DateTime)] = round(
datetimevalue.Value, 2)
gen["p_max"] = _make_time_series_dict(output_dict)
# set must-take for Hydro and RTPV
if gen["unit_type"] in ["HYDRO", "RTPV"]:
## copy is for safety when overwriting
gen["p_min"] = _make_time_series_dict(output_dict.copy())
## get this from the same place the prescient reader does
if t0_state is None:
unit_on_time_df = pd.read_table(os.path.join(
base_dir,
"../FormattedData/PLEXOS/PLEXOS_Solution/DAY_AHEAD Solution Files/noTX/on_time_7.12.csv"
),
header=0,
sep=",")
unit_on_time_df_as_dict = unit_on_time_df.to_dict(orient="split")
unit_on_t0_state_dict = {}
for i in range(0, len(unit_on_time_df_as_dict["columns"])):
gen_id = unit_on_time_df_as_dict["columns"][i]
unit_on_t0_state_dict[gen_id] = int(
unit_on_time_df_as_dict["data"][0][i])
for name, gen in md_obj.elements("generator",
generator_type="thermal"):
gen["initial_status"] = unit_on_t0_state_dict[name]
if gen["initial_status"] < 0:
gen["initial_p_output"] = 0.
gen["initial_q_output"] = 0.
else:
gen["initial_p_output"] = gen["p_min"]
gen["initial_q_output"] = max(0., gen["q_min"])
else:
for name, gen in md_obj.elements("generator",
generator_type="thermal"):
gen["initial_status"] = t0_state[name]["initial_status"]
gen["initial_p_output"] = t0_state[name]["initial_p_output"]
gen["initial_q_output"] = t0_state[name]["initial_q_output"]
return md_obj.data
def convert_load_by_area_to_source(data_dir,
begin_time,
end_time,
t0_state=None):
"""
Create a ModelData object from the input data. Assumes data is formatted like the RTS-GMLC repository's 'RTS_Data' directory.
Parameters
----------
data_dir : str
Path to data directory
begin_time : datetime.datetime or str
Beginning of time horizon. If str, date/time in "YYYY-MM-DD HH:MM:SS" or "YYYY-MM-DD" format,
the later of which assumes a midnight start.
end_time : datetime.datetime or str
End of time horizon. If str, date/time in "YYYY-MM-DD HH:MM:SS" or "YYYY-MM-DD" format,
the later of which assumes a midnight start.
t0_state : dict or Nonetype
Keys of this dict are thermal generator names, each element of which is another dictionary with
keys "initial_status", "initial_p_output", and "initial_q_output", which specify whether the
generator is on at t0, the real power output at t0, and the reactive power output at t0.
If this is None, default values are loaded.
"""
for simulation in ['DAY_AHEAD', 'REAL_TIME']:
simulation = simulation.upper()
base_dir = os.path.join(data_dir, 'SourceData')
begin_time, end_time = _get_datetimes(begin_time, end_time)
TimeSeriesPointer = namedtuple(
'TimeSeriesPointer',
['Object', 'Simulation', 'Parameter', 'DataFile'])
DateTimeValue = namedtuple('DateTimeValue', ['DateTime', 'Value'])
Load = namedtuple('Load', ['DateTime', 'Area1', 'Area2', 'Area3'])
timeseries_pointer_df = pd.read_csv(os.path.join(
base_dir, "timeseries_pointers_orig.csv"),
header=0,
sep=',')
time_delta = end_time - begin_time
hours = 24 * time_delta.days + math.ceil(time_delta.seconds / 3600.)
model_data = _create_rtsgmlc_skeleton(rts_gmlc_dir)
## create an object for easy iterating
md_obj = md.ModelData(model_data)
system = md_obj.data["system"]
elements = md_obj.data["elements"]
if simulation == "DAY_AHEAD":
system["time_period_length_minutes"] = 60
else:
system["time_period_length_minutes"] = 5
# compute aggregate load per area, and then compute
# load participation factors from each bus from that data.
region_total_load = {}
areas = ["Area" + str(i) for i in range(1, 4)]
for this_region in areas:
this_region_total_load = 0.0
## loads have exactly one bus
for name, load in md_obj.elements("load"):
bus = elements["bus"][load["bus"]]
if bus["area"] == this_region:
this_region_total_load += load["p_load"]
region_total_load[this_region] = this_region_total_load
bus_load_participation_factor_dict = {}
bus_Ql_over_Pl_dict = {}
for name, load in md_obj.elements("load"):
bus = elements["bus"][load["bus"]]
bus_load_participation_factor_dict[
name] = load["p_load"] / region_total_load[bus["area"]]
bus_Ql_over_Pl_dict[name] = load["q_load"] / load["p_load"]
timeseries_pointer_dict = {}
for timeseries_pointer_index in timeseries_pointer_df.index.tolist():
this_timeseries_pointer_dict = timeseries_pointer_df.loc[
timeseries_pointer_index].to_dict()
new_timeseries_pointer = TimeSeriesPointer(
this_timeseries_pointer_dict["Object"],
this_timeseries_pointer_dict["Simulation"],
this_timeseries_pointer_dict["Parameter"],
os.path.join(base_dir,
this_timeseries_pointer_dict["Data File"]))
timeseries_pointer_dict[(
new_timeseries_pointer.Object,
new_timeseries_pointer.Simulation)] = new_timeseries_pointer
load_timeseries_spec = timeseries_pointer_dict[("Load", simulation)]
load_timeseries_df = _read_rts_gmlc_table(
load_timeseries_spec.DataFile, simulation)
load_timeseries_df = load_timeseries_df.rename(
columns={"Year_Month_Day_Period": "DateTime"})
start_mask = load_timeseries_df["DateTime"] >= begin_time
end_mask = load_timeseries_df["DateTime"] < end_time
masked_load_timeseries_df = load_timeseries_df[start_mask & end_mask]
load_dict = masked_load_timeseries_df.to_dict(orient='split')
load_timeseries = []
for load_row in load_dict["data"]:
load_timeseries.append(
Load(load_row[0], float(load_row[1]), float(load_row[2]),
float(load_row[3])))
times = []
for load in load_timeseries:
times.append(str(load.DateTime))
system["time_indices"] = times
## load into grid_network object
## First, load Pl, Ql
for name, load in md_obj.elements("load"):
pl_dict, ql_dict = dict(), dict()
bus = elements["bus"][load["bus"]]
for load_time in load_timeseries:
area_load = getattr(load_time, bus["area"])
pl_dict[str(load_time.DateTime)] = round(
bus_load_participation_factor_dict[name] * area_load, 2)
ql_dict[str(load_time.DateTime)] = pl_dict[str(
load_time.DateTime)] * bus_Ql_over_Pl_dict[name]
load["p_load"] = _make_time_series_dict(pl_dict)
load["q_load"] = _make_time_series_dict(ql_dict)
new_load_time_series = []
day_ahead_load_file = '../timeseries_data_files/Load/new_load_time_series_DA.csv'
real_time_load_file = '../timeseries_data_files/Load/new_load_time_series_RT.csv'
for ix, load_time in enumerate(load_timeseries, start=0):
load_time_series_record = {}
load_time_series_record['Year'] = load_time.DateTime.year
load_time_series_record['Month'] = load_time.DateTime.month
load_time_series_record['Day'] = load_time.DateTime.day
if simulation == 'DAY_AHEAD':
load_time_series_record['Period'] = (ix % 24) + 1
else:
load_time_series_record['Period'] = (ix % (24 * 12)) + 1
for name, load in md_obj.elements('load'):
bus = elements['bus'][load['bus']]
area_load = getattr(load_time, bus['area'])
load_time_series_record[name] = round(
bus_load_participation_factor_dict[name] * area_load, 2)
new_load_time_series.append(load_time_series_record)
new_load_time_series_df = pd.DataFrame(new_load_time_series)
new_load_time_series_df = new_load_time_series_df[
['Year', 'Month', 'Day', 'Period'] +
new_load_time_series_df.columns[4:].tolist()]
new_load_time_series_fname = 'new_load_time_series_{0}.csv'.format(
'DA' if simulation == "DAY_AHEAD" else 'RT')
new_load_time_series_df.to_csv(os.path.join(
data_dir, 'timeseries_data_files', 'Load',
new_load_time_series_fname),
index=False)
# Augment time series pointer dataframe.
for name, load in md_obj.elements('load'):
new_load_timeseries_spec = {}
new_load_timeseries_spec['Object'] = name
new_load_timeseries_spec['Parameter'] = 'Requirement'
new_load_timeseries_spec['Simulation'] = 'DAY_AHEAD'
new_load_timeseries_spec['Data File'] = day_ahead_load_file
timeseries_pointer_df = timeseries_pointer_df.append(
new_load_timeseries_spec, ignore_index=True)
new_load_timeseries_spec = {}
new_load_timeseries_spec['Object'] = name
new_load_timeseries_spec['Parameter'] = 'Requirement'
new_load_timeseries_spec['Simulation'] = 'REAL_TIME'
new_load_timeseries_spec['Data File'] = real_time_load_file
timeseries_pointer_df = timeseries_pointer_df.append(
new_load_timeseries_spec, ignore_index=True)
timeseries_pointer_df.loc[
timeseries_pointer_df['Object'] != 'Load'].to_csv(os.path.join(
data_dir, 'SourceData', 'timeseries_pointers.csv'),
index=False)
