def create_ModelData(rts_gmlc_dir:str, begin_time:Union[datetime,str], end_time:Union[datetime,str], simulation:str="DAY_AHEAD", t0_state:Optional[dict] = None) -> md.ModelData: """ Create a ModelData object from the RTS-GMLC data. Parameters ---------- rts_gmlc_dir : str Path to directory holding csv files in RTS-GMLC format (bus.csv, gen.csv, etc). begin_time : datetime.datetime or str Beginning of time horizon. If str, date/time in "YYYY-MM-DD HH:MM:SS" or "YYYY-MM-DD" format, the later of which assumes a midnight start. end_time : datetime.datetime or str End of time horizon. If str, date/time in "YYYY-MM-DD HH:MM:SS" or "YYYY-MM-DD" format, the later of which assumes a midnight start. simulation : str Either "DAY_AHEAD" or "REAL_TIME", which specifies which time series the data is taken from, default is "DAY_AHEAD". t0_state : dict or None Keys of this dict are thermal generator names, each element of which is another dictionary with keys "initial_status", "initial_p_output", and "initial_q_output", which specify whether the generator is on at t0, the real power output at t0, and the reactive power output at t0. If t0_state is None, values are read from initial_status.csv in the rts_gmlc_dir. If that file does not exist, no initial state data is set in the model. Returns ------- egret.model_data.ModelData Returns a ModelData object with the timeseries data specified """ return md.ModelData(create_model_data_dict(rts_gmlc_dir, begin_time, end_time, simulation, t0_state))
def pysp_instance_creation_callback(scenario_name, node_names, cb_data): #print("Building instance for scenario =", scenario_name) scennum = sputils.extract_num(scenario_name) uc_model_params = pdp.get_uc_model() # Now using cb_data ##path = os.environ[UC_NUMSCENS_ENV_VAR] + "scenarios_r1" path = cb_data["path"] scenario_data = DataPortal(model=uc_model_params) scenario_data.load(filename=path + os.sep + "RootNode.dat") scenario_data.load(filename=path + os.sep + "Node" + str(scennum) + ".dat") scenario_params = uc_model_params.create_instance(scenario_data, report_timing=False, name=scenario_name) scenario_md = md.ModelData( pdp.create_model_data_dict_params(scenario_params, keep_names=True)) ## TODO: use the "power_balance_constraints" for now. In the future, networks should be ## handled with a custom callback -- also consider other base models scenario_instance = uc.create_tight_unit_commitment_model( scenario_md, network_constraints='power_balance_constraints') # hold over string attribute from Egret, # causes warning wth LShaped/Benders del scenario_instance.objective return scenario_instance
def pysp_instance_creation_callback(scenario_name, path=None, scenario_count=None): """ Notes: - The uc_cylinders.py code has a `scenario_count` kwarg that gets passed to the spokes, but it seems to be unused here... """ #print("Building instance for scenario =", scenario_name) scennum = sputils.extract_num(scenario_name) uc_model_params = pdp.get_uc_model() scenario_data = DataPortal(model=uc_model_params) scenario_data.load(filename=path + os.sep + "RootNode.dat") scenario_data.load(filename=path + os.sep + "Node" + str(scennum) + ".dat") scenario_params = uc_model_params.create_instance(scenario_data, report_timing=False, name=scenario_name) scenario_md = md.ModelData( pdp.create_model_data_dict_params(scenario_params, keep_names=True)) ## TODO: use the "power_balance_constraints" for now. In the future, networks should be ## handled with a custom callback -- also consider other base models scenario_instance = uc.create_tight_unit_commitment_model( scenario_md, network_constraints='power_balance_constraints') # hold over string attribute from Egret, # causes warning wth LShaped/Benders del scenario_instance.objective return scenario_instance
def create_ModelData(rts_gmlc_dir, begin_time, end_time, simulation="DAY_AHEAD", t0_state=None): """ Create a ModelData object from the RTS-GMLC data. Parameters ---------- rts_gmlc_dir : str Path to RTS-GMLC directory begin_time : datetime.datetime or str Beginning of time horizon. If str, date/time in "YYYY-MM-DD HH:MM:SS" or "YYYY-MM-DD" format, the later of which assumes a midnight start. end_time : datetime.datetime or str End of time horizon. If str, date/time in "YYYY-MM-DD HH:MM:SS" or "YYYY-MM-DD" format, the later of which assumes a midnight start. simulation : str Either "DAY_AHEAD" or "REAL_TIME", which specifies which time series the data is taken from, default is "DAY_AHEAD". t0_state : dict or Nonetype Keys of this dict are thermal generator names, each element of which is another dictionary with keys "initial_status", "initial_p_output", and "initial_q_output", which specify whether the generator is on at t0, the real power output at t0, and the reactive power output at t0. If this is None, default values are loaded. Returns ------- egret.model_data.ModelData Returns a ModelData object with the timeseries data specified """ return md.ModelData( create_model_data_dict(rts_gmlc_dir, begin_time, end_time, simulation, t0_state))
def create_ModelData(pglib_uc_filename): """ Parse a pglib-uc input file into a ModelData object containing the model_data dictionary Parameters ---------- pglib_uc_filename : str Path and filename of the pglib-uc input file you wish to load Returns ------- ModelData """ data = create_model_data_dict(pglib_uc_filename) return md.ModelData(data)
def create_ModelData(dat_file): ''' Create a ModelData object from a prescient dat file Parameters ---------- dat_file : str Path to prescient *.dat file Returns ------- egret.model_data.ModelData Returns a ModelData object with the dat file data specified ''' return md.ModelData(create_model_data_dict(dat_file))
def create_ModelData(matpower_filename): """ Parse a MATPOWER input file into a ModelData object containing the model_data dictionary Parameters ---------- matpower_filename : str Path and filename of the matpower inp file you wish to load Returns ------- ModelData """ data = create_model_data_dict(matpower_filename) return md.ModelData(data)
def create_model_data_dict(rts_gmlc_dir, begin_time, end_time, simulation="DAY_AHEAD", t0_state=None): """ Create a model_data dictionary from the RTS-GMLC data. Parameters ---------- rts_gmlc_dir : str Path to RTS-GMLC directory begin_time : datetime.datetime or str Beginning of time horizon. If str, date/time in "YYYY-MM-DD HH:MM:SS" or "YYYY-MM-DD" format, the later of which assumes a midnight start. end_time : datetime.datetime or str End of time horizon. If str, date/time in "YYYY-MM-DD HH:MM:SS" or "YYYY-MM-DD" format, the later of which assumes a midnight start. simulation : str Either "DAY_AHEAD" or "REAL_TIME", which specifies which time series the data is taken from, default is "DAY_AHEAD". t0_state : dict or Nonetype Keys of this dict are thermal generator names, each element of which is another dictionary with keys "initial_status", "initial_p_output", and "initial_q_output", which specify whether the generator is on at t0, the real power output at t0, and the reactive power output at t0. If this is None, default values are loaded. Returns ------- dict : A dictionary in the format required for the ModelData object. """ simulation = simulation.upper() if simulation not in ["DAY_AHEAD", "REAL_TIME"]: raise ValueError('simulation must be "DAY_AHEAD" or "REAL_TIME"') base_dir = os.path.join(rts_gmlc_dir, 'RTS_Data', 'SourceData') begin_time, end_time = _get_datetimes(begin_time, end_time) TimeSeriesPointer = namedtuple( 'TimeSeriesPointer', ['Object', 'Simulation', 'Parameter', 'DataFile'], verbose=False) DateTimeValue = namedtuple('DateTimeValue', ['DateTime', 'Value'], verbose=False) Load = namedtuple('Load', ['DateTime', 'Area1', 'Area2', 'Area3'], verbose=False) timeseries_pointer_df = pd.read_table(os.path.join( base_dir, "timeseries_pointers.csv"), header=0, sep=',') time_delta = end_time - begin_time hours = 24 * time_delta.days + math.ceil(time_delta.seconds / 3600.) model_data = _create_rtsgmlc_skeleton(rts_gmlc_dir) ## create an object for easy iterating md_obj = md.ModelData(model_data) system = md_obj.data["system"] elements = md_obj.data["elements"] if simulation == "DAY_AHEAD": system["time_period_length_minutes"] = 60 else: system["time_period_length_minutes"] = 5 # compute aggregate load per area, and then compute # load participation factors from each bus from that data. region_total_load = {} areas = ["Area" + str(i) for i in range(1, 4)] for this_region in areas: this_region_total_load = 0.0 ## loads have exactly one bus for name, load in md_obj.elements("load"): bus = elements["bus"][load["bus"]] if bus["area"] == this_region: this_region_total_load += load["p_load"] region_total_load[this_region] = this_region_total_load bus_load_participation_factor_dict = {} bus_Ql_over_Pl_dict = {} for name, load in md_obj.elements("load"): bus = elements["bus"][load["bus"]] bus_load_participation_factor_dict[ name] = load["p_load"] / region_total_load[bus["area"]] bus_Ql_over_Pl_dict[name] = load["q_load"] / load["p_load"] timeseries_pointer_dict = {} for timeseries_pointer_index in timeseries_pointer_df.index.tolist(): this_timeseries_pointer_dict = timeseries_pointer_df.loc[ timeseries_pointer_index].to_dict() new_timeseries_pointer = TimeSeriesPointer( this_timeseries_pointer_dict["Object"], this_timeseries_pointer_dict["Simulation"], this_timeseries_pointer_dict["Parameter"], os.path.join(base_dir, this_timeseries_pointer_dict["Data File"])) timeseries_pointer_dict[( new_timeseries_pointer.Object, new_timeseries_pointer.Simulation)] = new_timeseries_pointer filtered_timeseries = {} for name, gen in md_obj.elements("generator", generator_type="renewable"): if gen["fuel"] in ["Solar", "Wind", "Hydro"]: if (name, simulation) not in timeseries_pointer_dict: print( "***WARNING - No timeseries pointer entry found for generator=%s" % name) else: #print("Time series for generator=%s will be loaded from file=%s" % (name, timeseries_pointer_dict[(name,"DAY_AHEAD")].DataFile)) renewables_timeseries_df = _read_rts_gmlc_table( timeseries_pointer_dict[(name, simulation)].DataFile, simulation) this_source_timeseries_df = renewables_timeseries_df.loc[:, [ "Year_Month_Day_Period", name ]] this_source_timeseries_df = this_source_timeseries_df.rename( columns={"Year_Month_Day_Period": "DateTime"}) start_mask = this_source_timeseries_df["DateTime"] >= begin_time end_mask = this_source_timeseries_df["DateTime"] < end_time this_source_masked_timeseries_df = this_source_timeseries_df[ start_mask & end_mask] renewables_timeseries_dict = this_source_masked_timeseries_df.to_dict( orient='split') renewables_timeseries = [] for this_row in renewables_timeseries_dict["data"]: renewables_timeseries.append( DateTimeValue(this_row[0], float(this_row[1]))) filtered_timeseries[name] = renewables_timeseries load_timeseries_spec = timeseries_pointer_dict[("Load", simulation)] load_timeseries_df = _read_rts_gmlc_table(load_timeseries_spec.DataFile, simulation) load_timeseries_df = load_timeseries_df.rename( columns={"Year_Month_Day_Period": "DateTime"}) start_mask = load_timeseries_df["DateTime"] >= begin_time end_mask = load_timeseries_df["DateTime"] < end_time masked_load_timeseries_df = load_timeseries_df[start_mask & end_mask] load_dict = masked_load_timeseries_df.to_dict(orient='split') load_timeseries = [] for load_row in load_dict["data"]: load_timeseries.append( Load(load_row[0], float(load_row[1]), float(load_row[2]), float(load_row[3]))) reserves_dfs = {} spin_reserve_categories = ["Spin_Up_R1", "Spin_Up_R2", "Spin_Up_R3"] other_reserve_categories = [ "Reg_Down", "Reg_Up", ] ## flexiramp products only in day-ahead simulation if simulation == "DAY_AHEAD": other_reserve_categories += [ "Flex_Down", "Flex_Up", ] for reserve in spin_reserve_categories: reserves_dfs[reserve] = _read_rts_gmlc_table( timeseries_pointer_dict[(reserve, simulation)].DataFile, simulation) reserves_dict = {} for name, reserve_df in reserves_dfs.items(): reserve_df = reserve_df.rename( columns={"Year_Month_Day_Period": "DateTime"}) start_mask = reserve_df["DateTime"] >= begin_time end_mask = reserve_df["DateTime"] < end_time reserve_df = reserve_df[start_mask & end_mask] reserve_timeseries = [] for this_row in reserve_df.to_dict(orient='split')["data"]: reserve_timeseries.append( DateTimeValue(this_row[0], float(this_row[1]))) reserves_dict[name] = reserve_timeseries for reserve in other_reserve_categories: reserves_dict[reserve] = _read_rts_gmlc_reserve_table( timeseries_pointer_dict[(reserve, simulation)].DataFile, begin_time, end_time, simulation, ) times = [] for load in load_timeseries: times.append(str(load.DateTime)) system["time_indices"] = times ## load into grid_network object ## First, load Pl, Ql for name, load in md_obj.elements("load"): pl_dict, ql_dict = dict(), dict() bus = elements["bus"][load["bus"]] for load_time in load_timeseries: area_load = getattr(load_time, bus["area"]) pl_dict[str(load_time.DateTime)] = round( bus_load_participation_factor_dict[name] * area_load, 2) ql_dict[str(load_time.DateTime)] = pl_dict[str( load_time.DateTime)] * bus_Ql_over_Pl_dict[name] load["p_load"] = _make_time_series_dict(pl_dict) load["q_load"] = _make_time_series_dict(ql_dict) ## load in area reserve factors area_spin_map = { 'Area1': 'Spin_Up_R1', 'Area2': 'Spin_Up_R2', 'Area3': 'Spin_Up_R3' } for name, area in md_obj.elements("area"): spin_reserve_dict = dict() for datetimevalue in reserves_dict[area_spin_map[name]]: spin_reserve_dict[str(datetimevalue.DateTime)] = round( datetimevalue.Value, 2) area["spinning_reserve_requirement"] = _make_time_series_dict( spin_reserve_dict) ## load in global reserve factors rts_to_egret_reserve_map = { "Flex_Down": "flexible_ramp_down_requirement", "Flex_Up": "flexible_ramp_up_requirement", "Reg_Down": "regulation_down_requirement", "Reg_Up": "regulation_up_requirement", } for reserve in other_reserve_categories: system[rts_to_egret_reserve_map[reserve]] = _make_time_series_dict( reserves_dict[reserve]) ## now load renewable generator stuff for name, gen in md_obj.elements("generator", generator_type="renewable"): if gen["fuel"] not in ["Solar", "Wind", "Hydro"]: continue renewables_timeseries = filtered_timeseries[name] ## for safety, curtailable renewables can go down to 0 gen["p_min"] = 0. output_dict = dict() for datetimevalue in renewables_timeseries: output_dict[str(datetimevalue.DateTime)] = round( datetimevalue.Value, 2) gen["p_max"] = _make_time_series_dict(output_dict) # set must-take for Hydro and RTPV if gen["unit_type"] in ["HYDRO", "RTPV"]: ## copy is for safety when overwriting gen["p_min"] = _make_time_series_dict(output_dict.copy()) ## get this from the same place the prescient reader does if t0_state is None: unit_on_time_df = pd.read_table(os.path.join( base_dir, "../FormattedData/PLEXOS/PLEXOS_Solution/DAY_AHEAD Solution Files/noTX/on_time_7.12.csv" ), header=0, sep=",") unit_on_time_df_as_dict = unit_on_time_df.to_dict(orient="split") unit_on_t0_state_dict = {} for i in range(0, len(unit_on_time_df_as_dict["columns"])): gen_id = unit_on_time_df_as_dict["columns"][i] unit_on_t0_state_dict[gen_id] = int( unit_on_time_df_as_dict["data"][0][i]) for name, gen in md_obj.elements("generator", generator_type="thermal"): gen["initial_status"] = unit_on_t0_state_dict[name] if gen["initial_status"] < 0: gen["initial_p_output"] = 0. gen["initial_q_output"] = 0. else: gen["initial_p_output"] = gen["p_min"] gen["initial_q_output"] = max(0., gen["q_min"]) else: for name, gen in md_obj.elements("generator", generator_type="thermal"): gen["initial_status"] = t0_state[name]["initial_status"] gen["initial_p_output"] = t0_state[name]["initial_p_output"] gen["initial_q_output"] = t0_state[name]["initial_q_output"] return md_obj.data
def convert_load_by_area_to_source(data_dir, begin_time, end_time, t0_state=None): """ Create a ModelData object from the input data. Assumes data is formatted like the RTS-GMLC repository's 'RTS_Data' directory. Parameters ---------- data_dir : str Path to data directory begin_time : datetime.datetime or str Beginning of time horizon. If str, date/time in "YYYY-MM-DD HH:MM:SS" or "YYYY-MM-DD" format, the later of which assumes a midnight start. end_time : datetime.datetime or str End of time horizon. If str, date/time in "YYYY-MM-DD HH:MM:SS" or "YYYY-MM-DD" format, the later of which assumes a midnight start. t0_state : dict or Nonetype Keys of this dict are thermal generator names, each element of which is another dictionary with keys "initial_status", "initial_p_output", and "initial_q_output", which specify whether the generator is on at t0, the real power output at t0, and the reactive power output at t0. If this is None, default values are loaded. """ for simulation in ['DAY_AHEAD', 'REAL_TIME']: simulation = simulation.upper() base_dir = os.path.join(data_dir, 'SourceData') begin_time, end_time = _get_datetimes(begin_time, end_time) TimeSeriesPointer = namedtuple( 'TimeSeriesPointer', ['Object', 'Simulation', 'Parameter', 'DataFile']) DateTimeValue = namedtuple('DateTimeValue', ['DateTime', 'Value']) Load = namedtuple('Load', ['DateTime', 'Area1', 'Area2', 'Area3']) timeseries_pointer_df = pd.read_csv(os.path.join( base_dir, "timeseries_pointers_orig.csv"), header=0, sep=',') time_delta = end_time - begin_time hours = 24 * time_delta.days + math.ceil(time_delta.seconds / 3600.) model_data = _create_rtsgmlc_skeleton(rts_gmlc_dir) ## create an object for easy iterating md_obj = md.ModelData(model_data) system = md_obj.data["system"] elements = md_obj.data["elements"] if simulation == "DAY_AHEAD": system["time_period_length_minutes"] = 60 else: system["time_period_length_minutes"] = 5 # compute aggregate load per area, and then compute # load participation factors from each bus from that data. region_total_load = {} areas = ["Area" + str(i) for i in range(1, 4)] for this_region in areas: this_region_total_load = 0.0 ## loads have exactly one bus for name, load in md_obj.elements("load"): bus = elements["bus"][load["bus"]] if bus["area"] == this_region: this_region_total_load += load["p_load"] region_total_load[this_region] = this_region_total_load bus_load_participation_factor_dict = {} bus_Ql_over_Pl_dict = {} for name, load in md_obj.elements("load"): bus = elements["bus"][load["bus"]] bus_load_participation_factor_dict[ name] = load["p_load"] / region_total_load[bus["area"]] bus_Ql_over_Pl_dict[name] = load["q_load"] / load["p_load"] timeseries_pointer_dict = {} for timeseries_pointer_index in timeseries_pointer_df.index.tolist(): this_timeseries_pointer_dict = timeseries_pointer_df.loc[ timeseries_pointer_index].to_dict() new_timeseries_pointer = TimeSeriesPointer( this_timeseries_pointer_dict["Object"], this_timeseries_pointer_dict["Simulation"], this_timeseries_pointer_dict["Parameter"], os.path.join(base_dir, this_timeseries_pointer_dict["Data File"])) timeseries_pointer_dict[( new_timeseries_pointer.Object, new_timeseries_pointer.Simulation)] = new_timeseries_pointer load_timeseries_spec = timeseries_pointer_dict[("Load", simulation)] load_timeseries_df = _read_rts_gmlc_table( load_timeseries_spec.DataFile, simulation) load_timeseries_df = load_timeseries_df.rename( columns={"Year_Month_Day_Period": "DateTime"}) start_mask = load_timeseries_df["DateTime"] >= begin_time end_mask = load_timeseries_df["DateTime"] < end_time masked_load_timeseries_df = load_timeseries_df[start_mask & end_mask] load_dict = masked_load_timeseries_df.to_dict(orient='split') load_timeseries = [] for load_row in load_dict["data"]: load_timeseries.append( Load(load_row[0], float(load_row[1]), float(load_row[2]), float(load_row[3]))) times = [] for load in load_timeseries: times.append(str(load.DateTime)) system["time_indices"] = times ## load into grid_network object ## First, load Pl, Ql for name, load in md_obj.elements("load"): pl_dict, ql_dict = dict(), dict() bus = elements["bus"][load["bus"]] for load_time in load_timeseries: area_load = getattr(load_time, bus["area"]) pl_dict[str(load_time.DateTime)] = round( bus_load_participation_factor_dict[name] * area_load, 2) ql_dict[str(load_time.DateTime)] = pl_dict[str( load_time.DateTime)] * bus_Ql_over_Pl_dict[name] load["p_load"] = _make_time_series_dict(pl_dict) load["q_load"] = _make_time_series_dict(ql_dict) new_load_time_series = [] day_ahead_load_file = '../timeseries_data_files/Load/new_load_time_series_DA.csv' real_time_load_file = '../timeseries_data_files/Load/new_load_time_series_RT.csv' for ix, load_time in enumerate(load_timeseries, start=0): load_time_series_record = {} load_time_series_record['Year'] = load_time.DateTime.year load_time_series_record['Month'] = load_time.DateTime.month load_time_series_record['Day'] = load_time.DateTime.day if simulation == 'DAY_AHEAD': load_time_series_record['Period'] = (ix % 24) + 1 else: load_time_series_record['Period'] = (ix % (24 * 12)) + 1 for name, load in md_obj.elements('load'): bus = elements['bus'][load['bus']] area_load = getattr(load_time, bus['area']) load_time_series_record[name] = round( bus_load_participation_factor_dict[name] * area_load, 2) new_load_time_series.append(load_time_series_record) new_load_time_series_df = pd.DataFrame(new_load_time_series) new_load_time_series_df = new_load_time_series_df[ ['Year', 'Month', 'Day', 'Period'] + new_load_time_series_df.columns[4:].tolist()] new_load_time_series_fname = 'new_load_time_series_{0}.csv'.format( 'DA' if simulation == "DAY_AHEAD" else 'RT') new_load_time_series_df.to_csv(os.path.join( data_dir, 'timeseries_data_files', 'Load', new_load_time_series_fname), index=False) # Augment time series pointer dataframe. for name, load in md_obj.elements('load'): new_load_timeseries_spec = {} new_load_timeseries_spec['Object'] = name new_load_timeseries_spec['Parameter'] = 'Requirement' new_load_timeseries_spec['Simulation'] = 'DAY_AHEAD' new_load_timeseries_spec['Data File'] = day_ahead_load_file timeseries_pointer_df = timeseries_pointer_df.append( new_load_timeseries_spec, ignore_index=True) new_load_timeseries_spec = {} new_load_timeseries_spec['Object'] = name new_load_timeseries_spec['Parameter'] = 'Requirement' new_load_timeseries_spec['Simulation'] = 'REAL_TIME' new_load_timeseries_spec['Data File'] = real_time_load_file timeseries_pointer_df = timeseries_pointer_df.append( new_load_timeseries_spec, ignore_index=True) timeseries_pointer_df.loc[ timeseries_pointer_df['Object'] != 'Load'].to_csv(os.path.join( data_dir, 'SourceData', 'timeseries_pointers.csv'), index=False)