def copy_input(scenario_id):
"""Copies input file, converting matfile from v7.3 to v7 on the way.
:param str scenario_id: scenario id
"""
src = os.path.join(const.EXECUTE_DIR, 'scenario_%s' % scenario_id,
'output', 'input.mat')
dst = os.path.join(const.INPUT_DIR, '%s_grid.mat' % scenario_id)
print('loading and parsing input.mat')
input_mpc = load_mat73(src)
print('saving converted input.mat as %s_grid.mat' % scenario_id)
savemat(dst, input_mpc, do_compression=True)
def copy_input(execute_dir, mat_dir=None, scenario_id=None):
"""Copies Julia-saved input matfile (input.mat), converting matfile from v7.3 to v7 on the way.
:param str execute_dir: the directory containing the original input file
:param str mat_dir: the optional directory to which to save the converted input file, Defaults to execute_dir
:param str filename: optional name for the copied input.mat file. Defaults to "grid.mat"
"""
if not mat_dir:
mat_dir = execute_dir
src = os.path.join(execute_dir, "input.mat")
filename = scenario_id + "_grid.mat" if scenario_id else "grid.mat"
dst = os.path.join(mat_dir, filename)
print("loading and parsing input.mat")
input_mpc = load_mat73(src)
print(f"saving converted input.mat as {filename}")
savemat(dst, input_mpc, do_compression=True)
return dst
def extract_data(results):
"""Builds data frames of {PG, PF, LMP, CONGU, CONGL} from Julia simulation
output binary files produced by REISE.jl.
:param list results: list of result files
:return: (*tuple*) -- first element is a dictionary of Pandas data frames of:
PG, PF, LMP, CONGU, CONGL, LOAD_SHED, second is a list of strings of infeasibilities,
and the third element is a list of numpy.float64 costs for each file in the input results list
"""
infeasibilities = []
cost = []
extraction_vars = {"pf", "pg", "lmp", "congu", "congl"}
sparse_extraction_vars = {"congu", "congl", "load_shed"}
temps = {}
outputs = {}
tic = time.process_time()
for i, filename in tqdm(enumerate(results)):
# For each result_#.mat file
output = load_mat73(filename)
# Record cost for this mat file
try:
cost.append(output["mdo_save"]["results"]["f"][0][0])
except KeyError:
pass
# Check for infeasibilities
demand_scaling = output["mdo_save"]["demand_scaling"][0][0]
if demand_scaling < 1:
demand_change = round(100 * (1 - demand_scaling))
infeasibilities.append(f"{i}:{demand_change}")
# Extract various variables
output_mpc = output["mdo_save"]["flow"]["mpc"]
temps["pg"] = output_mpc["gen"]["PG"].T
temps["pf"] = output_mpc["branch"]["PF"].T
temps["lmp"] = output_mpc["bus"]["LAM_P"].T
temps["congu"] = output_mpc["branch"]["MU_SF"].T
temps["congl"] = output_mpc["branch"]["MU_ST"].T
# Extract optional variables (not present in all scenarios)
try:
temps["pf_dcline"] = output_mpc["dcline"]["PF_dcline"].T
extraction_vars |= {"pf_dcline"}
except KeyError:
pass
try:
temps["storage_pg"] = output_mpc["storage"]["PG"].T
temps["storage_e"] = output_mpc["storage"]["Energy"].T
extraction_vars |= {"storage_pg", "storage_e"}
except KeyError:
pass
try:
temps["load_shed"] = output_mpc["load_shed"]["load_shed"].T
extraction_vars |= {"load_shed"}
except KeyError:
pass
# Extract which number result currently being processed
i = result_num(filename)
for v in extraction_vars:
# Determine start, end indices of the outputs where this iteration belongs
interval_length, n_columns = temps[v].shape
start_hour, end_hour = (i * interval_length), ((i + 1) * interval_length)
# If this extraction variables hasn't been seen yet, initialize all zeros
if v not in outputs:
total_length = len(results) * interval_length
outputs[v] = pd.DataFrame(np.zeros((total_length, n_columns)))
# Update the output variables for the time frame with the extracted data
outputs[v].iloc[start_hour:end_hour, :] = temps[v]
# Record time to read all the data
toc = time.process_time()
print("Reading time " + str((toc - tic)) + "s")
# Convert everything except sparse variables to float32
for v in extraction_vars - sparse_extraction_vars:
outputs[v] = outputs[v].astype(np.float32)
# Convert outputs with many zero or near-zero values to sparse dtype
# As identified in sparse_extraction_vars
to_sparsify = extraction_vars & sparse_extraction_vars
print("sparsifying", to_sparsify)
for v in to_sparsify:
outputs[v] = outputs[v].round(6).astype(pd.SparseDtype("float", 0))
return outputs, infeasibilities, cost
def extract_data(scenario_info):
"""Builds data frames of {PG, PF, LMP, CONGU, CONGL} from Julia simulation
output binary files produced by REISE.jl.
:param dict scenario_info: scenario information.
:return: (*pandas.DataFrame*) -- data frames of:
PG, PF, LMP, CONGU, CONGL, LOAD_SHED.
"""
infeasibilities = []
cost = []
setup_time = []
solve_time = []
optimize_time = []
extraction_vars = {'pf', 'pg', 'lmp', 'congu', 'congl'}
sparse_extraction_vars = {'congu', 'congl', 'load_shed'}
temps = {}
outputs = {}
folder = os.path.join(const.EXECUTE_DIR,
'scenario_%s' % scenario_info['id'])
end_index = len(glob.glob(os.path.join(folder, 'output', 'result_*.mat')))
tic = time.process_time()
for i in tqdm(range(end_index)):
filename = 'result_' + str(i) + '.mat'
output = load_mat73(os.path.join(folder, 'output', filename))
try:
cost.append(output['mdo_save']['results']['f'][0][0])
except KeyError:
pass
demand_scaling = output['mdo_save']['demand_scaling'][0][0]
if demand_scaling < 1:
demand_change = round(100 * (1 - demand_scaling))
infeasibilities.append('%s:%s' % (str(i), str(demand_change)))
output_mpc = output['mdo_save']['flow']['mpc']
temps['pg'] = output_mpc['gen']['PG'].T
temps['pf'] = output_mpc['branch']['PF'].T
temps['lmp'] = output_mpc['bus']['LAM_P'].T
temps['congu'] = output_mpc['branch']['MU_SF'].T
temps['congl'] = output_mpc['branch']['MU_ST'].T
try:
temps['pf_dcline'] = output_mpc['dcline']['PF_dcline'].T
extraction_vars |= {'pf_dcline'}
except KeyError:
pass
try:
temps['storage_pg'] = output_mpc['storage']['PG'].T
temps['storage_e'] = output_mpc['storage']['Energy'].T
extraction_vars |= {'storage_pg', 'storage_e'}
except KeyError:
pass
try:
temps['load_shed'] = output_mpc['load_shed']['load_shed'].T
extraction_vars |= {'load_shed'}
except KeyError:
pass
for v in extraction_vars:
if v not in outputs:
interval_length, n_columns = temps[v].shape
total_length = end_index * interval_length
outputs[v] = pd.DataFrame(np.zeros((total_length, n_columns)))
outputs[v].name = str(scenario_info['id']) + '_' + v.upper()
start_hour, end_hour = (i * interval_length), ((i + 1) *
interval_length)
outputs[v].iloc[start_hour:end_hour, :] = temps[v]
print(extraction_vars)
toc = time.process_time()
print('Reading time ' + str(round(toc - tic)) + 's')
# Write infeasibilities
insert_in_file(const.SCENARIO_LIST, scenario_info['id'], '16',
'_'.join(infeasibilities))
# Build log: costs from matfiles, file attributes from ls/awk
log = pd.DataFrame(data={'cost': cost})
file_filter = os.path.join(folder, 'output', 'result_*.mat')
ls_options = '-lrt --time-style="+%Y-%m-%d %H:%M:%S" ' + file_filter
awk_options = "-v OFS=','"
awk_program = ("'BEGIN{print \"filesize,datetime,filename\"}; "
"NR >0 {print $5, $6\" \"$7, $8}'")
ls_call = "ls %s | awk %s %s" % (ls_options, awk_options, awk_program)
ls_output = subprocess.Popen(ls_call, shell=True, stdout=subprocess.PIPE)
utf_ls_output = io.StringIO(ls_output.communicate()[0].decode('utf-8'))
properties_df = pd.read_csv(utf_ls_output, sep=',', dtype=str)
log['filesize'] = properties_df.filesize
log['write_datetime'] = properties_df.datetime
# Write log
log_filename = scenario_info['id'] + '_log.csv'
log.to_csv(os.path.join(const.OUTPUT_DIR, log_filename), header=True)
# Set index of data frame
date_range = pd.date_range(scenario_info['start_date'],
scenario_info['end_date'],
freq='H')
for v in extraction_vars:
outputs[v].index = date_range
outputs[v].index.name = 'UTC'
# Get/set index column name of data frame
outputs_id = _get_outputs_id(folder)
for k in outputs:
index = outputs_id[k]
if isinstance(index, int):
outputs[k].columns = [index]
else:
outputs[k].columns = index.tolist()
print('converting to float32')
for v in extraction_vars:
outputs[v] = outputs[v].astype(np.float32)
# Convert outputs with many zero or near-zero values to sparse dtype
to_sparsify = set(extraction_vars) & sparse_extraction_vars
print('sparsifying', to_sparsify)
for v in to_sparsify:
outputs[v] = outputs[v].round(6).astype(pd.SparseDtype("float", 0))
return outputs