def main(args=None):
"""
:return:
"""
if args is None:
args = sys.argv[1:]
parsed_arguments = parse_arguments(args=args)
db_path = parsed_arguments.database
scenario_id_arg = parsed_arguments.scenario_id
scenario_name_arg = parsed_arguments.scenario
scenario_location = parsed_arguments.scenario_location
conn = connect_to_database(db_path=db_path)
c = conn.cursor()
if not parsed_arguments.quiet:
print("Processing results... (connected to database {})".format(db_path))
scenario_id, scenario_name = get_scenario_id_and_name(
scenario_id_arg=scenario_id_arg,
scenario_name_arg=scenario_name_arg,
c=c,
script="process_results",
)
# Determine scenario directory
scenario_directory = determine_scenario_directory(
scenario_location=scenario_location, scenario_name=scenario_name
)
# Go through modules
modules_to_use = determine_modules(scenario_directory=scenario_directory)
loaded_modules = load_modules(modules_to_use)
# Subscenarios
subscenarios = SubScenarios(conn=conn, scenario_id=scenario_id)
process_results(
loaded_modules=loaded_modules,
db=conn,
cursor=c,
scenario_id=scenario_id,
subscenarios=subscenarios,
quiet=parsed_arguments.quiet,
)
# Close the database connection
conn.close()
def set_up_gridpath_modules(scenario_directory, subproblem, stage):
"""
:return: list of the names of the modules the scenario uses, list of the
loaded modules, and the populated dynamic components for the scenario
Set up the modules and dynamic components for a scenario run problem
instance.
"""
# Determine and load modules
modules_to_use = determine_modules(scenario_directory=scenario_directory)
loaded_modules = load_modules(modules_to_use)
# Determine the dynamic components based on the needed modules and input
# data
# populate_dynamic_inputs(dynamic_components, loaded_modules,
# scenario_directory, subproblem, stage)
return modules_to_use, loaded_modules
def get_inputs_for_subproblem(
scenario_directory,
subproblem_structure,
subproblem,
make_subproblem_directories,
modules_to_use,
scenario_id,
subscenarios,
db_path,
):
loaded_modules = load_modules(modules_to_use=modules_to_use)
# First make inputs directory if needed
# If there are subproblems/stages, input directory will be nested
if make_subproblem_directories:
subproblem_str = str(subproblem)
else:
subproblem_str = ""
stages = subproblem_structure.SUBPROBLEM_STAGES[subproblem]
if len(stages) == 1:
make_stage_directories = False
else:
make_stage_directories = True
for stage in stages:
if make_stage_directories:
stage_str = str(stage)
else:
stage_str = ""
inputs_directory = os.path.join(scenario_directory, subproblem_str,
stage_str, "inputs")
if not os.path.exists(inputs_directory):
os.makedirs(inputs_directory)
# Delete input files that may have existed before to avoid
# phantom inputs
delete_prior_inputs(inputs_directory=inputs_directory)
# Write model input .tab files for each of the loaded_modules if
# appropriate. Note that all input files are saved in the
# input_directory, even the non-temporal inputs that are not
# dependent on the subproblem or stage. This simplifies the file
# structure at the expense of unnecessarily duplicating
# non-temporal input files such as projects.tab.
conn = connect_to_database(db_path=db_path)
for m in loaded_modules:
if hasattr(m, "write_model_inputs"):
m.write_model_inputs(
scenario_directory=scenario_directory,
scenario_id=scenario_id,
subscenarios=subscenarios,
subproblem=subproblem_str,
stage=stage_str,
conn=conn,
)
else:
pass
conn.close()
# If there are stages in the subproblem, we also need a pass-through
# directory and to write headers of the pass-through input file
# TODO: this should probably be moved to the module responsible for
# writing to this file
# TODO: how to deal with pass-through inputs
# TODO: we probably don't need a directory for the
# pass-through inputs, as it's only one file
if len(stages) > 1:
# Create the commitment pass-through file (also deletes any
# prior results)
# First create the pass-through directory if it doesn't
# exist
# TODO: need better handling of deleting prior results?
pass_through_directory = os.path.join(scenario_directory,
str(subproblem),
"pass_through_inputs")
if not os.path.exists(pass_through_directory):
os.makedirs(pass_through_directory)
with open(
os.path.join(pass_through_directory, "fixed_commitment.tab"),
"w",
newline="",
) as fixed_commitment_file:
fixed_commitment_writer = csv.writer(fixed_commitment_file,
delimiter="\t",
lineterminator="\n")
fixed_commitment_writer.writerow([
"project",
"timepoint",
"stage",
"final_commitment_stage",
"commitment",
])
def main(args=None):
"""
:return:
"""
# Retrieve scenario_id and/or name from args + "quiet" flag
if args is None:
args = sys.argv[1:]
parsed_arguments = parse_arguments(args=args)
if not parsed_arguments.quiet:
print("Validating inputs...")
db_path = parsed_arguments.database
scenario_id_arg = parsed_arguments.scenario_id
scenario_name_arg = parsed_arguments.scenario
conn = connect_to_database(db_path=db_path,
detect_types=sqlite3.PARSE_DECLTYPES)
c = conn.cursor()
scenario_id, scenario_name = get_scenario_id_and_name(
scenario_id_arg=scenario_id_arg,
scenario_name_arg=scenario_name_arg,
c=c,
script="validate_inputs")
# Reset input validation status and results
reset_input_validation(conn, scenario_id)
# TODO: this is very similar to what's in get_scenario_inputs.py,
# so we should consolidate
# Get scenario characteristics (features, scenario_id, subscenarios, subproblems)
optional_features = OptionalFeatures(conn=conn, scenario_id=scenario_id)
subscenarios = SubScenarios(conn=conn, scenario_id=scenario_id)
subproblems = SubProblems(conn=conn, scenario_id=scenario_id)
# Check whether subscenario_ids are valid
is_valid = validate_subscenario_ids(scenario_id, subscenarios,
optional_features, conn)
# Only do the detailed input validation if all required subscenario_ids
# are specified (otherwise will get errors when loading data)
if is_valid:
# Load modules for all requested features
feature_list = optional_features.get_active_features()
# If any subproblem's stage list is non-empty, we have stages, so set
# the stages_flag to True to pass to determine_modules below
# This tells the determine_modules function to include the
# stages-related modules
stages_flag = any([
len(subproblems.SUBPROBLEM_STAGE_DICT[subp]) > 1
for subp in subproblems.SUBPROBLEM_STAGE_DICT.keys()
])
modules_to_use = determine_modules(features=feature_list,
multi_stage=stages_flag)
loaded_modules = load_modules(modules_to_use=modules_to_use)
# Read in inputs from db and validate inputs for loaded modules
validate_inputs(subproblems, loaded_modules, scenario_id, subscenarios,
conn)
else:
if not parsed_arguments.quiet:
print(
"Invalid subscenario ID(s). Skipped detailed input validation."
)
# Update validation status:
update_validation_status(conn, scenario_id)
# Close the database connection explicitly
conn.close()
def main(import_rule, args=None):
"""
:return:
"""
if args is None:
args = sys.argv[1:]
parsed_arguments = parse_arguments(args=args)
db_path = parsed_arguments.database
scenario_id_arg = parsed_arguments.scenario_id
scenario_name_arg = parsed_arguments.scenario
scenario_location = parsed_arguments.scenario_location
quiet = parsed_arguments.quiet
conn = connect_to_database(db_path=db_path)
c = conn.cursor()
if not parsed_arguments.quiet:
print(
"Importing results... (connected to database {})".format(db_path))
scenario_id, scenario_name = get_scenario_id_and_name(
scenario_id_arg=scenario_id_arg,
scenario_name_arg=scenario_name_arg,
c=c,
script="import_scenario_results",
)
subproblem_structure = get_subproblem_structure_from_db(
conn=conn, scenario_id=scenario_id)
# Determine scenario directory
scenario_directory = determine_scenario_directory(
scenario_location=scenario_location, scenario_name=scenario_name)
# Check that the saved scenario_id matches
sc_df = pd.read_csv(
os.path.join(scenario_directory, "scenario_description.csv"),
header=None,
index_col=0,
)
scenario_id_saved = int(sc_df.loc["scenario_id", 1])
if scenario_id_saved != scenario_id:
raise AssertionError("ERROR: saved scenario_id does not match")
# Delete all previous results for this scenario_id
# Each module also makes sure results are deleted, but this step ensures
# that if a scenario_id was run with different modules before, we also
# delete previously imported "phantom" results
delete_scenario_results(conn=conn, scenario_id=scenario_id)
# Go through modules
modules_to_use = determine_modules(scenario_directory=scenario_directory)
loaded_modules = load_modules(modules_to_use)
# Import appropriate results into database
import_scenario_results_into_database(
import_rule=import_rule,
loaded_modules=loaded_modules,
scenario_id=scenario_id,
subproblems=subproblem_structure,
cursor=c,
db=conn,
scenario_directory=scenario_directory,
quiet=quiet,
)
# Close the database connection
conn.close()
def main(args=None):
"""
:return:
"""
# Retrieve DB location and scenario_id and/or name from args
if args is None:
args = sys.argv[1:]
parsed_arguments = parse_arguments(args=args)
db_path = parsed_arguments.database
scenario_id_arg = parsed_arguments.scenario_id
scenario_name_arg = parsed_arguments.scenario
scenario_location = parsed_arguments.scenario_location
conn = connect_to_database(db_path=db_path)
c = conn.cursor()
if not parsed_arguments.quiet:
print("Getting inputs... (connected to database {})".format(db_path))
scenario_id, scenario_name = get_scenario_id_and_name(
scenario_id_arg=scenario_id_arg,
scenario_name_arg=scenario_name_arg,
c=c,
script="get_scenario_inputs")
# Determine scenario directory and create it if needed
scenario_directory = determine_scenario_directory(
scenario_location=scenario_location, scenario_name=scenario_name)
create_directory_if_not_exists(directory=scenario_directory)
# Get scenario characteristics (features, scenario_id, subscenarios, subproblems)
# TODO: it seems these fail silently if empty; we may want to implement
# some validation
optional_features = OptionalFeatures(conn=conn, scenario_id=scenario_id)
subscenarios = SubScenarios(conn=conn, scenario_id=scenario_id)
subproblems = SubProblems(conn=conn, scenario_id=scenario_id)
solver_options = SolverOptions(conn=conn, scenario_id=scenario_id)
# Determine requested features and use this to determine what modules to
# load for Gridpath
feature_list = optional_features.get_active_features()
# If any subproblem's stage list is non-empty, we have stages, so set
# the stages_flag to True to pass to determine_modules below
# This tells the determine_modules function to include the
# stages-related modules
stages_flag = any([
len(subproblems.SUBPROBLEM_STAGE_DICT[subp]) > 1
for subp in subproblems.SUBPROBLEM_STAGE_DICT.keys()
])
# Figure out which modules to use and load the modules
modules_to_use = determine_modules(features=feature_list,
multi_stage=stages_flag)
loaded_modules = load_modules(modules_to_use=modules_to_use)
# Get appropriate inputs from database and write the .tab file model inputs
write_model_inputs(scenario_directory=scenario_directory,
subproblems=subproblems,
loaded_modules=loaded_modules,
scenario_id=scenario_id,
subscenarios=subscenarios,
conn=conn)
# Save the list of optional features to a file (will be used to determine
# modules without database connection)
write_features_csv(scenario_directory=scenario_directory,
feature_list=feature_list)
# Write full scenario description
write_scenario_description(scenario_directory=scenario_directory,
scenario_id=scenario_id,
scenario_name=scenario_name,
optional_features=optional_features,
subscenarios=subscenarios)
# Write the units used for all metrics
write_units_csv(scenario_directory, conn)
# Write the solver options file if needed
write_solver_options(scenario_directory=scenario_directory,
solver_options=solver_options)
# Write the subproblem linked timepoints map file if needed
write_linked_subproblems_map(scenario_directory, conn, subscenarios)
# Close the database connection
conn.close()