def main(args=None): """ :return: """ if args is None: args = sys.argv[1:] parsed_arguments = parse_arguments(args=args) db_path = parsed_arguments.database scenario_id_arg = parsed_arguments.scenario_id scenario_name_arg = parsed_arguments.scenario scenario_location = parsed_arguments.scenario_location conn = connect_to_database(db_path=db_path) c = conn.cursor() if not parsed_arguments.quiet: print("Processing results... (connected to database {})".format(db_path)) scenario_id, scenario_name = get_scenario_id_and_name( scenario_id_arg=scenario_id_arg, scenario_name_arg=scenario_name_arg, c=c, script="process_results", ) # Determine scenario directory scenario_directory = determine_scenario_directory( scenario_location=scenario_location, scenario_name=scenario_name ) # Go through modules modules_to_use = determine_modules(scenario_directory=scenario_directory) loaded_modules = load_modules(modules_to_use) # Subscenarios subscenarios = SubScenarios(conn=conn, scenario_id=scenario_id) process_results( loaded_modules=loaded_modules, db=conn, cursor=c, scenario_id=scenario_id, subscenarios=subscenarios, quiet=parsed_arguments.quiet, ) # Close the database connection conn.close()
def set_up_gridpath_modules(scenario_directory, subproblem, stage): """ :return: list of the names of the modules the scenario uses, list of the loaded modules, and the populated dynamic components for the scenario Set up the modules and dynamic components for a scenario run problem instance. """ # Determine and load modules modules_to_use = determine_modules(scenario_directory=scenario_directory) loaded_modules = load_modules(modules_to_use) # Determine the dynamic components based on the needed modules and input # data # populate_dynamic_inputs(dynamic_components, loaded_modules, # scenario_directory, subproblem, stage) return modules_to_use, loaded_modules
def get_inputs_for_subproblem( scenario_directory, subproblem_structure, subproblem, make_subproblem_directories, modules_to_use, scenario_id, subscenarios, db_path, ): loaded_modules = load_modules(modules_to_use=modules_to_use) # First make inputs directory if needed # If there are subproblems/stages, input directory will be nested if make_subproblem_directories: subproblem_str = str(subproblem) else: subproblem_str = "" stages = subproblem_structure.SUBPROBLEM_STAGES[subproblem] if len(stages) == 1: make_stage_directories = False else: make_stage_directories = True for stage in stages: if make_stage_directories: stage_str = str(stage) else: stage_str = "" inputs_directory = os.path.join(scenario_directory, subproblem_str, stage_str, "inputs") if not os.path.exists(inputs_directory): os.makedirs(inputs_directory) # Delete input files that may have existed before to avoid # phantom inputs delete_prior_inputs(inputs_directory=inputs_directory) # Write model input .tab files for each of the loaded_modules if # appropriate. Note that all input files are saved in the # input_directory, even the non-temporal inputs that are not # dependent on the subproblem or stage. This simplifies the file # structure at the expense of unnecessarily duplicating # non-temporal input files such as projects.tab. conn = connect_to_database(db_path=db_path) for m in loaded_modules: if hasattr(m, "write_model_inputs"): m.write_model_inputs( scenario_directory=scenario_directory, scenario_id=scenario_id, subscenarios=subscenarios, subproblem=subproblem_str, stage=stage_str, conn=conn, ) else: pass conn.close() # If there are stages in the subproblem, we also need a pass-through # directory and to write headers of the pass-through input file # TODO: this should probably be moved to the module responsible for # writing to this file # TODO: how to deal with pass-through inputs # TODO: we probably don't need a directory for the # pass-through inputs, as it's only one file if len(stages) > 1: # Create the commitment pass-through file (also deletes any # prior results) # First create the pass-through directory if it doesn't # exist # TODO: need better handling of deleting prior results? pass_through_directory = os.path.join(scenario_directory, str(subproblem), "pass_through_inputs") if not os.path.exists(pass_through_directory): os.makedirs(pass_through_directory) with open( os.path.join(pass_through_directory, "fixed_commitment.tab"), "w", newline="", ) as fixed_commitment_file: fixed_commitment_writer = csv.writer(fixed_commitment_file, delimiter="\t", lineterminator="\n") fixed_commitment_writer.writerow([ "project", "timepoint", "stage", "final_commitment_stage", "commitment", ])
def main(args=None): """ :return: """ # Retrieve scenario_id and/or name from args + "quiet" flag if args is None: args = sys.argv[1:] parsed_arguments = parse_arguments(args=args) if not parsed_arguments.quiet: print("Validating inputs...") db_path = parsed_arguments.database scenario_id_arg = parsed_arguments.scenario_id scenario_name_arg = parsed_arguments.scenario conn = connect_to_database(db_path=db_path, detect_types=sqlite3.PARSE_DECLTYPES) c = conn.cursor() scenario_id, scenario_name = get_scenario_id_and_name( scenario_id_arg=scenario_id_arg, scenario_name_arg=scenario_name_arg, c=c, script="validate_inputs") # Reset input validation status and results reset_input_validation(conn, scenario_id) # TODO: this is very similar to what's in get_scenario_inputs.py, # so we should consolidate # Get scenario characteristics (features, scenario_id, subscenarios, subproblems) optional_features = OptionalFeatures(conn=conn, scenario_id=scenario_id) subscenarios = SubScenarios(conn=conn, scenario_id=scenario_id) subproblems = SubProblems(conn=conn, scenario_id=scenario_id) # Check whether subscenario_ids are valid is_valid = validate_subscenario_ids(scenario_id, subscenarios, optional_features, conn) # Only do the detailed input validation if all required subscenario_ids # are specified (otherwise will get errors when loading data) if is_valid: # Load modules for all requested features feature_list = optional_features.get_active_features() # If any subproblem's stage list is non-empty, we have stages, so set # the stages_flag to True to pass to determine_modules below # This tells the determine_modules function to include the # stages-related modules stages_flag = any([ len(subproblems.SUBPROBLEM_STAGE_DICT[subp]) > 1 for subp in subproblems.SUBPROBLEM_STAGE_DICT.keys() ]) modules_to_use = determine_modules(features=feature_list, multi_stage=stages_flag) loaded_modules = load_modules(modules_to_use=modules_to_use) # Read in inputs from db and validate inputs for loaded modules validate_inputs(subproblems, loaded_modules, scenario_id, subscenarios, conn) else: if not parsed_arguments.quiet: print( "Invalid subscenario ID(s). Skipped detailed input validation." ) # Update validation status: update_validation_status(conn, scenario_id) # Close the database connection explicitly conn.close()
def main(import_rule, args=None): """ :return: """ if args is None: args = sys.argv[1:] parsed_arguments = parse_arguments(args=args) db_path = parsed_arguments.database scenario_id_arg = parsed_arguments.scenario_id scenario_name_arg = parsed_arguments.scenario scenario_location = parsed_arguments.scenario_location quiet = parsed_arguments.quiet conn = connect_to_database(db_path=db_path) c = conn.cursor() if not parsed_arguments.quiet: print( "Importing results... (connected to database {})".format(db_path)) scenario_id, scenario_name = get_scenario_id_and_name( scenario_id_arg=scenario_id_arg, scenario_name_arg=scenario_name_arg, c=c, script="import_scenario_results", ) subproblem_structure = get_subproblem_structure_from_db( conn=conn, scenario_id=scenario_id) # Determine scenario directory scenario_directory = determine_scenario_directory( scenario_location=scenario_location, scenario_name=scenario_name) # Check that the saved scenario_id matches sc_df = pd.read_csv( os.path.join(scenario_directory, "scenario_description.csv"), header=None, index_col=0, ) scenario_id_saved = int(sc_df.loc["scenario_id", 1]) if scenario_id_saved != scenario_id: raise AssertionError("ERROR: saved scenario_id does not match") # Delete all previous results for this scenario_id # Each module also makes sure results are deleted, but this step ensures # that if a scenario_id was run with different modules before, we also # delete previously imported "phantom" results delete_scenario_results(conn=conn, scenario_id=scenario_id) # Go through modules modules_to_use = determine_modules(scenario_directory=scenario_directory) loaded_modules = load_modules(modules_to_use) # Import appropriate results into database import_scenario_results_into_database( import_rule=import_rule, loaded_modules=loaded_modules, scenario_id=scenario_id, subproblems=subproblem_structure, cursor=c, db=conn, scenario_directory=scenario_directory, quiet=quiet, ) # Close the database connection conn.close()
def main(args=None): """ :return: """ # Retrieve DB location and scenario_id and/or name from args if args is None: args = sys.argv[1:] parsed_arguments = parse_arguments(args=args) db_path = parsed_arguments.database scenario_id_arg = parsed_arguments.scenario_id scenario_name_arg = parsed_arguments.scenario scenario_location = parsed_arguments.scenario_location conn = connect_to_database(db_path=db_path) c = conn.cursor() if not parsed_arguments.quiet: print("Getting inputs... (connected to database {})".format(db_path)) scenario_id, scenario_name = get_scenario_id_and_name( scenario_id_arg=scenario_id_arg, scenario_name_arg=scenario_name_arg, c=c, script="get_scenario_inputs") # Determine scenario directory and create it if needed scenario_directory = determine_scenario_directory( scenario_location=scenario_location, scenario_name=scenario_name) create_directory_if_not_exists(directory=scenario_directory) # Get scenario characteristics (features, scenario_id, subscenarios, subproblems) # TODO: it seems these fail silently if empty; we may want to implement # some validation optional_features = OptionalFeatures(conn=conn, scenario_id=scenario_id) subscenarios = SubScenarios(conn=conn, scenario_id=scenario_id) subproblems = SubProblems(conn=conn, scenario_id=scenario_id) solver_options = SolverOptions(conn=conn, scenario_id=scenario_id) # Determine requested features and use this to determine what modules to # load for Gridpath feature_list = optional_features.get_active_features() # If any subproblem's stage list is non-empty, we have stages, so set # the stages_flag to True to pass to determine_modules below # This tells the determine_modules function to include the # stages-related modules stages_flag = any([ len(subproblems.SUBPROBLEM_STAGE_DICT[subp]) > 1 for subp in subproblems.SUBPROBLEM_STAGE_DICT.keys() ]) # Figure out which modules to use and load the modules modules_to_use = determine_modules(features=feature_list, multi_stage=stages_flag) loaded_modules = load_modules(modules_to_use=modules_to_use) # Get appropriate inputs from database and write the .tab file model inputs write_model_inputs(scenario_directory=scenario_directory, subproblems=subproblems, loaded_modules=loaded_modules, scenario_id=scenario_id, subscenarios=subscenarios, conn=conn) # Save the list of optional features to a file (will be used to determine # modules without database connection) write_features_csv(scenario_directory=scenario_directory, feature_list=feature_list) # Write full scenario description write_scenario_description(scenario_directory=scenario_directory, scenario_id=scenario_id, scenario_name=scenario_name, optional_features=optional_features, subscenarios=subscenarios) # Write the units used for all metrics write_units_csv(scenario_directory, conn) # Write the solver options file if needed write_solver_options(scenario_directory=scenario_directory, solver_options=solver_options) # Write the subproblem linked timepoints map file if needed write_linked_subproblems_map(scenario_directory, conn, subscenarios) # Close the database connection conn.close()