def main(args=None): """ Parse the arguments, get the data in a df, and create the plot :return: if requested, return the plot as JSON object """ if args is None: args = sys.argv[1:] parsed_args = parse_arguments(arguments=args) conn = connect_to_database(db_path=parsed_args.database) c = conn.cursor() scenario_id, scenario = get_scenario_id_and_name( scenario_id_arg=parsed_args.scenario_id, scenario_name_arg=parsed_args.scenario, c=c, script="energy_plot") tech_colors = get_tech_colors(c) tech_plotting_order = get_tech_plotting_order(c) energy_unit = get_unit(c, "energy") plot_title = "{}Energy by Period - {} - Stage {}".format( "{} - ".format(scenario) if parsed_args.scenario_name_in_title else "", parsed_args.load_zone, parsed_args.stage) plot_name = "EnergyPlot-{}-{}".format(parsed_args.load_zone, parsed_args.stage) df = get_plotting_data(conn=conn, scenario_id=scenario_id, load_zone=parsed_args.load_zone, stage=parsed_args.stage) source, x_col_reordered = process_stacked_plot_data( df=df, y_col="energy_mwh", x_col=["period"], category_col="technology") # Multi-level index in CDS will be joined into one column with "_" separator x_col_cds = "_".join(x_col_reordered) x_col_label = ", ".join([x.capitalize() for x in x_col_reordered]) plot = create_stacked_bar_plot(source=source, x_col=x_col_cds, x_label=x_col_label, y_label="Energy ({})".format(energy_unit), category_label="Technology", category_colors=tech_colors, category_order=tech_plotting_order, title=plot_title, ylimit=parsed_args.ylimit) # Show plot in HTML browser file if requested if parsed_args.show: show_plot(plot=plot, plot_name=plot_name, plot_write_directory=parsed_args.plot_write_directory, scenario=scenario) # Return plot in json format if requested if parsed_args.return_json: return json_item(plot, plot_name)
def main(args=None): """ Parse the arguments, get the data in a df, and create the plot :return: if requested, return the plot as JSON object """ if args is None: args = sys.argv[1:] parsed_args = parse_arguments(arguments=args) conn = connect_to_database(db_path=parsed_args.database) c = conn.cursor() scenario_id, scenario = get_scenario_id_and_name( scenario_id_arg=parsed_args.scenario_id, scenario_name_arg=parsed_args.scenario, c=c, script="carbon_plot" ) carbon_unit = get_unit(c, "carbon_emissions") cost_unit = get_unit(c, "cost") plot_title = "{}Carbon Emissions by Period - {} - Subproblem {} - Stage {}"\ .format( "{} - ".format(scenario) if parsed_args.scenario_name_in_title else "", parsed_args.carbon_cap_zone, parsed_args.subproblem, parsed_args.stage) plot_name = "CarbonPlot-{}-{}-{}".format( parsed_args.carbon_cap_zone, parsed_args.subproblem, parsed_args.stage) df = get_plotting_data( conn=conn, scenario_id=scenario_id, carbon_cap_zone=parsed_args.carbon_cap_zone, subproblem=parsed_args.subproblem, stage=parsed_args.stage ) plot = create_plot( df=df, title=plot_title, carbon_unit=carbon_unit, cost_unit=cost_unit, ylimit=parsed_args.ylimit ) # Show plot in HTML browser file if requested if parsed_args.show: show_plot(plot=plot, plot_name=plot_name, plot_write_directory=parsed_args.plot_write_directory, scenario=scenario) # Return plot in json format if requested if parsed_args.return_json: return json_item(plot, "plotHTMLTarget")
def main(args=None): """ Parse the arguments, get the data in a df, and create the plot :return: if requested, return the plot as JSON object """ if args is None: args = sys.argv[1:] parsed_args = parse_arguments(arguments=args) conn = connect_to_database(db_path=parsed_args.database) c = conn.cursor() scenario_id, scenario = get_scenario_id_and_name( scenario_id_arg=parsed_args.scenario_id, scenario_name_arg=parsed_args.scenario, c=c, script="dispatch_plot") tech_colors = get_tech_colors(c) tech_plotting_order = get_tech_plotting_order(c) power_unit = get_unit(c, "power") plot_title = "{}Dispatch Plot - {} - Stage {} - Timepoints {}-{}".format( "{} - ".format(scenario) if parsed_args.scenario_name_in_title else "", parsed_args.load_zone, parsed_args.stage, parsed_args.starting_tmp, parsed_args.ending_tmp) plot_name = "dispatchPlot-{}-{}-{}-{}".format(parsed_args.load_zone, parsed_args.stage, parsed_args.starting_tmp, parsed_args.ending_tmp) df = get_plotting_data(conn=conn, scenario_id=scenario_id, load_zone=parsed_args.load_zone, starting_tmp=parsed_args.starting_tmp, ending_tmp=parsed_args.ending_tmp, stage=parsed_args.stage) plot = create_plot( df=df, title=plot_title, power_unit=power_unit, tech_colors=tech_colors, tech_plotting_order=tech_plotting_order, ylimit=parsed_args.ylimit, ) # Show plot in HTML browser file if requested if parsed_args.show: show_plot(plot=plot, plot_name=plot_name, plot_write_directory=parsed_args.plot_write_directory, scenario=scenario) # Return plot in json format if requested if parsed_args.return_json: return json_item(plot, "plotHTMLTarget")
def main(args=None): """ Parse the arguments, get the data in a df, and create the plot :return: if requested, return the plot as JSON object """ if args is None: args = sys.argv[1:] parsed_args = parse_arguments(arguments=args) conn = connect_to_database(db_path=parsed_args.database) c = conn.cursor() scenario_id, scenario = get_scenario_id_and_name( scenario_id_arg=parsed_args.scenario_id, scenario_name_arg=parsed_args.scenario, c=c, script="curtailment_variable_heatmap_plot", ) energy_unit = get_unit(c, "energy") plot_title = "{}VER Curtailment by Month-Hour - {} - {} - {}".format( "{} - ".format(scenario) if parsed_args.scenario_name_in_title else "", parsed_args.load_zone, parsed_args.period, parsed_args.stage, ) plot_name = "VariableCurtailmentPlot-{}-{}-{}".format( parsed_args.load_zone, parsed_args.period, parsed_args.stage) df = get_plotting_data( conn=conn, scenario_id=scenario_id, load_zone=parsed_args.load_zone, period=parsed_args.period, stage=parsed_args.stage, ) plot = create_plot(df=df, title=plot_title, energy_unit=energy_unit, ylimit=parsed_args.ylimit) # Show plot in HTML browser file if requested if parsed_args.show: show_plot( plot=plot, plot_name=plot_name, plot_write_directory=parsed_args.plot_write_directory, scenario=scenario, ) # Return plot in json format if requested if parsed_args.return_json: return json_item(plot, "plotHTMLTarget")
def main(args=None): """ :return: """ if args is None: args = sys.argv[1:] parsed_arguments = parse_arguments(args=args) db_path = parsed_arguments.database scenario_id_arg = parsed_arguments.scenario_id scenario_name_arg = parsed_arguments.scenario scenario_location = parsed_arguments.scenario_location conn = connect_to_database(db_path=db_path) c = conn.cursor() if not parsed_arguments.quiet: print("Processing results... (connected to database {})".format(db_path)) scenario_id, scenario_name = get_scenario_id_and_name( scenario_id_arg=scenario_id_arg, scenario_name_arg=scenario_name_arg, c=c, script="process_results", ) # Determine scenario directory scenario_directory = determine_scenario_directory( scenario_location=scenario_location, scenario_name=scenario_name ) # Go through modules modules_to_use = determine_modules(scenario_directory=scenario_directory) loaded_modules = load_modules(modules_to_use) # Subscenarios subscenarios = SubScenarios(conn=conn, scenario_id=scenario_id) process_results( loaded_modules=loaded_modules, db=conn, cursor=c, scenario_id=scenario_id, subscenarios=subscenarios, quiet=parsed_arguments.quiet, ) # Close the database connection conn.close()
def main(args=None): """ Parse the arguments, get the data in a df, and create the plot :return: if requested, return the plot as JSON object """ if args is None: args = sys.argv[1:] parsed_args = parse_arguments(arguments=args) conn = connect_to_database(db_path=parsed_args.database) c = conn.cursor() scenario_id, scenario = get_scenario_id_and_name( scenario_id_arg=parsed_args.scenario_id, scenario_name_arg=parsed_args.scenario, c=c, script="capacity_factor_plot", ) tech_colors = get_tech_colors(c) plot_title = "{}Capacity Factors by Period - {} - Stage {}".format( "{} - ".format(scenario) if parsed_args.scenario_name_in_title else "", parsed_args.load_zone, parsed_args.stage, ) plot_name = "CapFactorPlot-{}-{}".format(parsed_args.load_zone, parsed_args.stage) df = get_plotting_data( conn=conn, scenario_id=scenario_id, load_zone=parsed_args.load_zone, stage=parsed_args.stage, ) plot = create_plot(df=df, title=plot_title, tech_colors=tech_colors) # Show plot in HTML browser file if requested if parsed_args.show: show_plot( plot=plot, plot_name=plot_name, plot_write_directory=parsed_args.plot_write_directory, scenario=scenario, ) # Return plot in json format if requested if parsed_args.return_json: return json_item(plot, "plotHTMLTarget")
def exit_gracefully(): """ Clean up before exit """ print("Exiting gracefully") args = sys.argv[1:] parsed_args = parse_arguments(args) db_path = parsed_args.database conn = connect_to_database(db_path) scenario_id, scenario = get_scenario_id_and_name( scenario_id_arg=parsed_args.scenario_id, scenario_name_arg=parsed_args.scenario, c=conn.cursor(), script="run_end_to_end", ) # Check if running from queue queue_order_id = check_if_in_queue(db_path, scenario) remove_from_queue_if_in_queue(db_path, scenario, queue_order_id) update_run_status(db_path, scenario, 4) conn.close()
def main(args=None): """ :return: """ # Retrieve DB location and scenario_id and/or name from args if args is None: args = sys.argv[1:] parsed_arguments = parse_arguments(args=args) db_path = parsed_arguments.database scenario_id_arg = parsed_arguments.scenario_id scenario_name_arg = parsed_arguments.scenario scenario_location = parsed_arguments.scenario_location conn = connect_to_database(db_path=db_path) c = conn.cursor() if not parsed_arguments.quiet: print("Getting inputs... (connected to database {})".format(db_path)) scenario_id, scenario_name = get_scenario_id_and_name( scenario_id_arg=scenario_id_arg, scenario_name_arg=scenario_name_arg, c=c, script="get_scenario_inputs", ) # Determine scenario directory and create it if needed scenario_directory = determine_scenario_directory( scenario_location=scenario_location, scenario_name=scenario_name) create_directory_if_not_exists(directory=scenario_directory) # Get scenario characteristics (features, scenario_id, subscenarios, subproblems) # TODO: it seems these fail silently if empty; we may want to implement # some validation optional_features = OptionalFeatures(conn=conn, scenario_id=scenario_id) subscenarios = SubScenarios(conn=conn, scenario_id=scenario_id) subproblem_structure = get_subproblem_structure_from_db( conn=conn, scenario_id=scenario_id) solver_options = SolverOptions(conn=conn, scenario_id=scenario_id) # Determine requested features and use this to determine what modules to # load for Gridpath feature_list = optional_features.get_active_features() # If any subproblem's stage list is non-empty, we have stages, so set # the stages_flag to True to pass to determine_modules below # This tells the determine_modules function to include the # stages-related modules stages_flag = any([ len(subproblem_structure.SUBPROBLEM_STAGES[subp]) > 1 for subp in list(subproblem_structure.SUBPROBLEM_STAGES.keys()) ]) # Figure out which modules to use and load the modules modules_to_use = determine_modules(features=feature_list, multi_stage=stages_flag) # Get appropriate inputs from database and write the .tab file model inputs write_model_inputs( scenario_directory=scenario_directory, subproblem_structure=subproblem_structure, modules_to_use=modules_to_use, scenario_id=scenario_id, subscenarios=subscenarios, db_path=db_path, n_parallel_subproblems=int(parsed_arguments.n_parallel_get_inputs), ) # Save the list of optional features to a file (will be used to determine # modules without database connection) write_features_csv(scenario_directory=scenario_directory, feature_list=feature_list) # Write full scenario description write_scenario_description( scenario_directory=scenario_directory, scenario_id=scenario_id, scenario_name=scenario_name, optional_features=optional_features, subscenarios=subscenarios, ) # Write the units used for all metrics write_units_csv(scenario_directory, conn) # Write the solver options file if needed write_solver_options(scenario_directory=scenario_directory, solver_options=solver_options) # Write the subproblem linked timepoints map file if needed write_linked_subproblems_map(scenario_directory, conn, subscenarios) # Close the database connection conn.close()
def main(args=None): """ Parse the arguments, get the data in a df, and create the plot :return: if requested, return the plot as JSON object """ if args is None: args = sys.argv[1:] parsed_args = parse_arguments(arguments=args) conn = connect_to_database(db_path=parsed_args.database) c = conn.cursor() scenario_id, scenario = get_scenario_id_and_name( scenario_id_arg=parsed_args.scenario_id, scenario_name_arg=parsed_args.scenario, c=c, script="project_operations_plot", ) power_unit = get_unit(c, "power") plot_title = "{}Operations Plot - {} - {} - Stage {}".format( "{} - ".format(scenario) if parsed_args.scenario_name_in_title else "", parsed_args.project, parsed_args.period, parsed_args.stage, ) plot_name = "OperationsPlot-{}-{}-{}".format(parsed_args.project, parsed_args.period, parsed_args.stage) start = parsed_args.horizon_start end = parsed_args.horizon_end if start is not None and end is not None: appendix = " - Horizon {}-{}".format(start, end) elif start is not None and end is None: appendix = " - Horizon {}-end".format(start) elif start is None and end is not None: appendix = " - Horizon start-{}".format(end) else: appendix = "" plot_title += appendix df = get_plotting_data( conn=conn, scenario_id=scenario_id, project=parsed_args.project, period=parsed_args.period, stage=parsed_args.stage, horizon_start=parsed_args.horizon_start, horizon_end=parsed_args.horizon_end, ) plot = create_plot(df=df, title=plot_title, power_unit=power_unit, ylimit=parsed_args.ylimit) # Show plot in HTML browser file if requested if parsed_args.show: show_plot( plot=plot, plot_name=plot_name, plot_write_directory=parsed_args.plot_write_directory, scenario=scenario, ) # Return plot in json format if requested if parsed_args.return_json: return json_item(plot, "plotHTMLTarget")
def main(args=None): """ :return: """ # Retrieve scenario_id and/or name from args + "quiet" flag if args is None: args = sys.argv[1:] parsed_arguments = parse_arguments(args=args) if not parsed_arguments.quiet: print("Validating inputs...") db_path = parsed_arguments.database scenario_id_arg = parsed_arguments.scenario_id scenario_name_arg = parsed_arguments.scenario conn = connect_to_database(db_path=db_path, detect_types=sqlite3.PARSE_DECLTYPES) c = conn.cursor() scenario_id, scenario_name = get_scenario_id_and_name( scenario_id_arg=scenario_id_arg, scenario_name_arg=scenario_name_arg, c=c, script="validate_inputs") # Reset input validation status and results reset_input_validation(conn, scenario_id) # TODO: this is very similar to what's in get_scenario_inputs.py, # so we should consolidate # Get scenario characteristics (features, scenario_id, subscenarios, subproblems) optional_features = OptionalFeatures(conn=conn, scenario_id=scenario_id) subscenarios = SubScenarios(conn=conn, scenario_id=scenario_id) subproblems = SubProblems(conn=conn, scenario_id=scenario_id) # Check whether subscenario_ids are valid is_valid = validate_subscenario_ids(scenario_id, subscenarios, optional_features, conn) # Only do the detailed input validation if all required subscenario_ids # are specified (otherwise will get errors when loading data) if is_valid: # Load modules for all requested features feature_list = optional_features.get_active_features() # If any subproblem's stage list is non-empty, we have stages, so set # the stages_flag to True to pass to determine_modules below # This tells the determine_modules function to include the # stages-related modules stages_flag = any([ len(subproblems.SUBPROBLEM_STAGE_DICT[subp]) > 1 for subp in subproblems.SUBPROBLEM_STAGE_DICT.keys() ]) modules_to_use = determine_modules(features=feature_list, multi_stage=stages_flag) loaded_modules = load_modules(modules_to_use=modules_to_use) # Read in inputs from db and validate inputs for loaded modules validate_inputs(subproblems, loaded_modules, scenario_id, subscenarios, conn) else: if not parsed_arguments.quiet: print( "Invalid subscenario ID(s). Skipped detailed input validation." ) # Update validation status: update_validation_status(conn, scenario_id) # Close the database connection explicitly conn.close()
def main(args=None): """ :param args: :return: """ # Get process ID and start_time process_id = os.getpid() start_time = datetime.datetime.now() # Signal-handling directives signal.signal(signal.SIGTERM, sigterm_handler) signal.signal(signal.SIGINT, sigint_handler) if args is None: args = sys.argv[1:] parsed_args = parse_arguments(args) # Log the run if requested # If directed to do so, log e2e run scenario_directory = determine_scenario_directory( scenario_location=parsed_args.scenario_location, scenario_name=parsed_args.scenario, ) if parsed_args.log: logs_directory = create_logs_directory_if_not_exists( scenario_directory=scenario_directory, subproblem="", stage="") # Save sys.stdout so we can return to it later stdout_original = sys.stdout stderr_original = sys.stderr # The print statement will call the write() method of any object # you assign to sys.stdout (in this case the Logging object). The # write method of Logging writes both to sys.stdout and a log file # (see auxiliary/auxiliary.py) logger = Logging( logs_dir=logs_directory, start_time=start_time, e2e=True, process_id=process_id, ) sys.stdout = logger sys.stderr = logger # Create connection db_path = parsed_args.database conn = connect_to_database(db_path=db_path) scenario_id, scenario = get_scenario_id_and_name( scenario_id_arg=parsed_args.scenario_id, scenario_name_arg=parsed_args.scenario, c=conn.cursor(), script="run_end_to_end", ) conn.close() if not parsed_args.quiet: print("Running scenario {} end to end".format(scenario)) # Check if running from queue queue_order_id = check_if_in_queue(db_path, scenario) # Update run status to 'running' update_run_status(db_path, scenario, 1) # Record process ID and process start time in database if not parsed_args.quiet: print("Process ID is {}".format(process_id)) print("End-to-end run started on {}".format(start_time)) record_process_id_and_start_time(db_path, parsed_args.scenario, process_id, start_time) # Figure out which steps we are skipping if user has requested a single # E2E step; start by assuming we'll skip and reverse skipping if the # step is specified skip_get_inputs = True skip_run_scenario = True skip_import_results = True skip_process_results = True if parsed_args.single_e2e_step_only == "get_inputs": skip_get_inputs = False elif parsed_args.single_e2e_step_only == "run_scenario": skip_run_scenario = False elif parsed_args.single_e2e_step_only == "import_results": skip_import_results = False elif parsed_args.single_e2e_step_only == "process_results": skip_process_results = False else: skip_get_inputs = False skip_run_scenario = False skip_import_results = False skip_process_results = False # Go through the steps if user has not requested to skip them if not skip_get_inputs and not parsed_args.skip_get_inputs: try: get_scenario_inputs.main(args=args) except Exception as e: logging.exception(e) end_time = update_db_for_run_end( db_path=db_path, scenario=scenario, queue_order_id=queue_order_id, process_id=process_id, run_status_id=3, ) print( "Error encountered when getting inputs from the database for " "scenario {}. End time: {}.".format(scenario, end_time)) sys.exit(1) if not skip_run_scenario and not parsed_args.skip_run_scenario: try: # make sure run_scenario.py gets the required --scenario argument run_scenario_args = args + ["--scenario", scenario] expected_objective_values = run_scenario.main( args=run_scenario_args) except Exception as e: logging.exception(e) end_time = update_db_for_run_end( db_path=db_path, scenario=scenario, queue_order_id=queue_order_id, process_id=process_id, run_status_id=3, ) print("Error encountered when running scenario {}. End time: {}.". format(scenario, end_time)) sys.exit(1) else: expected_objective_values = None if not skip_import_results and not parsed_args.skip_import_results: try: import_scenario_results.main(import_rule=_import_rule, args=args) except Exception as e: logging.exception(e) end_time = update_db_for_run_end( db_path=db_path, scenario=scenario, queue_order_id=queue_order_id, process_id=process_id, run_status_id=3, ) print("Error encountered when importing results for " "scenario {}. End time: {}.".format(scenario, end_time)) sys.exit(1) if not skip_process_results and not parsed_args.skip_process_results: try: process_results.main(args=args) except Exception as e: logging.exception(e) end_time = update_db_for_run_end( db_path=db_path, scenario=scenario, queue_order_id=queue_order_id, process_id=process_id, run_status_id=3, ) print("Error encountered when importing results for " "scenario {}. End time: {}.".format(scenario, end_time)) sys.exit(1) # If we make it here, mark run as complete and update run end time end_time = update_db_for_run_end( db_path=db_path, scenario=scenario, queue_order_id=queue_order_id, process_id=process_id, run_status_id=2, ) # TODO: should the process ID be set back to NULL? if not parsed_args.quiet: print("Done. Run finished on {}.".format(end_time)) # If logging, we need to return sys.stdout to original (i.e. stop writing # to log file) if parsed_args.log: sys.stdout = stdout_original sys.stderr = stderr_original # Return expected objective values (for testing) return expected_objective_values
def main(import_rule, args=None): """ :return: """ if args is None: args = sys.argv[1:] parsed_arguments = parse_arguments(args=args) db_path = parsed_arguments.database scenario_id_arg = parsed_arguments.scenario_id scenario_name_arg = parsed_arguments.scenario scenario_location = parsed_arguments.scenario_location quiet = parsed_arguments.quiet conn = connect_to_database(db_path=db_path) c = conn.cursor() if not parsed_arguments.quiet: print( "Importing results... (connected to database {})".format(db_path)) scenario_id, scenario_name = get_scenario_id_and_name( scenario_id_arg=scenario_id_arg, scenario_name_arg=scenario_name_arg, c=c, script="import_scenario_results", ) subproblem_structure = get_subproblem_structure_from_db( conn=conn, scenario_id=scenario_id) # Determine scenario directory scenario_directory = determine_scenario_directory( scenario_location=scenario_location, scenario_name=scenario_name) # Check that the saved scenario_id matches sc_df = pd.read_csv( os.path.join(scenario_directory, "scenario_description.csv"), header=None, index_col=0, ) scenario_id_saved = int(sc_df.loc["scenario_id", 1]) if scenario_id_saved != scenario_id: raise AssertionError("ERROR: saved scenario_id does not match") # Delete all previous results for this scenario_id # Each module also makes sure results are deleted, but this step ensures # that if a scenario_id was run with different modules before, we also # delete previously imported "phantom" results delete_scenario_results(conn=conn, scenario_id=scenario_id) # Go through modules modules_to_use = determine_modules(scenario_directory=scenario_directory) loaded_modules = load_modules(modules_to_use) # Import appropriate results into database import_scenario_results_into_database( import_rule=import_rule, loaded_modules=loaded_modules, scenario_id=scenario_id, subproblems=subproblem_structure, cursor=c, db=conn, scenario_directory=scenario_directory, quiet=quiet, ) # Close the database connection conn.close()