def validate_inputs(scenario_id, subscenarios, subproblem, stage, conn):
"""
Get inputs from database and validate the inputs
:param subscenarios: SubScenarios object with all subscenario info
:param subproblem:
:param stage:
:param conn: database connection
:return:
"""
# Validate operational chars table inputs
validate_opchars(scenario_id, subscenarios, subproblem, stage, conn,
"gen_simple")
# Other module specific validations
c = conn.cursor()
heat_rates = c.execute("""
SELECT project, load_point_fraction
FROM inputs_project_portfolios
INNER JOIN
(SELECT project, operational_type, heat_rate_curves_scenario_id
FROM inputs_project_operational_chars
WHERE project_operational_chars_scenario_id = {}
AND operational_type = '{}') AS op_char
USING(project)
INNER JOIN
(SELECT project, heat_rate_curves_scenario_id, load_point_fraction
FROM inputs_project_heat_rate_curves) as heat_rates
USING(project, heat_rate_curves_scenario_id)
WHERE project_portfolio_scenario_id = {}
""".format(
subscenarios.PROJECT_OPERATIONAL_CHARS_SCENARIO_ID,
"gen_simple",
subscenarios.PROJECT_PORTFOLIO_SCENARIO_ID,
))
# Convert inputs to dataframe
hr_df = cursor_to_df(heat_rates)
# Check that there is only one load point (constant heat rate)
write_validation_to_database(
conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_heat_rate_curves",
severity="Mid",
errors=validate_single_input(
df=hr_df,
msg="gen_simple can only have one load "
"point (constant heat rate).",
),
)
def test_validate_single_input(self):
"""
:return:
"""
df_columns = ["project", "load_point_fraction"]
test_cases = {
# Make sure correct inputs don't throw error
1: {
"df": pd.DataFrame(columns=df_columns, data=[["nuclear",
100]]),
"idx_col": "project",
"msg": "test msg",
"result": [],
},
# Make sure multiple inputs per index are flagged
2: {
"df":
pd.DataFrame(
columns=df_columns,
data=[["nuclear", 100], ["nuclear", 200], ["gas_ct", 10]],
),
"idx_col":
"project",
"msg":
"test msg",
"result": [
"project(s) 'nuclear': Too many inputs! Maximum 1 input per project. test msg"
],
},
}
for test_case in test_cases.keys():
expected_list = test_cases[test_case]["result"]
actual_list = module_to_test.validate_single_input(
df=test_cases[test_case]["df"],
idx_col=test_cases[test_case]["idx_col"],
msg=test_cases[test_case]["msg"],
)
self.assertListEqual(expected_list, actual_list)
def validate_inputs(scenario_id, subscenarios, subproblem, stage, conn):
"""
Get inputs from database and validate the inputs
:param subscenarios: SubScenarios object with all subscenario info
:param subproblem:
:param stage:
:param conn: database connection
:return:
"""
hrzs, hrz_tmps = get_inputs_from_database(scenario_id, subscenarios,
subproblem, stage, conn)
c = conn.cursor()
periods_horizons = c.execute("""
SELECT balancing_type_horizon, period, horizon
FROM periods_horizons
WHERE temporal_scenario_id = {}
AND subproblem_id = {}
and stage_id = {}
""".format(subscenarios.TEMPORAL_SCENARIO_ID, subproblem, stage))
df_hrzs = cursor_to_df(hrzs)
df_hrz_tmps = cursor_to_df(hrz_tmps)
df_periods_hrzs = cursor_to_df(periods_horizons)
# Get expected dtypes
expected_dtypes = get_expected_dtypes(
conn=conn,
tables=[
"inputs_temporal_horizons", "inputs_temporal_horizon_timepoints"
])
# Check dtypes horizons
dtype_errors, error_columns = validate_dtypes(df_hrzs, expected_dtypes)
write_validation_to_database(conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_temporal_horizons",
severity="High",
errors=dtype_errors)
# Check dtypes horizon_timepoints
dtype_errors, error_columns = validate_dtypes(df_hrz_tmps, expected_dtypes)
write_validation_to_database(conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_temporal_horizon_timepoints",
severity="High",
errors=dtype_errors)
# Check valid numeric columns are non-negative - horizons
numeric_columns = [
c for c in df_hrzs.columns if expected_dtypes[c] == "numeric"
]
valid_numeric_columns = set(numeric_columns) - set(error_columns)
write_validation_to_database(conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_temporal_horizons",
severity="Mid",
errors=validate_values(df_hrzs,
valid_numeric_columns,
"horizon",
min=0))
# Check valid numeric columns are non-negative - horizon_timepoints
numeric_columns = [
c for c in df_hrzs.columns if expected_dtypes[c] == "numeric"
]
valid_numeric_columns = set(numeric_columns) - set(error_columns)
write_validation_to_database(
conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_temporal_horizon_timepoints",
severity="Mid",
errors=validate_values(df_hrz_tmps,
valid_numeric_columns, ["horizon", "timepoint"],
min=0))
# One horizon cannot straddle multiple periods
msg = "All timepoints within a horizon should belong to the same period."
write_validation_to_database(
conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_temporal_horizon_timepoints",
severity="High",
errors=validate_single_input(
df=df_periods_hrzs,
idx_col=["balancing_type_horizon", "horizon"],
msg=msg))
# Make sure there are no missing horizon inputs
write_validation_to_database(
conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_temporal_horizon_timepoints",
severity="High",
errors=validate_missing_inputs(
df=df_hrz_tmps,
col="horizon",
idx_col=["balancing_type_horizon", "timepoint"]))
def validate_module_specific_inputs(scenario_id, subscenarios, subproblem,
stage, conn):
"""
Get inputs from database and validate the inputs
:param subscenarios: SubScenarios object with all subscenario info
:param subproblem:
:param stage:
:param conn: database connection
:return:
"""
# Validate operational chars table inputs
opchar_df = validate_opchars(scenario_id, subscenarios, subproblem, stage,
conn, "gen_must_run")
# Other module specific validations
c = conn.cursor()
heat_rates = c.execute("""
SELECT project, load_point_fraction
FROM inputs_project_portfolios
INNER JOIN
(SELECT project, operational_type, heat_rate_curves_scenario_id
FROM inputs_project_operational_chars
WHERE project_operational_chars_scenario_id = {}
AND operational_type = '{}') AS op_char
USING(project)
INNER JOIN
(SELECT project, heat_rate_curves_scenario_id, load_point_fraction
FROM inputs_project_heat_rate_curves) as heat_rates
USING(project, heat_rate_curves_scenario_id)
WHERE project_portfolio_scenario_id = {}
""".format(subscenarios.PROJECT_OPERATIONAL_CHARS_SCENARIO_ID,
"gen_must_run", subscenarios.PROJECT_PORTFOLIO_SCENARIO_ID))
# Convert inputs to data frame
hr_df = cursor_to_df(heat_rates)
# Check that there is only one load point (constant heat rate)
write_validation_to_database(conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_heat_rate_curves",
severity="Mid",
errors=validate_single_input(
df=hr_df,
msg="gen_must_run can only have one load "
"point (constant heat rate)."))
# Check that the project does not show up in any of the
# inputs_project_reserve_bas tables since gen_must_run can't provide any
# reserves
projects_by_reserve = get_projects_by_reserve(scenario_id, subscenarios,
conn)
for reserve, projects_w_ba in projects_by_reserve.items():
table = "inputs_project_" + reserve + "_bas"
reserve_errors = validate_idxs(
actual_idxs=opchar_df["project"],
invalid_idxs=projects_w_ba,
msg="gen_must_run cannot provide {}.".format(reserve))
write_validation_to_database(conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table=table,
severity="Mid",
errors=reserve_errors)