def validate_inputs(scenario_id, subscenarios, subproblem, stage, conn):
"""
Get inputs from database and validate the inputs
:param subscenarios: SubScenarios object with all subscenario info
:param subproblem:
:param stage:
:param conn: database connection
:return:
"""
(
min_max_builds,
supply_curve_count,
supply_curve_id,
supply_curve,
) = get_model_inputs_from_database(
scenario_id, subscenarios, subproblem, stage, conn
)
projects = get_projects(conn, scenario_id, subscenarios, "capacity_type", "dr_new")
# Convert input data into pandas DataFrame
df = cursor_to_df(min_max_builds)
df_sc = cursor_to_df(supply_curve)
dr_projects = df_sc["project"].unique()
# Check for missing project potential inputs
cols = ["min_cumulative_new_build_mwh", "max_cumulative_new_build_mwh"]
write_validation_to_database(
conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_new_potential",
severity="High",
errors=validate_missing_inputs(df, cols),
)
# Check for missing supply curve inputs
write_validation_to_database(
conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_shiftable_load_supply_curve",
severity="High",
errors=validate_idxs(
actual_idxs=dr_projects, req_idxs=projects, idx_label="project"
),
)
def validate_module_specific_inputs(scenario_id, subscenarios, subproblem,
stage, conn):
"""
Get inputs from database and validate the inputs
:param subscenarios: SubScenarios object with all subscenario info
:param subproblem:
:param stage:
:param conn: database connection
:return:
"""
gen_ret_bin_params = get_module_specific_inputs_from_database(
scenario_id, subscenarios, subproblem, stage, conn)
projects = get_projects(conn, scenario_id, subscenarios, "capacity_type",
"gen_ret_bin")
# Convert input data into pandas DataFrame and extract data
df = cursor_to_df(gen_ret_bin_params)
spec_projects = df["project"].unique()
# Get expected dtypes
expected_dtypes = get_expected_dtypes(
conn=conn,
tables=[
"inputs_project_specified_capacity",
"inputs_project_specified_fixed_cost"
])
# Check dtypes
dtype_errors, error_columns = validate_dtypes(df, expected_dtypes)
write_validation_to_database(conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_specified_capacity, "
"inputs_project_specified_fixed_cost",
severity="High",
errors=dtype_errors)
# Check valid numeric columns are non-negative
numeric_columns = [
c for c in df.columns if expected_dtypes[c] == "numeric"
]
valid_numeric_columns = set(numeric_columns) - set(error_columns)
write_validation_to_database(conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_specified_capacity, "
"inputs_project_specified_fixed_cost",
severity="High",
errors=validate_values(df,
valid_numeric_columns,
min=0))
# Ensure project capacity & fixed cost is specified in at least 1 period
msg = "Expected specified capacity & fixed costs for at least one period."
write_validation_to_database(conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_specified_capacity, "
"inputs_project_specified_fixed_cost",
severity="High",
errors=validate_idxs(
actual_idxs=spec_projects,
req_idxs=projects,
idx_label="project",
msg=msg))
# Check for missing values (vs. missing row entries above)
cols = ["specified_capacity_mw", "annual_fixed_cost_per_mw_year"]
write_validation_to_database(conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_specified_capacity, "
"inputs_project_specified_fixed_cost",
severity="High",
errors=validate_missing_inputs(df, cols))
def validate_inputs(scenario_id, subscenarios, subproblem, stage, conn):
"""
Get inputs from database and validate the inputs
:param subscenarios: SubScenarios object with all subscenario info
:param subproblem:
:param stage:
:param conn: database connection
:return:
"""
new_stor_costs = get_model_inputs_from_database(
scenario_id, subscenarios, subproblem, stage, conn
)
projects = get_projects(
conn, scenario_id, subscenarios, "capacity_type", "stor_new_lin"
)
# Convert input data into pandas DataFrame
cost_df = cursor_to_df(new_stor_costs)
df_cols = cost_df.columns
# get the project lists
cost_projects = cost_df["project"].unique()
# Get expected dtypes
expected_dtypes = get_expected_dtypes(
conn=conn, tables=["inputs_project_new_cost", "inputs_project_new_potential"]
)
# Check dtypes
dtype_errors, error_columns = validate_dtypes(cost_df, expected_dtypes)
write_validation_to_database(
conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_new_cost",
severity="High",
errors=dtype_errors,
)
# Check valid numeric columns are non-negative
numeric_columns = [c for c in cost_df.columns if expected_dtypes[c] == "numeric"]
valid_numeric_columns = set(numeric_columns) - set(error_columns)
write_validation_to_database(
conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_new_cost",
severity="High",
errors=validate_values(cost_df, valid_numeric_columns, min=0),
)
# Check that all binary new build projects are available in >=1 vintage
msg = "Expected cost data for at least one vintage."
write_validation_to_database(
conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_new_cost",
severity="Mid",
errors=validate_idxs(
actual_idxs=cost_projects, req_idxs=projects, idx_label="project", msg=msg
),
)
cols = ["min_cumulative_new_build_mw", "max_cumulative_new_build_mw"]
# Check that maximum new build doesn't decrease
if cols[1] in df_cols:
write_validation_to_database(
conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_new_potential",
severity="Mid",
errors=validate_row_monotonicity(
df=cost_df, col=cols[1], rank_col="vintage"
),
)
# check that min build <= max build
if set(cols).issubset(set(df_cols)):
write_validation_to_database(
conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_new_potential",
severity="High",
errors=validate_column_monotonicity(
df=cost_df, cols=cols, idx_col=["project", "vintage"]
),
)
cols = ["min_cumulative_new_build_mwh", "max_cumulative_new_build_mwh"]
# Check that maximum new build doesn't decrease - MWh
if cols[1] in df_cols:
write_validation_to_database(
conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_new_potential",
severity="Mid",
errors=validate_row_monotonicity(
df=cost_df, col=cols[1], rank_col="vintage"
),
)
# check that min build <= max build - MWh
if set(cols).issubset(set(df_cols)):
write_validation_to_database(
conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_new_potential",
severity="High",
errors=validate_column_monotonicity(
df=cost_df, cols=cols, idx_col=["project", "vintage"]
),
)
def validate_module_specific_inputs(scenario_id, subscenarios, subproblem,
stage, conn):
"""
Get inputs from database and validate the inputs
:param subscenarios: SubScenarios object with all subscenario info
:param subproblem:
:param stage:
:param conn: database connection
:return:
"""
# TODO: check that there are no minimum duration inputs for this type
# (duration is specified by specifying the build size in mw and mwh)
# Maybe also check all other required / not required inputs?
# --> see example in gen_must_run operational_type. Seems very verbose and
# hard to maintain. Is there a way to generalize this?
# Get the binary build generator inputs
new_stor_costs, new_stor_build_size = \
get_module_specific_inputs_from_database(
scenario_id, subscenarios, subproblem, stage, conn)
projects = get_projects(conn, scenario_id, subscenarios, "capacity_type",
"stor_new_bin")
# Convert input data into pandas DataFrame
cost_df = cursor_to_df(new_stor_costs)
bld_size_df = cursor_to_df(new_stor_build_size)
# get the project lists
cost_projects = cost_df["project"].unique()
bld_size_projects = bld_size_df["project"]
# Get expected dtypes
expected_dtypes = get_expected_dtypes(
conn=conn,
tables=[
"inputs_project_new_cost", "inputs_project_new_binary_build_size"
])
# Check dtypes - cost_df
dtype_errors, error_columns = validate_dtypes(cost_df, expected_dtypes)
write_validation_to_database(conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_new_cost",
severity="High",
errors=dtype_errors)
# Check valid numeric columns are non-negative - cost_df
numeric_columns = [
c for c in cost_df.columns if expected_dtypes[c] == "numeric"
]
valid_numeric_columns = set(numeric_columns) - set(error_columns)
write_validation_to_database(conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_new_cost",
severity="High",
errors=validate_values(cost_df,
valid_numeric_columns,
min=0))
# Check dtypes - bld_size_df
dtype_errors, error_columns = validate_dtypes(bld_size_df, expected_dtypes)
write_validation_to_database(
conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_new_binary_build_size",
severity="High",
errors=dtype_errors)
# Check valid numeric columns are non-negative - bld_size_df
numeric_columns = [
c for c in bld_size_df.columns if expected_dtypes[c] == "numeric"
]
valid_numeric_columns = set(numeric_columns) - set(error_columns)
write_validation_to_database(
conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_new_binary_build_size",
severity="High",
errors=validate_values(bld_size_df, valid_numeric_columns, min=0))
# Check that all binary new build projects are available in >=1 vintage
msg = "Expected cost data for at least one vintage."
write_validation_to_database(conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_new_cost",
severity="High",
errors=validate_idxs(
actual_idxs=cost_projects,
req_idxs=projects,
idx_label="project",
msg=msg))
# Check that all binary new build projects have build size specified
write_validation_to_database(
conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_new_binary_build_size",
severity="High",
errors=validate_idxs(actual_idxs=bld_size_projects,
req_idxs=projects,
idx_label="project"))
def validate_module_specific_inputs(scenario_id, subscenarios, subproblem, stage, conn):
"""
Get inputs from database and validate the inputs
:param subscenarios: SubScenarios object with all subscenario info
:param subproblem:
:param stage:
:param conn: database connection
:return:
"""
# Get the binary build generator inputs
new_gen_costs, new_build_size = get_module_specific_inputs_from_database(
scenario_id, subscenarios, subproblem, stage, conn)
projects = get_projects(conn, scenario_id, subscenarios, "capacity_type", "gen_new_bin")
# Convert input data into pandas DataFrame
cost_df = cursor_to_df(new_gen_costs)
bld_size_df = cursor_to_df(new_build_size)
# get the project lists
cost_projects = cost_df["project"].unique()
bld_size_projects = bld_size_df["project"]
# Get expected dtypes
expected_dtypes = get_expected_dtypes(
conn=conn,
tables=["inputs_project_new_cost",
"inputs_project_new_binary_build_size"]
)
# Check dtypes - cost_df
dtype_errors, error_columns = validate_dtypes(cost_df, expected_dtypes)
write_validation_to_database(
conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_new_cost",
severity="High",
errors=dtype_errors
)
# Check valid numeric columns are non-negative - cost_df
numeric_columns = [c for c in cost_df.columns
if expected_dtypes[c] == "numeric"]
valid_numeric_columns = set(numeric_columns) - set(error_columns)
write_validation_to_database(
conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_new_cost",
severity="High",
errors=validate_values(cost_df, valid_numeric_columns, min=0)
)
# Check dtypes - bld_size_df
dtype_errors, error_columns = validate_dtypes(bld_size_df, expected_dtypes)
write_validation_to_database(
conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_new_binary_build_size",
severity="High",
errors=dtype_errors
)
# Check valid numeric columns are non-negative - bld_size_df
numeric_columns = [c for c in bld_size_df.columns
if expected_dtypes[c] == "numeric"]
valid_numeric_columns = set(numeric_columns) - set(error_columns)
write_validation_to_database(
conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_new_binary_build_size",
severity="High",
errors=validate_values(bld_size_df, valid_numeric_columns, min=0)
)
# Check that all binary new build projects are available in >=1 vintage
msg = "Expected cost data for at least one vintage."
write_validation_to_database(
conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_new_cost",
severity="Mid",
errors=validate_idxs(actual_idxs=cost_projects,
req_idxs=projects,
idx_label="project",
msg=msg)
)
# Check that all binary new build projects have build size specified
write_validation_to_database(
conn=conn,
scenario_id=scenario_id,
subproblem_id=subproblem,
stage_id=stage,
gridpath_module=__name__,
db_table="inputs_project_new_binary_build_size",
severity="High",
errors=validate_idxs(actual_idxs=bld_size_projects,
req_idxs=projects,
idx_label="project")
)