def _run_setup_model(
model_file, scenario, model_format, override_dict):
"""
Build model in CLI commands. Returns ``model``, a ready-to-run
calliope.Model instance.
"""
# Try to determine model file type if not given explicitly
if model_format is None:
if model_file.split('.')[-1] in ['yaml', 'yml']:
model_format = 'yaml'
elif model_file.split('.')[-1] in ['nc', 'nc4', 'netcdf']:
model_format = 'netcdf'
else:
raise ValueError(
'Cannot determine model file format based on file '
'extension for "{}". Set format explicitly with '
'--model_format.'.format(model_file)
)
if model_format == 'yaml':
model = Model(
model_file, scenario=scenario, override_dict=override_dict
)
elif model_format == 'netcdf':
if scenario is not None or override_dict is not None:
raise ValueError(
'When loading a pre-built model from NetCDF, the '
'--scenario and --override_dict options are not available.'
)
model = read_netcdf(model_file)
else:
raise ValueError('Invalid model format: {}'.format(model_format))
return model
def _run_setup_model(model_file, scenario, model_format, override_dict):
"""
Build model in CLI commands. Returns ``model``, a ready-to-run
calliope.Model instance.
"""
# Try to determine model file type if not given explicitly
if model_format is None:
if model_file.split(".")[-1] in ["yaml", "yml"]:
model_format = "yaml"
elif model_file.split(".")[-1] in ["nc", "nc4", "netcdf"]:
model_format = "netcdf"
else:
raise ValueError(
"Cannot determine model file format based on file "
'extension for "{}". Set format explicitly with '
"--model_format.".format(model_file))
if model_format == "yaml":
model = Model(model_file,
scenario=scenario,
override_dict=override_dict)
elif model_format == "netcdf":
if scenario is not None or override_dict is not None:
raise ValueError(
"When loading a pre-built model from NetCDF, the "
"--scenario and --override_dict options are not available.")
model = read_netcdf(model_file)
else:
raise ValueError("Invalid model format: {}".format(model_format))
return model
def run(config_file, override_file, save_netcdf, save_csv, save_plots,
save_logs, debug, pdb, profile, profile_filename):
"""Execute the given model."""
if debug:
print(_get_version())
logging.captureWarnings(True)
start_time = datetime.datetime.now()
with format_exceptions(debug, pdb, profile, profile_filename, start_time):
if save_csv is None and save_netcdf is None:
print(
'!!!\nWARNING: Neither save_csv nor save_netcdf have been '
'specified. Model will run without saving results!\n!!!\n'
)
tstart = start_time.strftime(_time_format)
print('Calliope run starting at {}\n'.format(tstart))
override_dict = {
'run.save_logs': save_logs
}
model = Model(
config_file, override_file=override_file, override_dict=override_dict
)
model_name = model._model_run.get_key('model.name', default='None')
print('Model name: {}'.format(model_name))
msize = '{locs} locations, {techs} technologies, {times} timesteps'.format(
locs=len(model._model_run.sets['locs']),
techs=(
len(model._model_run.sets['techs_non_transmission']) +
len(model._model_run.sets['techs_transmission_names'])
),
times=len(model._model_run.sets['timesteps']))
print('Model size: {}\n'.format(msize))
print('Starting model run...')
model.run()
if save_csv:
print('Saving CSV results to directory: {}'.format(save_csv))
model.to_csv(save_csv)
if save_netcdf:
print('Saving NetCDF results to file: {}'.format(save_netcdf))
model.to_netcdf(save_netcdf)
if save_plots:
print('Saving HTML file with plots to: {}'.format(save_plots))
model.plot.summary(out_file=save_plots)
print_end_time(start_time)
def run_model(run_id, model_path, user_id, *args, **kwargs):
"""
Run calliope model using asynchronous way with Celery.
Please refer to
https://calliope.readthedocs.io/en/stable/_modules/calliope/core/model.html#Model
Parameters
----------
run_id: int, the id of api.models.Run instance.
model_path: str, the path to the model yaml file.
user_id: int, the user id of current task.
Returns
-------
dict, the outputs of model run, including netcdf, csv and plots.
"""
# Logging Config
log_file = os.path.join(os.path.dirname(os.path.dirname(model_path)),
"logs.html")
logger = get_model_logger(log_file)
# Update run status
run = Run.objects.get(id=run_id)
run.status = task_status.RUNNING
run.save()
# Model run
model = CalliopeModel(config=model_path, *args, **kwargs)
model.run()
logger.info("Backend: Model runs successfully!")
# Model outputs in csv
base_path = os.path.dirname(os.path.dirname(model_path))
logger.info("Backend: Saving model results...")
try:
if not os.path.exists(base_path + "/outputs"):
os.makedirs(base_path + "/outputs", exist_ok=True)
save_outputs = os.path.join(base_path, "outputs/model_outputs")
if os.path.exists(save_outputs):
shutil.rmtree(save_outputs)
model.to_csv(save_outputs)
logger.info("Backend: Model outputs was saved.")
except Exception as e:
logger.error("Backend: Failed to save model outputs.")
logger.error(str(e))
save_outputs = ""
# Model plots in html
try:
if not os.path.exists(base_path + "/plots"):
os.makedirs(base_path + "/plots", exist_ok=True)
save_plots = os.path.join(base_path, "plots/model_plots.html")
if os.path.exists(save_plots):
os.remove(save_plots)
model.plot.summary(to_file=save_plots)
logger.info("Backend: Model plots was saved.")
except Exception as e:
logger.error("Backend: Failed to save model plots.")
logger.error(str(e))
save_plots = ""
# Model logs in plain text
save_logs = logger.handlers[0].baseFilename
return {
"run_id": run_id,
"save_outputs": save_outputs,
"save_plots": save_plots,
"save_logs": save_logs
}
def run(model_file, override_file, save_netcdf, save_csv, save_plots,
save_logs, model_format, debug, quiet, pdb, profile, profile_filename):
"""
Execute the given model. Tries to guess from the file extension whether
``model_file`` is a YAML file or a pre-built model saved to NetCDF.
This can also explicitly be set with the --model_format=yaml or
--model_format=netcdf option.
"""
if debug:
print(_get_version())
set_quietness_level(quiet)
logging.captureWarnings(True)
pywarning_logger = logging.getLogger('py.warnings')
pywarning_logger.addHandler(console)
start_time = datetime.datetime.now()
with format_exceptions(debug, pdb, profile, profile_filename, start_time):
if save_csv is None and save_netcdf is None:
click.secho(
'\n!!!\nWARNING: No options to save results have been '
'specified.\nModel will run without saving results!\n!!!\n',
fg='red',
bold=True)
tstart = start_time.strftime(_time_format)
print('Calliope {} starting at {}\n'.format(__version__, tstart))
# Try to determine model file type if not given explicitly
if model_format is None:
if model_file.split('.')[-1] in ['yaml', 'yml']:
model_format = 'yaml'
elif model_file.split('.')[-1] in ['nc', 'nc4', 'netcdf']:
model_format = 'netcdf'
else:
raise ValueError(
'Cannot determine model file format based on file '
'extension for "{}". Set format explicitly with '
'--model_format.'.format(model_file))
if model_format == 'yaml':
override_dict = {'run.save_logs': save_logs}
model = Model(model_file,
override_file=override_file,
override_dict=override_dict)
elif model_format == 'netcdf':
if override_file is not None:
raise ValueError(
'Overrides cannot be applied when loading a pre-built '
'model from NetCDF. Please run without --override options.'
)
model = read_netcdf(model_file)
if save_logs is not None:
model._model_data.attrs['run.save_logs'] = save_logs
else:
raise ValueError('Invalid model format: {}'.format(model_format))
print(model.info() + '\n')
print('Starting model run...')
model.run()
termination = model._model_data.attrs.get('termination_condition',
'unknown')
if save_csv:
print('Saving CSV results to directory: {}'.format(save_csv))
model.to_csv(save_csv)
if save_netcdf:
print('Saving NetCDF results to file: {}'.format(save_netcdf))
model.to_netcdf(save_netcdf)
if save_plots:
if termination == 'optimal':
print('Saving HTML file with plots to: {}'.format(save_plots))
model.plot.summary(to_file=save_plots)
else:
click.secho(
'Model termination condition non-optimal. Not saving plots',
fg='red',
bold=True)
print_end_time(start_time)
def builtin_model(self):
model = Model()
model.run()
return model
def run(model_file, override_file, save_netcdf, save_csv, save_plots,
save_logs, model_format,
debug, pdb, profile, profile_filename):
"""
Execute the given model. Tries to guess from the file extension whether
``model_file`` is a YAML file or a pre-built model saved to NetCDF.
This can also explicitly be set with the --model_format=yaml or
--model_format=netcdf option.
"""
if debug:
print(_get_version())
logging.captureWarnings(True)
start_time = datetime.datetime.now()
with format_exceptions(debug, pdb, profile, profile_filename, start_time):
if save_csv is None and save_netcdf is None:
print(
'!!!\nWARNING: Neither save_csv nor save_netcdf have been '
'specified. Model will run without saving results!\n!!!\n'
)
tstart = start_time.strftime(_time_format)
print('Calliope run starting at {}\n'.format(tstart))
# Try to determine model file type if not given explicitly
if model_format is None:
if model_file.split('.')[-1] in ['yaml', 'yml']:
model_format = 'yaml'
elif model_file.split('.')[-1] in ['nc', 'nc4', 'netcdf']:
model_format = 'netcdf'
else:
raise ValueError(
'Cannot determine model file format based on file '
'extension for "{}". Set format explicitly with '
'--model_format.'.format(model_file)
)
if model_format == 'yaml':
override_dict = {
'run.save_logs': save_logs
}
model = Model(
model_file, override_file=override_file, override_dict=override_dict
)
elif model_format == 'netcdf':
if override_file is not None:
raise ValueError(
'Overrides cannot be applied when loading a pre-built '
'model from NetCDF. Please run without --override options.'
)
model = read_netcdf(model_file)
if save_logs is not None:
model._model_data.attrs['run.save_logs'] = save_logs
else:
raise ValueError('Invalid model format: {}'.format(model_format))
# FIXME: get this from model._model_data rather than _model_run
model_name = model._model_data.attrs.get('model.name', 'None')
print('Model name: {}'.format(model_name))
msize = '{locs} locations, {techs} technologies, {times} timesteps'.format(
locs=len(model._model_data.coords['locs'].values),
techs=(
len(model._model_data.coords['techs_non_transmission'].values) +
len(model._model_data.coords['techs_transmission_names'].values)
),
times=len(model._model_data.coords['timesteps'].values))
print('Model size: {}\n'.format(msize))
print('Starting model run...')
model.run()
if save_csv:
print('Saving CSV results to directory: {}'.format(save_csv))
model.to_csv(save_csv)
if save_netcdf:
print('Saving NetCDF results to file: {}'.format(save_netcdf))
model.to_netcdf(save_netcdf)
if save_plots:
print('Saving HTML file with plots to: {}'.format(save_plots))
model.plot.summary(out_file=save_plots)
print_end_time(start_time)
def _has_consistent_outputs_6_region(
model: calliope.Model) -> bool: # pragma: no cover
"""Check if model outputs (costs, generation levels, emissions) are internally consistent.
Log errors whenever they are not.
Parameters:
-----------
model: instance of SixRegionModel
"""
passing = True # Changes to False if any outputs are found to be inconsistent
cost_total_v1 = 0
costs = _get_technology_info(model=model)
sum_out = model.get_summary_outputs()
ts_out = model.get_timeseries_outputs()
res = model.results
# Normalise install costs to same temporal scale as generation costs
corrfac = model.num_timesteps / 8760
# Get list of tech-location pairs
regions = list(model._model_run['locations'].keys())
tech_locations = [i.split('_') for i in costs.index]
generation_tech_locations = [
i for i in tech_locations if i[0] != 'transmission'
]
transmission_tech_locations = [
i for i in tech_locations if i[0] == 'transmission'
]
# Test if generation technology installation costs are consistent
if model.run_mode == 'plan':
for tech, region in generation_tech_locations:
if tech == 'unmet':
continue # Unmet demand doesn't have meaningful install cost
cost_v1 = corrfac * float(costs.loc[f'{tech}_{region}', 'install']
* sum_out.loc[f'cap_{tech}_{region}'])
cost_v2 = float(
res.cost_investment.loc['monetary',
f'{region}::{tech}_{region}'])
if not np.isclose(cost_v1, cost_v2, rtol=0., atol=0.1):
logger.error(
f'Cannot recreate {tech} install costs in {region} -- '
f'manual: {cost_v1}, model: {cost_v2}.')
passing = False
cost_total_v1 += cost_v1
# Test if transmission technology installation costs are consistent
if model.run_mode == 'plan':
for tech, region, region_to in transmission_tech_locations:
cost_v1 = corrfac * float(
costs.loc[f'{tech}_{region}_{region_to}', 'install'] *
sum_out.loc[f'cap_transmission_{region}_{region_to}'])
cost_v2 = 2 * float(res.cost_investment.loc[
'monetary',
f'{region}::{tech}_{region}_{region_to}:{region_to}'])
if not np.isclose(cost_v1, cost_v2, rtol=0., atol=0.1):
logger.error(
f'Cannot recreate {tech} install costs from {region} to {region_to} -- '
f'manual: {cost_v1}, model: {cost_v2}.')
passing = False
cost_total_v1 += cost_v1
# Test if generation costs are consistent
for tech, region in generation_tech_locations:
cost_v1 = float(costs.loc[f'{tech}_{region}', 'generation'] *
sum_out.loc[f'gen_{tech}_{region}'])
cost_v2 = float(res.cost_var.loc['monetary',
f'{region}::{tech}_{region}'].sum())
if not np.isclose(cost_v1, cost_v2, rtol=0., atol=0.1):
logger.error(
f'Cannot recreate {tech} generation costs in {region} -- '
f'manual: {cost_v1}, model: {cost_v2}.')
passing = False
cost_total_v1 += cost_v1
# Test if total costs are consistent
if model.run_mode == 'plan':
cost_total_v2 = float(res.cost.loc['monetary'].sum())
if not np.isclose(cost_total_v1, cost_total_v2, rtol=0., atol=0.1):
logger.error(
f'Cannot recreate system cost -- manual: {cost_total_v1}, model: {cost_total_v2}.'
)
passing = False
# Test if emissions are consistent
for tech, region in generation_tech_locations:
emission_v1 = float(costs.loc[f'{tech}_{region}', 'emissions'] *
sum_out.loc[f'gen_{tech}_{region}'])
emission_v2 = float(
res.cost_var.loc['emissions', f'{region}::{tech}_{region}'].sum())
if not np.isclose(cost_v1, cost_v2, rtol=0., atol=0.1):
logger.error(f'Cannot recreate {tech} emissions in {region} -- '
f'manual: {emission_v1}, model: {emission_v2}.')
passing = False
cost_total_v1 += cost_v1
# Test if supply matches demand
generation_total = float(
sum_out.filter(regex='gen_.*_region.*', axis=0).sum())
demand_total = float(sum_out.loc['demand_total'])
if not np.isclose(generation_total, demand_total, rtol=0., atol=0.1):
logger.error(
f'Supply-demand mismatch -- generation: {generation_total}, demand: {demand_total}.'
)
passing = False
# Test that generation levels are all nonnegative and add up to the demand
if not (ts_out.filter(like='gen', axis=1) >= 0).all().all():
logger.error(
f'Some generation/demand levels are negative:\n\n{ts_out}\n.')
passing = False
if not np.allclose(ts_out.filter(like='gen', axis=1).sum(axis=1),
ts_out.filter(like='demand', axis=1).sum(axis=1),
rtol=0.,
atol=0.1):
logger.error(f'Generation does not add up to demand:\n\n{ts_out}\n.')
passing = False
# Test regional power balance: generation equals demand + transmission out of region
for region in regions:
generation_total_region = ts_out.filter(regex=f'gen_.*_{region}',
axis=1).sum(axis=1)
demand_total_region = ts_out.filter(regex=f'demand_{region}',
axis=1).sum(axis=1)
transmission_total_from_region = (
ts_out.filter(regex=f'transmission_{region}_region.*',
axis=1).sum(axis=1) -
ts_out.filter(regex=f'transmission_region.*_{region}',
axis=1).sum(axis=1))
if not np.allclose(
generation_total_region,
demand_total_region + transmission_total_from_region,
rtol=0.,
atol=0.1):
balance_info = pd.DataFrame()
balance_info['generation'] = generation_total_region
balance_info['demand'] = demand_total_region
balance_info['transmission_out'] = transmission_total_from_region
logger.error(
f'Power balance not satisfied in {region}:\n\n{balance_info}\n.'
)
passing = False
return passing
def _has_consistent_outputs_1_region(
model: calliope.Model) -> bool: # pragma: no cover
"""Check if model outputs (costs, generation levels, emissions) are internally consistent.
Log errors whenever they are not.
Parameters:
-----------
model: instance of OneRegionModel
"""
passing = True # Changes to False if any outputs are found to be inconsistent
cost_total_v1 = 0
costs = _get_technology_info(model=model)
techs = list(costs.index)
sum_out = model.get_summary_outputs()
ts_out = model.get_timeseries_outputs()
res = model.results
# Normalise install costs to same temporal scale as generation costs
corrfac = model.num_timesteps / 8760
# Test if generation technology installation costs are consistent
if model.run_mode == 'plan':
for tech in techs:
if tech == 'unmet':
continue # Unmet demand doesn't have meaningful install cost
cost_v1 = corrfac * float(
costs.loc[tech, 'install'] * sum_out.loc[f'cap_{tech}_total'])
cost_v2 = float(res.cost_investment.loc['monetary',
f'region1::{tech}'])
if not np.isclose(cost_v1, cost_v2, rtol=0., atol=0.1):
logger.error(
f'Cannot recreate {tech} install costs -- manual: {cost_v1}, model: {cost_v2}.'
)
passing = False
cost_total_v1 += cost_v1
# Test if generation costs are consistent
for tech in techs:
cost_v1 = float(costs.loc[tech, 'generation'] *
sum_out.loc[f'gen_{tech}_total'])
cost_v2 = float(res.cost_var.loc['monetary', f'region1::{tech}'].sum())
if not np.isclose(cost_v1, cost_v2, rtol=0., atol=0.1):
logger.error(
f'Cannot recreate {tech} generation costs -- manual: {cost_v1}, model: {cost_v2}.'
)
passing = False
cost_total_v1 += cost_v1
# Test if total costs are consistent
if model.run_mode == 'plan':
cost_total_v2 = float(res.cost.loc['monetary'].sum())
if not np.isclose(cost_total_v1, cost_total_v2, rtol=0., atol=0.1):
logger.error(
f'Cannot recreate system cost -- manual: {cost_total_v1}, model: {cost_total_v2}.'
)
passing = False
# Test if emissions are consistent
for tech in techs:
emission_v1 = float(costs.loc[tech, 'emissions'] *
sum_out.loc[f'gen_{tech}_total'])
emission_v2 = float(res.cost_var.loc['emissions',
f'region1::{tech}'].sum())
if not np.isclose(cost_v1, cost_v2, rtol=0., atol=0.1):
logger.error(
f'Cannot recreate {tech} emissions -- manual: {emission_v1}, model: {emission_v2}.'
)
passing = False
cost_total_v1 += cost_v1
# Test if supply matches demand
generation_total = float(
sum_out.filter(regex='gen_.*_total', axis=0).sum())
demand_total = float(sum_out.loc['demand_total'])
if not np.isclose(generation_total, demand_total, rtol=0., atol=0.1):
logger.error(
f'Supply-demand mismatch -- generation: {generation_total}, demand: {demand_total}.'
)
passing = False
# Test that generation levels are all nonnegative and add up to the demand
if not (ts_out.filter(like='gen', axis=1) >= 0).all().all():
logger.error(
f'Some generation/demand levels are negative:\n\n{ts_out}\n.')
passing = False
if not np.allclose(ts_out.filter(like='gen', axis=1).sum(axis=1),
ts_out.filter(like='demand', axis=1).sum(axis=1),
rtol=0.,
atol=0.1):
logger.error(f'Generation does not add up to demand:\n\n{ts_out}\n.')
passing = False
return passing
def run(model_file, scenario, save_netcdf, save_csv, save_plots,
save_logs, model_format, override_dict,
debug, quiet, pdb, profile, profile_filename):
"""
Execute the given model. Tries to guess from the file extension whether
``model_file`` is a YAML file or a pre-built model saved to NetCDF.
This can also explicitly be set with the --model_format=yaml or
--model_format=netcdf option.
"""
if debug:
print(_get_version())
set_quietness_level(quiet)
logging.captureWarnings(True)
pywarning_logger = logging.getLogger('py.warnings')
pywarning_logger.addHandler(console)
start_time = datetime.datetime.now()
with format_exceptions(debug, pdb, profile, profile_filename, start_time):
if save_csv is None and save_netcdf is None:
click.secho(
'\n!!!\nWARNING: No options to save results have been '
'specified.\nModel will run without saving results!\n!!!\n',
fg='red', bold=True
)
tstart = start_time.strftime(_time_format)
print('Calliope {} starting at {}\n'.format(__version__, tstart))
# Try to determine model file type if not given explicitly
if model_format is None:
if model_file.split('.')[-1] in ['yaml', 'yml']:
model_format = 'yaml'
elif model_file.split('.')[-1] in ['nc', 'nc4', 'netcdf']:
model_format = 'netcdf'
else:
raise ValueError(
'Cannot determine model file format based on file '
'extension for "{}". Set format explicitly with '
'--model_format.'.format(model_file)
)
if model_format == 'yaml':
model = Model(
model_file, scenario=scenario, override_dict=override_dict
)
elif model_format == 'netcdf':
if scenario is not None or override_dict is not None:
raise ValueError(
'When loading a pre-built model from NetCDF, the '
'--scenario and --override_dict options are not available.'
)
model = read_netcdf(model_file)
else:
raise ValueError('Invalid model format: {}'.format(model_format))
if save_logs:
model._model_data.attrs['run.save_logs'] = save_logs
print(model.info() + '\n')
print('Starting model run...')
model.run()
termination = model._model_data.attrs.get(
'termination_condition', 'unknown')
if save_csv:
print('Saving CSV results to directory: {}'.format(save_csv))
model.to_csv(save_csv)
if save_netcdf:
print('Saving NetCDF results to file: {}'.format(save_netcdf))
model.to_netcdf(save_netcdf)
if save_plots:
if termination == 'optimal':
print('Saving HTML file with plots to: {}'.format(save_plots))
model.plot.summary(to_file=save_plots)
else:
click.secho(
'Model termination condition non-optimal. Not saving plots',
fg='red', bold=True
)
print_end_time(start_time)