def apply_overrides(config, scenario=None, override_dict=None): """ Generate processed Model configuration, applying any scenarios overrides. Parameters ---------- config : AttrDict a model configuration AttrDict scenario : str, optional override_dict : str or dict or AttrDict, optional If a YAML string, converted to AttrDict """ debug_comments = AttrDict() base_model_config_file = os.path.join( os.path.dirname(calliope.__file__), 'config', 'model.yaml' ) config_model = AttrDict.from_yaml(base_model_config_file) # Interpret timeseries_data_path as relative config.model.timeseries_data_path = relative_path( config.config_path, config.model.timeseries_data_path ) # The input files are allowed to override other model defaults config_model.union(config, allow_override=True) # First pass of applying override dict before applying scenarios, # so that can override scenario definitions by override_dict if override_dict: if isinstance(override_dict, str): override_dict = AttrDict.from_yaml_string(override_dict) elif not isinstance(override_dict, AttrDict): override_dict = AttrDict(override_dict) warnings = checks.check_overrides(config_model, override_dict) exceptions.print_warnings_and_raise_errors(warnings=warnings) config_model.union( override_dict, allow_override=True, allow_replacement=True ) if scenario: scenarios = config_model.get('scenarios', {}) if scenario in scenarios: # Manually defined scenario names cannot be the same as single # overrides or any combination of semicolon-delimited overrides if all([i in config_model.get('overrides', {}) for i in scenario.split(',')]): raise exceptions.ModelError( 'Manually defined scenario cannot be a combination of override names.' ) if not isinstance(scenarios[scenario], str): raise exceptions.ModelError( 'Scenario definition must be string of comma-separated overrides.' ) overrides = scenarios[scenario].split(',') logger.info( 'Using scenario `{}` leading to the application of ' 'overrides `{}`.'.format(scenario, scenarios[scenario]) ) else: overrides = str(scenario).split(',') logger.info( 'Applying overrides `{}` without a ' 'specific scenario name.'.format(scenario) ) overrides_from_scenario = combine_overrides(config_model, overrides) warnings = checks.check_overrides(config_model, overrides_from_scenario) exceptions.print_warnings_and_raise_errors(warnings=warnings) config_model.union( overrides_from_scenario, allow_override=True, allow_replacement=True ) for k, v in overrides_from_scenario.as_dict_flat().items(): debug_comments.set_key( '{}'.format(k), 'Applied from override') else: overrides = [] # Second pass of applying override dict after applying scenarios, # so that scenario-based overrides are overridden by override_dict! if override_dict: config_model.union( override_dict, allow_override=True, allow_replacement=True ) for k, v in override_dict.as_dict_flat().items(): debug_comments.set_key( '{}'.format(k), 'Overridden via override dictionary.') return config_model, debug_comments, overrides, scenario
def generate_simple_sets(model_run): """ Generate basic sets for a given pre-processed ``model_run``. Parameters ---------- model_run : AttrDict """ sets = AttrDict() flat_techs = model_run.techs.as_dict(flat=True) flat_locations = model_run.locations.as_dict(flat=True) sets.resources = set( flatten_list(v for k, v in flat_techs.items() if '.carrier' in k)) sets.carriers = sets.resources - set(['resource']) sets.carrier_tiers = set( key.split('.carrier_')[1] for key in flat_techs.keys() if '.carrier_' in key) sets.costs = set( k.split('costs.')[-1].split('.')[0] for k in flat_locations.keys() if '.costs.' in k) sets.locs = set(model_run.locations.keys()) sets.techs_non_transmission = set() tech_groups = [ 'demand', 'supply', 'supply_plus', 'conversion', 'conversion_plus', 'storage' ] for tech_group in tech_groups: sets['techs_{}'.format(tech_group)] = set( k for k, v in model_run.techs.items() if v.inheritance[-1] == tech_group) sets.techs_non_transmission.update(sets['techs_{}'.format(tech_group)]) sets.techs_transmission_names = set(k for k, v in model_run.techs.items() if v.inheritance[-1] == 'transmission') # This builds the "tech:loc" expansion of transmission technologies techs_transmission = set() for loc_name, loc_config in model_run.locations.items(): for link_name, link_config in loc_config.get('links', {}).items(): for tech_name in link_config.techs: techs_transmission.add('{}:{}'.format(tech_name, link_name)) sets.techs_transmission = techs_transmission sets.techs = sets.techs_non_transmission | sets.techs_transmission_names # this extracts location coordinate information coordinates = set( k.split('.')[-1] for k in flat_locations.keys() if '.coordinates.' in k) if coordinates: sets.coordinates = coordinates # `timesteps` set is built from the results of timeseries_data processing sets.timesteps = list(model_run.timesteps.astype(str)) model_run.del_key('timesteps') # `techlists` are strings with comma-separated techs used for grouping in # some model-wide constraints sets.techlists = set() for k in model_run.model.get_key('group_share', {}).keys(): sets.techlists.add(k) # `constraint_groups` are the group names per constraint that is defined # at a group level sets.group_constraints = set() group_constraints = AttrDict({ name: data for name, data in model_run['group_constraints'].items() if data.get("exists", True) }) if len(group_constraints.keys()) > 0: sets.group_constraints.update( i.split('.')[1] for i in group_constraints.as_dict_flat().keys() if i.split('.')[1] not in ['techs', 'locs']) for constr in sets.group_constraints: sets['group_names_' + constr] = set( k for k, v in group_constraints.items() if constr in v.keys()) return sets
def apply_overrides(config, scenario=None, override_dict=None): """ Generate processed Model configuration, applying any scenarios overrides. Parameters ---------- config : AttrDict a model configuration AttrDict scenario : str, optional override_dict : str or dict or AttrDict, optional If a YAML string, converted to AttrDict """ debug_comments = AttrDict() config_model = AttrDict.from_yaml( os.path.join(os.path.dirname(calliope.__file__), "config", "defaults.yaml")) # Interpret timeseries_data_path as relative if "timeseries_data_path" in config.model: config.model.timeseries_data_path = relative_path( config.config_path, config.model.timeseries_data_path) # FutureWarning: check if config includes an explicit objective cost class. # Added in 0.6.4-dev, to be removed in v0.7.0-dev. has_explicit_cost_class = isinstance( config.get_key("run.objective_options.cost_class", None), dict) # The input files are allowed to override other model defaults config_model.union(config, allow_override=True) # First pass of applying override dict before applying scenarios, # so that can override scenario definitions by override_dict if override_dict: if isinstance(override_dict, str): override_dict = AttrDict.from_yaml_string(override_dict) elif not isinstance(override_dict, AttrDict): override_dict = AttrDict(override_dict) warning_messages = checks.check_overrides(config_model, override_dict) exceptions.print_warnings_and_raise_errors(warnings=warning_messages) # FutureWarning: If config does not include an explicit objective cost class, check override dict. # Added in 0.6.4-dev, to be removed in v0.7.0-dev. if has_explicit_cost_class is False: has_explicit_cost_class = isinstance( override_dict.get_key("run.objective_options.cost_class", None), dict) config_model.union(override_dict, allow_override=True, allow_replacement=True) if scenario: scenario_overrides = load_overrides_from_scenario( config_model, scenario) if not all(i in config_model.get("overrides", {}) for i in scenario_overrides): raise exceptions.ModelError( "Scenario definition must be a list of override or other scenario names." ) else: logger.info( "Applying the following overrides from scenario definition: {} " .format(scenario_overrides)) overrides_from_scenario = combine_overrides(config_model, scenario_overrides) warning_messages = checks.check_overrides(config_model, overrides_from_scenario) exceptions.print_warnings_and_raise_errors(warnings=warning_messages) # FutureWarning: If config nor override_dict include an explicit objective cost class, check scenario dict. # Added in 0.6.4-dev, to be removed in v0.7.0-dev if has_explicit_cost_class is False: has_explicit_cost_class = isinstance( overrides_from_scenario.get_key( "run.objective_options.cost_class", None), dict, ) config_model.union(overrides_from_scenario, allow_override=True, allow_replacement=True) for k, v in overrides_from_scenario.as_dict_flat().items(): debug_comments.set_key("{}".format(k), "Applied from override") else: scenario_overrides = [] # Second pass of applying override dict after applying scenarios, # so that scenario-based overrides are overridden by override_dict! if override_dict: config_model.union(override_dict, allow_override=True, allow_replacement=True) for k, v in override_dict.as_dict_flat().items(): debug_comments.set_key("{}".format(k), "Overridden via override dictionary.") # FutureWarning: raise cost class warning here. # Warning that there will be no default cost class in 0.7.0 # # Added in 0.6.4-dev, to be removed in v0.7.0-dev if has_explicit_cost_class is False: warnings.warn( "There will be no default cost class for the objective function in " 'v0.7.0 (currently "monetary" with a weight of 1). ' "Explicitly specify the cost class(es) you would like to use " 'under `run.objective_options.cost_class`. E.g. `{"monetary": 1}` to ' "replicate the current default.", FutureWarning, ) # Drop default nodes, links, and techs config_model.del_key("techs.default_tech") config_model.del_key("nodes.default_node") config_model.del_key("links.default_node_from,default_node_to") return config_model, debug_comments, scenario_overrides, scenario
def apply_overrides(config, scenario=None, override_dict=None): """ Generate processed Model configuration, applying any scenarios overrides. Parameters ---------- config : AttrDict a model configuration AttrDict scenario : str, optional override_dict : str or dict or AttrDict, optional If a YAML string, converted to AttrDict """ debug_comments = AttrDict() config_model = AttrDict.from_yaml(os.path.join( os.path.dirname(calliope.__file__), 'config', 'defaults.yaml' )) # Interpret timeseries_data_path as relative config.model.timeseries_data_path = relative_path( config.config_path, config.model.timeseries_data_path ) # FutureWarning: check if config includes an explicit objective cost class. # Added in 0.6.4-dev, to be removed in v0.7.0-dev. has_explicit_cost_class = isinstance(config.get_key('run.objective_options.cost_class', None), dict) # The input files are allowed to override other model defaults config_model.union(config, allow_override=True) # First pass of applying override dict before applying scenarios, # so that can override scenario definitions by override_dict if override_dict: if isinstance(override_dict, str): override_dict = AttrDict.from_yaml_string(override_dict) elif not isinstance(override_dict, AttrDict): override_dict = AttrDict(override_dict) warning_messages = checks.check_overrides(config_model, override_dict) exceptions.print_warnings_and_raise_errors(warnings=warning_messages) # FutureWarning: If config does not include an explicit objective cost class, check override dict. # Added in 0.6.4-dev, to be removed in v0.7.0-dev. if has_explicit_cost_class is False: has_explicit_cost_class = isinstance(override_dict.get_key('run.objective_options.cost_class', None), dict) config_model.union( override_dict, allow_override=True, allow_replacement=True ) if scenario: scenarios = config_model.get('scenarios', {}) if scenario in scenarios.keys(): # Manually defined scenario names cannot be the same as single # overrides or any combination of semicolon-delimited overrides if all([i in config_model.get('overrides', {}) for i in scenario.split(',')]): raise exceptions.ModelError( 'Manually defined scenario cannot be a combination of override names.' ) if not isinstance(scenarios[scenario], list): raise exceptions.ModelError( 'Scenario definition must be a list of override names.' ) overrides = [str(i) for i in scenarios[scenario]] logger.info( 'Using scenario `{}` leading to the application of ' 'overrides `{}`.'.format(scenario, overrides) ) else: overrides = str(scenario).split(',') logger.info( 'Applying the following overrides without a ' 'specific scenario name: {}'.format(overrides) ) overrides_from_scenario = combine_overrides(config_model, overrides) warning_messages = checks.check_overrides(config_model, overrides_from_scenario) exceptions.print_warnings_and_raise_errors(warnings=warning_messages) # FutureWarning: If config nor override_dict include an explicit objective cost class, check scenario dict. # Added in 0.6.4-dev, to be removed in v0.7.0-dev if has_explicit_cost_class is False: has_explicit_cost_class = isinstance(overrides_from_scenario.get_key('run.objective_options.cost_class', None), dict) config_model.union( overrides_from_scenario, allow_override=True, allow_replacement=True ) for k, v in overrides_from_scenario.as_dict_flat().items(): debug_comments.set_key( '{}'.format(k), 'Applied from override') else: overrides = [] # Second pass of applying override dict after applying scenarios, # so that scenario-based overrides are overridden by override_dict! if override_dict: config_model.union( override_dict, allow_override=True, allow_replacement=True ) for k, v in override_dict.as_dict_flat().items(): debug_comments.set_key( '{}'.format(k), 'Overridden via override dictionary.') # FutureWarning: raise cost class warning here. # Warning that there will be no default cost class in 0.7.0 # # Added in 0.6.4-dev, to be removed in v0.7.0-dev if has_explicit_cost_class is False: warnings.warn( 'There will be no default cost class for the objective function in ' 'v0.7.0 (currently "monetary" with a weight of 1). ' 'Explicitly specify the cost class(es) you would like to use ' 'under `run.objective_options.cost_class`. E.g. `{"monetary": 1}` to ' 'replicate the current default.', FutureWarning ) # Drop default locations, links, and techs config_model.del_key('techs.default_tech') config_model.del_key('locations.default_location') config_model.del_key('links.default_location_from,default_location_to') config_model.del_key('group_constraints.default_group') return config_model, debug_comments, overrides, scenario
def apply_overrides(config, override_file=None, override_dict=None): """ Generate processed Model configuration, applying any overrides. Parameters ---------- config : AttrDict a model configuration AttrDict override_file : str, optional override_dict : dict or AttrDict, optional """ debug_comments = AttrDict() base_model_config_file = os.path.join(os.path.dirname(calliope.__file__), 'config', 'model.yaml') config_model = AttrDict.from_yaml(base_model_config_file) default_tech_groups = list(config_model.tech_groups.keys()) # README CHANGED: `model` is not a list any longer - # it is now always a single file # README CHANGED: order of arguments to relative_path reversed # README CHANGED: data_path option removed -- need to make sure # that for parallel runs, data_path relative to the currently # open model config file always works # Interpret timeseries_data_path as relative config.model.timeseries_data_path = relative_path( config.config_path, config.model.timeseries_data_path) # Check if overriding coordinates are in the same coordinate system. If not, # delete all incumbent coordinates, ready for the new coordinates to come in def check_and_remove_coordinates(config_model, override): if (any( ['coordinates' in k for k in config_model.as_dict_flat().keys()]) and any( ['coordinates' in k for k in override.as_dict_flat().keys()])): config_keys = [ k for k in config_model.as_dict_flat().keys() if 'coordinates.' in k ] config_coordinates = [ k.split('coordinates.')[-1] for k in config_keys ] override_coordinates = set( k.split('coordinates.')[-1] for k in override.as_dict_flat().keys() if 'coordinates.' in k) if config_coordinates != override_coordinates: for key in config_keys: config_model.del_key(key) # The input files are allowed to override other model defaults config_model.union(config, allow_override=True) # FIXME: if applying an override that doesn't exist in model, should warn # the user about possible mis-spelling # Apply overrides via 'override_file', which contains the path to a YAML file if override_file: # Due to the possible occurrance of `C:\path_to_file\file.yaml:override` we have to split # override_file into `path_to_file`, `file.yaml` and `override` before # merging `path_to_file` and `file.yaml` back together path_to_file, override_file_with_group = os.path.split(override_file) override_file, override_groups = override_file_with_group.split(':') override_file_path = os.path.join(path_to_file, override_file) override_from_file = combine_overrides(override_file_path, override_groups) check_and_remove_coordinates(config_model, override_from_file) config_model.union(override_from_file, allow_override=True, allow_replacement=True) for k, v in override_from_file.as_dict_flat().items(): debug_comments.set_key( '{}'.format(k), 'Overridden via override: {}'.format(override_file)) # Apply overrides via 'override', which is an AttrDict if override_dict: if not isinstance(override_dict, AttrDict): override_dict = AttrDict(override_dict) check_and_remove_coordinates(config_model, override_dict) config_model.union(override_dict, allow_override=True, allow_replacement=True) for k, v in override_dict.as_dict_flat().items(): debug_comments.set_key('{}'.format(k), 'Overridden via override dictionary.') return config_model, debug_comments
def apply_overrides(config, override_file=None, override_dict=None): """ Generate processed Model configuration, applying any overrides. Parameters ---------- config : AttrDict a model configuration AttrDict override_file : str, optional override_dict : dict or AttrDict, optional """ debug_comments = AttrDict() base_model_config_file = os.path.join(os.path.dirname(calliope.__file__), 'config', 'model.yaml') config_model = AttrDict.from_yaml(base_model_config_file) default_tech_groups = list(config_model.tech_groups.keys()) # README CHANGED: `model` is not a list any longer - # it is now always a single file # README CHANGED: order of arguments to relative_path reversed # README CHANGED: data_path option removed -- need to make sure # that for parallel runs, data_path relative to the currently # open model config file always works # Interpret timeseries_data_path as relative config.model.timeseries_data_path = relative_path( config.config_path, config.model.timeseries_data_path) # The input files are allowed to override other model defaults config_model.union(config, allow_override=True) # Apply overrides via 'override_file', which contains the path to a YAML file if override_file: # Due to the possible occurrance of `C:\path_to_file\file.yaml:override` we have to split # override_file into `path_to_file`, `file.yaml` and `override` before # merging `path_to_file` and `file.yaml` back together path_to_file, override_file_with_group = os.path.split(override_file) override_file, override_groups = override_file_with_group.split(':') override_file_path = os.path.join(path_to_file, override_file) override_from_file = combine_overrides(override_file_path, override_groups) warnings = checks.check_overrides(config_model, override_from_file) exceptions.print_warnings_and_raise_errors(warnings=warnings) config_model.union(override_from_file, allow_override=True, allow_replacement=True) for k, v in override_from_file.as_dict_flat().items(): debug_comments.set_key( '{}'.format(k), 'Overridden via override: {}'.format(override_file)) # Apply overrides via 'override', which is an AttrDict if override_dict: if not isinstance(override_dict, AttrDict): override_dict = AttrDict(override_dict) warnings = checks.check_overrides(config_model, override_dict) exceptions.print_warnings_and_raise_errors(warnings=warnings) config_model.union(override_dict, allow_override=True, allow_replacement=True) for k, v in override_dict.as_dict_flat().items(): debug_comments.set_key('{}'.format(k), 'Overridden via override dictionary.') return config_model, debug_comments
def apply_overrides(config, scenario=None, override_dict=None): """ Generate processed Model configuration, applying any scenarios overrides. Parameters ---------- config : AttrDict a model configuration AttrDict scenario : str, optional override_dict : str or dict or AttrDict, optional If a YAML string, converted to AttrDict """ debug_comments = AttrDict() base_model_config_file = os.path.join( os.path.dirname(calliope.__file__), 'config', 'model.yaml' ) config_model = AttrDict.from_yaml(base_model_config_file) # Interpret timeseries_data_path as relative config.model.timeseries_data_path = relative_path( config.config_path, config.model.timeseries_data_path ) # The input files are allowed to override other model defaults config_model.union(config, allow_override=True) # First pass of applying override dict before applying scenarios, # so that can override scenario definitions by override_dict if override_dict: if isinstance(override_dict, str): override_dict = AttrDict.from_yaml_string(override_dict) elif not isinstance(override_dict, AttrDict): override_dict = AttrDict(override_dict) warnings = checks.check_overrides(config_model, override_dict) exceptions.print_warnings_and_raise_errors(warnings=warnings) config_model.union( override_dict, allow_override=True, allow_replacement=True ) if scenario: scenarios = config_model.get('scenarios', {}) if scenario in scenarios.keys(): # Manually defined scenario names cannot be the same as single # overrides or any combination of semicolon-delimited overrides if all([i in config_model.get('overrides', {}) for i in scenario.split(',')]): raise exceptions.ModelError( 'Manually defined scenario cannot be a combination of override names.' ) if not isinstance(scenarios[scenario], list): raise exceptions.ModelError( 'Scenario definition must be a list of override names.' ) overrides = [str(i) for i in scenarios[scenario]] logger.info( 'Using scenario `{}` leading to the application of ' 'overrides `{}`.'.format(scenario, overrides) ) else: overrides = str(scenario).split(',') logger.info( 'Applying the following overrides without a ' 'specific scenario name: {}'.format(overrides) ) overrides_from_scenario = combine_overrides(config_model, overrides) warnings = checks.check_overrides(config_model, overrides_from_scenario) exceptions.print_warnings_and_raise_errors(warnings=warnings) config_model.union( overrides_from_scenario, allow_override=True, allow_replacement=True ) for k, v in overrides_from_scenario.as_dict_flat().items(): debug_comments.set_key( '{}'.format(k), 'Applied from override') else: overrides = [] # Second pass of applying override dict after applying scenarios, # so that scenario-based overrides are overridden by override_dict! if override_dict: config_model.union( override_dict, allow_override=True, allow_replacement=True ) for k, v in override_dict.as_dict_flat().items(): debug_comments.set_key( '{}'.format(k), 'Overridden via override dictionary.') return config_model, debug_comments, overrides, scenario