def add_attributes(model_run): attr_dict = AttrDict() attr_dict['model'] = model_run.model.copy() attr_dict['run'] = model_run.run.copy() # Some keys are killed right away for k in ['model.time', 'model.data_path', 'model.timeseries_data_path', 'run.config_run_path', 'run.model']: try: attr_dict.del_key(k) except KeyError: pass # Now we flatten the AttrDict into a dict attr_dict = attr_dict.as_dict(flat=True) # Anything empty or None in the flattened dict is also killed for k in list(attr_dict.keys()): val = attr_dict[k] if val is None or (hasattr(val, '__iter__') and not val): del attr_dict[k] attr_dict['calliope_version'] = __version__ attr_dict['applied_overrides'] = model_run['applied_overrides'] attr_dict['scenario'] = model_run['scenario'] default_tech_dict = checks.defaults.default_tech.as_dict() default_location_dict = checks.defaults.default_location.as_dict() attr_dict['defaults'] = ruamel.yaml.dump({ **default_tech_dict['constraints'], **{'cost_{}'.format(k): v for k, v in default_tech_dict['costs']['default'].items()}, **default_location_dict }) return attr_dict
def add_attributes(model_run): attr_dict = AttrDict() attr_dict['model'] = model_run.model.copy() attr_dict['run'] = model_run.run.copy() # Some keys are killed right away for k in ['model.time', 'model.data_path', 'model.timeseries_data_path', 'run.config_run_path', 'run.model']: try: attr_dict.del_key(k) except KeyError: pass # Now we flatten the AttrDict into a dict attr_dict = attr_dict.as_dict(flat=True) # Anything empty or None in the flattened dict is also killed for k in list(attr_dict.keys()): val = attr_dict[k] if val is None or (hasattr(val, '__iter__') and not val): del attr_dict[k] attr_dict['calliope_version'] = __version__ default_tech_dict = checks.defaults.default_tech.as_dict() default_location_dict = checks.defaults.default_location.as_dict() attr_dict['defaults'] = ruamel.yaml.dump({ **default_tech_dict['constraints'], **{'cost_{}'.format(k): v for k, v in default_tech_dict['costs']['default'].items()}, **default_location_dict }) return attr_dict
def process_locations(model_config, modelrun_techs): """ Process locations by taking an AttrDict that may include compact keys such as ``1,2,3``, and returning an AttrDict with: * exactly one key per location with all of its settings * fully resolved installed technologies for each location * fully expanded transmission links for each location Parameters ---------- model_config : AttrDict modelrun_techs : AttrDict Returns ------- locations : AttrDict locations_comments : AttrDict """ techs_in = model_config.techs.copy() tech_groups_in = model_config.tech_groups locations_in = model_config.locations links_in = model_config.get("links", AttrDict()) allowed_from_file = DEFAULTS.model.file_allowed warnings = [] errors = [] locations_comments = AttrDict() ## # Expand compressed `loc1,loc2,loc3,loc4: ...` definitions ## locations = AttrDict() for key in locations_in: if ("--" in key) or ("," in key): key_locs = explode_locations(key) for subkey in key_locs: _set_loc_key(locations, subkey, locations_in[key]) else: _set_loc_key(locations, key, locations_in[key]) ## # Kill any locations that the modeller does not want to exist ## for loc in list(locations.keys()): if not locations[loc].get("exists", True): locations.del_key(loc) ## # Process technologies ## techs_to_delete = [] for tech_name in techs_in: if not techs_in[tech_name].get("exists", True): techs_to_delete.append(tech_name) continue # Get inheritance chain generated in process_techs() inheritance_chain = modelrun_techs[tech_name].inheritance # Get and save list of required_constraints from base technology base_tech = inheritance_chain[-1] rq = model_config.tech_groups[base_tech].required_constraints # locations[loc_name].techs[tech_name].required_constraints = rq techs_in[tech_name].required_constraints = rq # Kill any techs that the modeller does not want to exist for tech_name in techs_to_delete: del techs_in[tech_name] ## # Fully expand all installed technologies for the location, # filling in any undefined parameters from defaults ## location_techs_to_delete = [] for loc_name, loc in locations.items(): if "techs" not in loc: # Mark this as a transmission-only node if it has not allowed # any technologies locations[loc_name].transmission_node = True locations_comments.set_key( "{}.transmission_node".format(loc_name), "Automatically inserted: specifies that this node is " "a transmission-only node.", ) continue # No need to process any technologies at this node for tech_name in loc.techs: if tech_name in techs_to_delete: # Techs that were removed need not be further considered continue if not isinstance(locations[loc_name].techs[tech_name], dict): locations[loc_name].techs[tech_name] = AttrDict() # Starting at top of the inheritance chain, for each level, # check if the level has location-specific group settings # and keep merging together the settings, overwriting as we # go along. tech_settings = AttrDict() for parent in reversed(modelrun_techs[tech_name].inheritance): # Does the parent group have model-wide settings? tech_settings.union(tech_groups_in[parent], allow_override=True) # Does the parent group have location-specific settings? if ("tech_groups" in locations[loc_name] and parent in locations[loc_name].tech_groups): tech_settings.union( locations[loc_name].tech_groups[parent], allow_override=True) # Now overwrite with the tech's own model-wide # and location-specific settings tech_settings.union(techs_in[tech_name], allow_override=True) if tech_name in locations[loc_name].techs: tech_settings.union(locations[loc_name].techs[tech_name], allow_override=True) tech_settings = cleanup_undesired_keys(tech_settings) # Resolve columns in filename if necessary file_or_df_configs = [ i for i in tech_settings.keys_nested() if (isinstance(tech_settings.get_key(i), str) and ( "file=" in tech_settings.get_key(i) or "df=" in tech_settings.get_key(i))) ] for config_key in file_or_df_configs: config_value = tech_settings.get_key(config_key, "") if ":" not in config_value: config_value = "{}:{}".format(config_value, loc_name) tech_settings.set_key(config_key, config_value) tech_settings = check_costs_and_compute_depreciation_rates( tech_name, loc_name, tech_settings, warnings, errors) # Now merge the tech settings into the location-specific # tech dict -- but if a tech specifies ``exists: false``, # we kill it at this location if not tech_settings.get("exists", True): location_techs_to_delete.append("{}.techs.{}".format( loc_name, tech_name)) else: locations[loc_name].techs[tech_name].union(tech_settings, allow_override=True) for k in location_techs_to_delete: locations.del_key(k) # Generate all transmission links processed_links = AttrDict() for link in links_in: loc_from, loc_to = [i.strip() for i in link.split(",")] # Skip this link entirely if it has been told not to exist if not links_in[link].get("exists", True): continue # Also skip this link - and warn about it - if it links to a # now-inexistant (because removed) location if loc_from not in locations.keys() or loc_to not in locations.keys(): warnings.append( "Not building the link {},{} because one or both of its " "locations have been removed from the model by setting " "``exists: false``".format(loc_from, loc_to)) continue processed_transmission_techs = AttrDict() for tech_name in links_in[link].techs: # Skip techs that have been told not to exist # for this particular link if not links_in[link].get_key("techs.{}.exists".format(tech_name), True): continue if tech_name not in processed_transmission_techs: tech_settings = AttrDict() # Combine model-wide settings from all parent groups for parent in reversed(modelrun_techs[tech_name].inheritance): tech_settings.union(tech_groups_in[parent], allow_override=True) # Now overwrite with the tech's own model-wide settings tech_settings.union(techs_in[tech_name], allow_override=True) # Add link-specific constraint overrides if links_in[link].techs[tech_name]: tech_settings.union(links_in[link].techs[tech_name], allow_override=True) tech_settings = cleanup_undesired_keys(tech_settings) tech_settings = process_per_distance_constraints( tech_name, tech_settings, locations, locations_comments, loc_from, loc_to, ) tech_settings = check_costs_and_compute_depreciation_rates( tech_name, link, tech_settings, warnings, errors) processed_transmission_techs[tech_name] = tech_settings else: tech_settings = processed_transmission_techs[tech_name] processed_links.set_key( "{}.links.{}.techs.{}".format(loc_from, loc_to, tech_name), tech_settings.copy(), ) processed_links.set_key( "{}.links.{}.techs.{}".format(loc_to, loc_from, tech_name), tech_settings.copy(), ) # If this is a one-way link, we set the constraints for energy_prod # and energy_con accordingly on both parts of the link if tech_settings.get_key("constraints.one_way", False): processed_links.set_key( "{}.links.{}.techs.{}.constraints.energy_prod".format( loc_from, loc_to, tech_name), False, ) processed_links.set_key( "{}.links.{}.techs.{}.constraints.energy_con".format( loc_to, loc_from, tech_name), False, ) locations.union(processed_links, allow_override=True) return locations, locations_comments, list(set(warnings)), list( set(errors))
def process_locations(model_config, modelrun_techs): """ Process locations by taking an AttrDict that may include compact keys such as ``1,2,3``, and returning an AttrDict with: * exactly one key per location with all of its settings * fully resolved installed technologies for each location * fully expanded transmission links for each location Parameters ---------- model_config : AttrDict modelrun_techs : AttrDict Returns ------- locations : AttrDict locations_comments : AttrDict """ techs_in = model_config.techs.copy() tech_groups_in = model_config.tech_groups locations_in = model_config.locations links_in = model_config.get('links', AttrDict()) allowed_from_file = defaults['file_allowed'] warnings = [] errors = [] locations_comments = AttrDict() ## # Expand compressed `loc1,loc2,loc3,loc4: ...` definitions ## locations = AttrDict() for key in locations_in: if ('--' in key) or (',' in key): key_locs = explode_locations(key) for subkey in key_locs: _set_loc_key(locations, subkey, locations_in[key]) else: _set_loc_key(locations, key, locations_in[key]) ## # Kill any locations that the modeller does not want to exist ## for loc in list(locations.keys()): if not locations[loc].get('exists', True): locations.del_key(loc) ## # Process technologies ## techs_to_delete = [] for tech_name in techs_in: if not techs_in[tech_name].get('exists', True): techs_to_delete.append(tech_name) continue # Get inheritance chain generated in process_techs() inheritance_chain = modelrun_techs[tech_name].inheritance # Get and save list of required_constraints from base technology base_tech = inheritance_chain[-1] rq = model_config.tech_groups[base_tech].required_constraints # locations[loc_name].techs[tech_name].required_constraints = rq techs_in[tech_name].required_constraints = rq # Kill any techs that the modeller does not want to exist for tech_name in techs_to_delete: del techs_in[tech_name] ## # Fully expand all installed technologies for the location, # filling in any undefined parameters from defaults ## location_techs_to_delete = [] for loc_name, loc in locations.items(): if 'techs' not in loc: # Mark this as a transmission-only node if it has not allowed # any technologies locations[loc_name].transmission_node = True locations_comments.set_key( '{}.transmission_node'.format(loc_name), 'Automatically inserted: specifies that this node is ' 'a transmission-only node.' ) continue # No need to process any technologies at this node for tech_name in loc.techs: if tech_name in techs_to_delete: # Techs that were removed need not be further considered continue if not isinstance(locations[loc_name].techs[tech_name], dict): locations[loc_name].techs[tech_name] = AttrDict() # Starting at top of the inheritance chain, for each level, # check if the level has location-specific group settings # and keep merging together the settings, overwriting as we # go along. tech_settings = AttrDict() for parent in reversed(modelrun_techs[tech_name].inheritance): # Does the parent group have model-wide settings? tech_settings.union(tech_groups_in[parent], allow_override=True) # Does the parent group have location-specific settings? if ('tech_groups' in locations[loc_name] and parent in locations[loc_name].tech_groups): tech_settings.union( locations[loc_name].tech_groups[parent], allow_override=True) # Now overwrite with the tech's own model-wide # and location-specific settings tech_settings.union(techs_in[tech_name], allow_override=True) if tech_name in locations[loc_name].techs: tech_settings.union( locations[loc_name].techs[tech_name], allow_override=True) tech_settings = cleanup_undesired_keys(tech_settings) # Resolve columns in filename if necessary file_configs = [ i for i in tech_settings.keys_nested() if (isinstance(tech_settings.get_key(i), str) and 'file=' in tech_settings.get_key(i)) ] for config_key in file_configs: if config_key.split('.')[-1] not in allowed_from_file: # Allow any custom settings that end with _time_varying # FIXME: add this to docs if config_key.endswith('_time_varying'): warn('Using custom constraint ' '{} with time-varying data.'.format(config_key)) else: raise ModelError('`file=` not allowed in {}'.format(config_key)) config_value = tech_settings.get_key(config_key, '') if ':' not in config_value: config_value = '{}:{}'.format(config_value, loc_name) tech_settings.set_key(config_key, config_value) tech_settings = compute_depreciation_rates(tech_name, tech_settings, warnings, errors) # Now merge the tech settings into the location-specific # tech dict -- but if a tech specifies ``exists: false``, # we kill it at this location if not tech_settings.get('exists', True): location_techs_to_delete.append('{}.techs.{}'.format(loc_name, tech_name)) else: locations[loc_name].techs[tech_name].union( tech_settings, allow_override=True ) for k in location_techs_to_delete: locations.del_key(k) # Generate all transmission links processed_links = AttrDict() for link in links_in: loc_from, loc_to = link.split(',') # Skip this link entirely if it has been told not to exist if not links_in[link].get('exists', True): continue # Also skip this link - and warn about it - if it links to a # now-inexistant (because removed) location if (loc_from not in locations.keys() or loc_to not in locations.keys()): warnings.append( 'Not building the link {},{} because one or both of its ' 'locations have been removed from the model by setting ' '``exists: false``'.format(loc_from, loc_to) ) continue processed_transmission_techs = AttrDict() for tech_name in links_in[link].techs: # Skip techs that have been told not to exist # for this particular link if not links_in[link].get_key('techs.{}.exists'.format(tech_name), True): continue if tech_name not in processed_transmission_techs: tech_settings = AttrDict() # Combine model-wide settings from all parent groups for parent in reversed(modelrun_techs[tech_name].inheritance): tech_settings.union( tech_groups_in[parent], allow_override=True ) # Now overwrite with the tech's own model-wide settings tech_settings.union( techs_in[tech_name], allow_override=True ) # Add link-specific constraint overrides if links_in[link].techs[tech_name]: tech_settings.union( links_in[link].techs[tech_name], allow_override=True ) tech_settings = cleanup_undesired_keys(tech_settings) tech_settings = process_per_distance_constraints(tech_name, tech_settings, locations, locations_comments, loc_from, loc_to) tech_settings = compute_depreciation_rates(tech_name, tech_settings, warnings, errors) processed_transmission_techs[tech_name] = tech_settings else: tech_settings = processed_transmission_techs[tech_name] processed_links.set_key( '{}.links.{}.techs.{}'.format(loc_from, loc_to, tech_name), tech_settings.copy() ) processed_links.set_key( '{}.links.{}.techs.{}'.format(loc_to, loc_from, tech_name), tech_settings.copy() ) # If this is a one-way link, we set the constraints for energy_prod # and energy_con accordingly on both parts of the link if tech_settings.get_key('constraints.one_way', False): processed_links.set_key( '{}.links.{}.techs.{}.constraints.energy_prod'.format(loc_from, loc_to, tech_name), False) processed_links.set_key( '{}.links.{}.techs.{}.constraints.energy_con'.format(loc_to, loc_from, tech_name), False) locations.union(processed_links, allow_override=True) return locations, locations_comments, list(set(warnings)), list(set(errors))