def process_locations(model_config, modelrun_techs): """ Process locations by taking an AttrDict that may include compact keys such as ``1,2,3``, and returning an AttrDict with: * exactly one key per location with all of its settings * fully resolved installed technologies for each location * fully expanded transmission links for each location Parameters ---------- model_config : AttrDict modelrun_techs : AttrDict Returns ------- locations : AttrDict locations_comments : AttrDict """ techs_in = model_config.techs.copy() tech_groups_in = model_config.tech_groups locations_in = model_config.locations links_in = model_config.get("links", AttrDict()) allowed_from_file = DEFAULTS.model.file_allowed warnings = [] errors = [] locations_comments = AttrDict() ## # Expand compressed `loc1,loc2,loc3,loc4: ...` definitions ## locations = AttrDict() for key in locations_in: if ("--" in key) or ("," in key): key_locs = explode_locations(key) for subkey in key_locs: _set_loc_key(locations, subkey, locations_in[key]) else: _set_loc_key(locations, key, locations_in[key]) ## # Kill any locations that the modeller does not want to exist ## for loc in list(locations.keys()): if not locations[loc].get("exists", True): locations.del_key(loc) ## # Process technologies ## techs_to_delete = [] for tech_name in techs_in: if not techs_in[tech_name].get("exists", True): techs_to_delete.append(tech_name) continue # Get inheritance chain generated in process_techs() inheritance_chain = modelrun_techs[tech_name].inheritance # Get and save list of required_constraints from base technology base_tech = inheritance_chain[-1] rq = model_config.tech_groups[base_tech].required_constraints # locations[loc_name].techs[tech_name].required_constraints = rq techs_in[tech_name].required_constraints = rq # Kill any techs that the modeller does not want to exist for tech_name in techs_to_delete: del techs_in[tech_name] ## # Fully expand all installed technologies for the location, # filling in any undefined parameters from defaults ## location_techs_to_delete = [] for loc_name, loc in locations.items(): if "techs" not in loc: # Mark this as a transmission-only node if it has not allowed # any technologies locations[loc_name].transmission_node = True locations_comments.set_key( "{}.transmission_node".format(loc_name), "Automatically inserted: specifies that this node is " "a transmission-only node.", ) continue # No need to process any technologies at this node for tech_name in loc.techs: if tech_name in techs_to_delete: # Techs that were removed need not be further considered continue if not isinstance(locations[loc_name].techs[tech_name], dict): locations[loc_name].techs[tech_name] = AttrDict() # Starting at top of the inheritance chain, for each level, # check if the level has location-specific group settings # and keep merging together the settings, overwriting as we # go along. tech_settings = AttrDict() for parent in reversed(modelrun_techs[tech_name].inheritance): # Does the parent group have model-wide settings? tech_settings.union(tech_groups_in[parent], allow_override=True) # Does the parent group have location-specific settings? if ("tech_groups" in locations[loc_name] and parent in locations[loc_name].tech_groups): tech_settings.union( locations[loc_name].tech_groups[parent], allow_override=True) # Now overwrite with the tech's own model-wide # and location-specific settings tech_settings.union(techs_in[tech_name], allow_override=True) if tech_name in locations[loc_name].techs: tech_settings.union(locations[loc_name].techs[tech_name], allow_override=True) tech_settings = cleanup_undesired_keys(tech_settings) # Resolve columns in filename if necessary file_or_df_configs = [ i for i in tech_settings.keys_nested() if (isinstance(tech_settings.get_key(i), str) and ( "file=" in tech_settings.get_key(i) or "df=" in tech_settings.get_key(i))) ] for config_key in file_or_df_configs: config_value = tech_settings.get_key(config_key, "") if ":" not in config_value: config_value = "{}:{}".format(config_value, loc_name) tech_settings.set_key(config_key, config_value) tech_settings = check_costs_and_compute_depreciation_rates( tech_name, loc_name, tech_settings, warnings, errors) # Now merge the tech settings into the location-specific # tech dict -- but if a tech specifies ``exists: false``, # we kill it at this location if not tech_settings.get("exists", True): location_techs_to_delete.append("{}.techs.{}".format( loc_name, tech_name)) else: locations[loc_name].techs[tech_name].union(tech_settings, allow_override=True) for k in location_techs_to_delete: locations.del_key(k) # Generate all transmission links processed_links = AttrDict() for link in links_in: loc_from, loc_to = [i.strip() for i in link.split(",")] # Skip this link entirely if it has been told not to exist if not links_in[link].get("exists", True): continue # Also skip this link - and warn about it - if it links to a # now-inexistant (because removed) location if loc_from not in locations.keys() or loc_to not in locations.keys(): warnings.append( "Not building the link {},{} because one or both of its " "locations have been removed from the model by setting " "``exists: false``".format(loc_from, loc_to)) continue processed_transmission_techs = AttrDict() for tech_name in links_in[link].techs: # Skip techs that have been told not to exist # for this particular link if not links_in[link].get_key("techs.{}.exists".format(tech_name), True): continue if tech_name not in processed_transmission_techs: tech_settings = AttrDict() # Combine model-wide settings from all parent groups for parent in reversed(modelrun_techs[tech_name].inheritance): tech_settings.union(tech_groups_in[parent], allow_override=True) # Now overwrite with the tech's own model-wide settings tech_settings.union(techs_in[tech_name], allow_override=True) # Add link-specific constraint overrides if links_in[link].techs[tech_name]: tech_settings.union(links_in[link].techs[tech_name], allow_override=True) tech_settings = cleanup_undesired_keys(tech_settings) tech_settings = process_per_distance_constraints( tech_name, tech_settings, locations, locations_comments, loc_from, loc_to, ) tech_settings = check_costs_and_compute_depreciation_rates( tech_name, link, tech_settings, warnings, errors) processed_transmission_techs[tech_name] = tech_settings else: tech_settings = processed_transmission_techs[tech_name] processed_links.set_key( "{}.links.{}.techs.{}".format(loc_from, loc_to, tech_name), tech_settings.copy(), ) processed_links.set_key( "{}.links.{}.techs.{}".format(loc_to, loc_from, tech_name), tech_settings.copy(), ) # If this is a one-way link, we set the constraints for energy_prod # and energy_con accordingly on both parts of the link if tech_settings.get_key("constraints.one_way", False): processed_links.set_key( "{}.links.{}.techs.{}.constraints.energy_prod".format( loc_from, loc_to, tech_name), False, ) processed_links.set_key( "{}.links.{}.techs.{}.constraints.energy_con".format( loc_to, loc_from, tech_name), False, ) locations.union(processed_links, allow_override=True) return locations, locations_comments, list(set(warnings)), list( set(errors))
def generate_model_run(config, debug_comments, applied_overrides, scenario): """ Returns a processed model_run configuration AttrDict and a debug YAML object with comments attached, ready to write to disk. Parameters ---------- config : AttrDict debug_comments : AttrDict """ model_run = AttrDict() model_run['scenario'] = scenario model_run['applied_overrides'] = ';'.join(applied_overrides) # 1) Initial checks on model configuration warnings, errors = checks.check_initial(config) exceptions.print_warnings_and_raise_errors(warnings=warnings, errors=errors) # 2) Fully populate techs # Raises ModelError if necessary model_run['techs'], debug_techs, errors = process_techs(config) debug_comments.set_key('model_run.techs', debug_techs) exceptions.print_warnings_and_raise_errors(errors=errors) # 3) Fully populate tech_groups model_run['tech_groups'] = process_tech_groups(config, model_run['techs']) # 4) Fully populate locations model_run['locations'], debug_locs, warnings, errors = locations.process_locations( config, model_run['techs'] ) debug_comments.set_key('model_run.locations', debug_locs) exceptions.print_warnings_and_raise_errors(warnings=warnings, errors=errors) # 5) Fully populate timeseries data # Raises ModelErrors if there are problems with timeseries data at this stage model_run['timeseries_data'], model_run['timesteps'] = ( process_timeseries_data(config, model_run) ) # 6) Grab additional relevant bits from run and model config model_run['run'] = config['run'] model_run['model'] = config['model'] # 7) Initialize sets all_sets = sets.generate_simple_sets(model_run) all_sets.union(sets.generate_loc_tech_sets(model_run, all_sets)) all_sets = AttrDict({k: list(v) for k, v in all_sets.items()}) model_run['sets'] = all_sets model_run['constraint_sets'] = constraint_sets.generate_constraint_sets(model_run) # 8) Final sense-checking final_check_comments, warnings, errors = checks.check_final(model_run) debug_comments.union(final_check_comments) exceptions.print_warnings_and_raise_errors(warnings=warnings, errors=errors) # 9) Build a debug data dict with comments and the original configs debug_data = AttrDict({ 'comments': debug_comments, 'config_initial': config, }) return model_run, debug_data
def generate_simple_sets(model_run): """ Generate basic sets for a given pre-processed ``model_run``. Parameters ---------- model_run : AttrDict """ sets = AttrDict() flat_techs = model_run.techs.as_dict(flat=True) flat_locations = model_run.locations.as_dict(flat=True) sets.resources = set( flatten_list(v for k, v in flat_techs.items() if '.carrier' in k)) sets.carriers = sets.resources - set(['resource']) sets.carrier_tiers = set( key.split('.carrier_')[1] for key in flat_techs.keys() if '.carrier_' in key) sets.costs = set( k.split('costs.')[-1].split('.')[0] for k in flat_locations.keys() if '.costs.' in k) sets.locs = set(model_run.locations.keys()) sets.techs_non_transmission = set() tech_groups = [ 'demand', 'supply', 'supply_plus', 'conversion', 'conversion_plus', 'storage' ] for tech_group in tech_groups: sets['techs_{}'.format(tech_group)] = set( k for k, v in model_run.techs.items() if v.inheritance[-1] == tech_group) sets.techs_non_transmission.update(sets['techs_{}'.format(tech_group)]) sets.techs_transmission_names = set(k for k, v in model_run.techs.items() if v.inheritance[-1] == 'transmission') # This builds the "tech:loc" expansion of transmission technologies techs_transmission = set() for loc_name, loc_config in model_run.locations.items(): for link_name, link_config in loc_config.get('links', {}).items(): for tech_name in link_config.techs: techs_transmission.add('{}:{}'.format(tech_name, link_name)) sets.techs_transmission = techs_transmission sets.techs = sets.techs_non_transmission | sets.techs_transmission_names # this extracts location coordinate information coordinates = set( k.split('.')[-1] for k in flat_locations.keys() if '.coordinates.' in k) if coordinates: sets.coordinates = coordinates # `timesteps` set is built from the results of timeseries_data processing sets.timesteps = list(model_run.timesteps.astype(str)) model_run.del_key('timesteps') # `techlists` are strings with comma-separated techs used for grouping in # some model-wide constraints sets.techlists = set() for k in model_run.model.get_key('group_share', {}).keys(): sets.techlists.add(k) # `constraint_groups` are the group names per constraint that is defined # at a group level sets.group_constraints = set() group_constraints = AttrDict({ name: data for name, data in model_run['group_constraints'].items() if data.get("exists", True) }) if len(group_constraints.keys()) > 0: sets.group_constraints.update( i.split('.')[1] for i in group_constraints.as_dict_flat().keys() if i.split('.')[1] not in ['techs', 'locs']) for constr in sets.group_constraints: sets['group_names_' + constr] = set( k for k, v in group_constraints.items() if constr in v.keys()) return sets
def convert_model_dict(in_dict, conversion_dict, state, tech_groups=None): out_dict = AttrDict() # process techs if 'techs' in in_dict: for k, v in in_dict.techs.items(): # Remove now unsupported `unmet_demand` techs if (v.get('parent', '') in ['unmet_demand', 'unmet_demand_as_supply_tech'] or 'unmet_demand_' in k): out_dict.set_key('__disabled.techs.{}'.format(k), v) # We will want to enable ``ensure_feasibility`` to replace # ``unmet_demand`` state['ensure_feasibility'] = True continue new_tech_config = convert_subdict(v, conversion_dict['tech_config']) if 'constraints_per_distance' in v: # Convert loss to efficiency if 'e_loss' in v.constraints_per_distance: v.constraints_per_distance.e_loss = 1 - v.constraints_per_distance.e_loss new_tech_config.update( convert_subdict( v.constraints_per_distance, conversion_dict['tech_constraints_per_distance_config'] )) # Costs are a little more involved -- need to get each cost class # as a subdict and merge the results back together new_cost_dict = AttrDict() if 'costs' in v: for cost_class in v.costs: new_cost_dict[cost_class] = convert_subdict( v.costs[cost_class], conversion_dict['tech_costs_config']) if 'costs_per_distance' in v: for cost_class in v.costs_per_distance: # FIXME update not overwrite per_distance_config = convert_subdict( v.costs_per_distance[cost_class], conversion_dict['tech_costs_per_distance_config']) if cost_class in new_cost_dict: new_cost_dict[cost_class].union(per_distance_config) else: new_cost_dict[cost_class] = per_distance_config if 'depreciation' in v: # 'depreciation.interest.{cost_class}' goes to 'costs.{cost_class}.interest_rate' if 'interest' in v.depreciation: for cost_class, interest in v.depreciation.interest.items( ): new_cost_dict.set_key( '{}.interest_rate'.format(cost_class), interest) # 'depreciation.lifetime' goes to 'constraints.lifetime' if 'lifetime' in v.depreciation: new_tech_config.set_key('constraints.lifetime', v.depreciation.lifetime) if new_cost_dict: new_tech_config['costs'] = new_cost_dict # After conversion, remove legacy _per_distance top-level entries try: del new_tech_config['constraints_per_distance'] del new_tech_config['costs_per_distance'] except KeyError: pass # Assign converted techs to either tech_groups or techs if tech_groups and k in tech_groups: out_key = 'tech_groups.{}'.format(k) else: out_key = 'techs.{}'.format(k) out_dict.set_key(out_key, new_tech_config) del in_dict['techs'] # process locations if 'locations' in in_dict: new_locations_dict = AttrDict() for k, v in in_dict.locations.items(): new_locations_dict[k] = convert_subdict( v, conversion_dict['location_config']) # convert per-location constraints now in [locname].techs[techname].constraints for k, v in new_locations_dict.items(): if 'techs' in v: for tech, tech_dict in v.techs.items(): new_locations_dict[k].techs[tech] = convert_subdict( tech_dict, conversion_dict['tech_config']) # Add techs that do not specify any overrides as keys missing_techs = set(v.get_key('__disabled.techs', [])) - set( v.get('techs', {}).keys()) for tech in missing_techs: new_locations_dict[k].set_key('techs.{}'.format(tech), None) # Remove now unsupported `unmet_demand` techs for k, v in new_locations_dict.items(): for tech in list(v.techs.keys()): parent = v.get_key('techs.{}.parent'.format(tech), '') if (parent in ['unmet_demand', 'unmet_demand_as_supply_tech'] or 'unmet_demand_' in tech): new_locations_dict[k].del_key('techs.{}'.format(tech)) if '__disabled.techs' in new_locations_dict[k]: new_locations_dict[k].get_key( '__disabled.techs').append(tech) else: new_locations_dict[k].set_key('__disabled.techs', [tech]) out_dict['locations'] = new_locations_dict del in_dict['locations'] # process links if 'links' in in_dict: new_links_dict = AttrDict() for k, v in in_dict.links.items(): for tech, tech_dict in v.items(): new_links_dict.set_key( '{}.techs.{}'.format(k, tech), convert_subdict(tech_dict, conversion_dict['tech_config'])) out_dict['links'] = new_links_dict del in_dict['links'] # process metadata if 'metadata' in in_dict: # manually transfer location coordinates if 'location_coordinates' in in_dict.metadata: for k, v in in_dict.metadata.location_coordinates.items(): if isinstance(v, list): # Assume it was lat/lon new_coords = AttrDict({'lat': v[0], 'lon': v[1]}) else: new_coords = v in_dict.set_key('locations.{}.coordinates'.format(k), new_coords) del in_dict['metadata'] # Fix up any 'resource' keys that refer to 'file' only for k in [i for i in out_dict.keys_nested() if i.endswith('.resource')]: if out_dict.get_key(k) == 'file': tech = k.split('techs.')[-1].split('.')[0] out_dict.set_key(k, 'file={}_r.csv'.format(tech)) # process remaining top-level entries out_dict.union(convert_subdict(in_dict, conversion_dict['model_config'])) return out_dict
def process_timeseries_data(config_model, model_run): if config_model.model.timeseries_data is None: timeseries_data = AttrDict() else: timeseries_data = config_model.model.timeseries_data def _parser(x, dtformat): return pd.to_datetime(x, format=dtformat, exact=False) if 'timeseries_data_path' in config_model.model: dtformat = config_model.model['timeseries_dateformat'] # Generate the set of all files we want to read from file location_config = model_run.locations.as_dict_flat() model_config = config_model.model.as_dict_flat() get_filenames = lambda config: set([ v.split('=')[1].rsplit(':', 1)[0] for v in config.values() if 'file=' in str(v) ]) constraint_filenames = get_filenames(location_config) cluster_filenames = get_filenames(model_config) datetime_min = [] datetime_max = [] for file in constraint_filenames | cluster_filenames: file_path = os.path.join(config_model.model.timeseries_data_path, file) # load the data, without parsing the dates, to catch errors in the data df = pd.read_csv(file_path, index_col=0) try: df.apply(pd.to_numeric) except ValueError as e: raise exceptions.ModelError( 'Error in loading data from {}. Ensure all entries are ' 'numeric. Full error: {}'.format(file, e) ) # Now parse the dates, checking for errors specific to this try: df.index = _parser(df.index, dtformat) except ValueError as e: raise exceptions.ModelError( 'Error in parsing dates in timeseries data from {}, ' 'using datetime format `{}`: {}'.format(file, dtformat, e) ) timeseries_data[file] = df datetime_min.append(df.index[0].date()) datetime_max.append(df.index[-1].date()) # Apply time subsetting, if supplied in model_run subset_time_config = config_model.model.subset_time if subset_time_config is not None: # Test parsing dates first, to make sure they fit our required subset format try: subset_time = _parser(subset_time_config, '%Y-%m-%d %H:%M:%S') except ValueError as e: raise exceptions.ModelError( 'Timeseries subset must be in ISO format (anything up to the ' 'detail of `%Y-%m-%d %H:%M:%S`.\n User time subset: {}\n ' 'Error caused: {}'.format(subset_time_config, e) ) if isinstance(subset_time_config, list) and len(subset_time_config) == 2: time_slice = slice(subset_time_config[0], subset_time_config[1]) # Don't allow slicing outside the range of input data if (subset_time[0].date() < max(datetime_min) or subset_time[1].date() > min(datetime_max)): raise exceptions.ModelError( 'subset time range {} is outside the input data time range ' '[{}, {}]'.format(subset_time_config, max(datetime_min).strftime('%Y-%m-%d'), min(datetime_max).strftime('%Y-%m-%d')) ) elif isinstance(subset_time_config, list): raise exceptions.ModelError( 'Invalid subset_time value: {}'.format(subset_time_config) ) else: time_slice = str(subset_time_config) for k in timeseries_data.keys(): timeseries_data[k] = timeseries_data[k].loc[time_slice, :] if timeseries_data[k].empty: raise exceptions.ModelError( 'The time slice {} creates an empty timeseries array for {}' .format(time_slice, k) ) # Ensure all timeseries have the same index indices = [ (file, df.index) for file, df in timeseries_data.items() if file not in cluster_filenames ] first_file, first_index = indices[0] for file, idx in indices[1:]: if not first_index.equals(idx): raise exceptions.ModelError( 'Time series indices do not match ' 'between {} and {}'.format(first_file, file) ) return timeseries_data, first_index
def process_timeseries_data(config_model, model_run): if config_model.model.timeseries_data is None: timeseries_data = AttrDict() else: timeseries_data = config_model.model.timeseries_data if 'timeseries_data_path' in config_model.model: dtformat = config_model.model['timeseries_dateformat'] # Generate the set of all files we want to read from file flattened_config = model_run.locations.as_dict_flat() csv_files = set([ v.split('=')[1].rsplit(':', 1)[0] for v in flattened_config.values() if 'file=' in str(v) ]) for file in csv_files: file_path = os.path.join(config_model.model.timeseries_data_path, file) parser = lambda x: datetime.datetime.strptime(x, dtformat) try: df = pd.read_csv(file_path, index_col=0, parse_dates=True, date_parser=parser) except ValueError as e: raise exceptions.ModelError( "Incorrect datetime format used in {}, expecting " "`{}`, got `{}` instead" "".format(file, dtformat, e.args[0].split("'")[1])) timeseries_data[file] = df datetime_range = df.index # Apply time subsetting, if supplied in model_run subset_time_config = config_model.model.subset_time if subset_time_config is not None: if isinstance(subset_time_config, list): if len(subset_time_config) == 2: time_slice = slice(subset_time_config[0], subset_time_config[1]) if pd.to_datetime( subset_time_config[0]).date() < datetime_range[0].date( ) or pd.to_datetime(subset_time_config[1]).date( ) > datetime_range[-1].date(): raise exceptions.ModelError( 'subset time range {} is outside the input data time range [{}, {}]' .format(subset_time_config, datetime_range[0].strftime('%Y-%m-%d'), datetime_range[-1].strftime('%Y-%m-%d'))) else: raise exceptions.ModelError( 'Invalid subset_time value: {}'.format(subset_time_config)) else: time_slice = str(subset_time_config) for k in timeseries_data.keys(): timeseries_data[k] = timeseries_data[k].loc[time_slice, :] if timeseries_data[k].empty: raise exceptions.ModelError( 'The time slice {} creates an empty timeseries array for {}' .format(time_slice, k)) # Ensure all timeseries have the same index indices = [(file, df.index) for file, df in timeseries_data.items()] first_file, first_index = indices[0] for file, idx in indices[1:]: if not first_index.equals(idx): raise exceptions.ModelError('Time series indices do not match ' 'between {} and {}'.format( first_file, file)) return timeseries_data
def convert_model_dict(in_dict, conversion_dict): out_dict = AttrDict() # process techs if 'techs' in in_dict: for k, v in in_dict.techs.items(): new_tech_config = convert_subdict(v, conversion_dict['tech_config']) if 'constraints_per_distance' in v: new_tech_config.update( convert_subdict( v.constraints_per_distance, conversion_dict['tech_constraints_per_distance_config'] )) # Costs are a little more involved -- need to get each cost class # as a subdict and merge the results back together new_cost_dict = AttrDict() if 'costs' in v: for cost_class in v.costs: new_cost_dict[cost_class] = convert_subdict( v.costs[cost_class], conversion_dict['tech_costs_config']) if 'costs_per_distance' in v: for cost_class in v.costs_per_distance: # FIXME update not overwrite per_distance_config = convert_subdict( v.costs_per_distance[cost_class], conversion_dict['tech_costs_config']) if cost_class in new_cost_dict: new_cost_dict[cost_class].union(per_distance_config) else: new_cost_dict[cost_class] = per_distance_config if 'depreciation' in v: # 'depreciation.interest.{cost_class}' goes to 'costs.{cost_class}.interest_rate' if 'interest' in v.depreciation: for cost_class, interest in v.depreciation.interest.items( ): new_cost_dict.set_key( '{}.interest_rate'.format(cost_class), interest) # 'depreciation.lifetime' goes to 'constraints.lifetime' if 'lifetime' in v.depreciation: new_tech_config.set_key('constraints.lifetime', v.depreciation.lifetime) if new_cost_dict: new_tech_config['costs'] = new_cost_dict out_dict.set_key('techs.{}'.format(k), new_tech_config) del in_dict['techs'] # process locations if 'locations' in in_dict: new_locations_dict = AttrDict() for k, v in in_dict.locations.items(): new_locations_dict[k] = convert_subdict( v, conversion_dict['location_config']) # convert per-location constraints now in [locname].techs[techname].constraints for k, v in new_locations_dict.items(): if 'techs' in v: for tech, tech_dict in v.techs.items(): new_locations_dict[k].techs[tech] = convert_subdict( tech_dict, conversion_dict['tech_config']) # Add techs that do not specify any overrides as keys missing_techs = set(v.get_key('__disabled.techs', [])) - set( v.get('techs', {}).keys()) for tech in missing_techs: new_locations_dict[k].set_key('techs.{}'.format(tech), None) out_dict['locations'] = new_locations_dict del in_dict['locations'] # process metadata if 'metadata' in in_dict: # manually transfer location coordinates if 'location_coordinates' in in_dict.metadata: for k, v in in_dict.metadata.location_coordinates.items(): if isinstance(v, list): # Assume it was lat/lon new_coords = AttrDict({'lat': v[0], 'lon': v[1]}) else: new_coords = v in_dict.set_key('locations.{}.coordinates'.format(k), new_coords) del in_dict['metadata'] # process remaining top-level entries out_dict.union(convert_subdict(in_dict, conversion_dict['model_config'])) return out_dict
def generate_model_run( config, timeseries_dataframes, debug_comments, applied_overrides, scenario ): """ Returns a processed model_run configuration AttrDict and a debug YAML object with comments attached, ready to write to disk. Parameters ---------- config : AttrDict timeseries_dataframes : dict debug_comments : AttrDict scenario : str """ model_run = AttrDict() model_run["scenario"] = scenario model_run["applied_overrides"] = ";".join(applied_overrides) # 1) Initial checks on model configuration warning_messages, errors = checks.check_initial(config) exceptions.print_warnings_and_raise_errors(warnings=warning_messages, errors=errors) # 2) Fully populate techs # Raises ModelError if necessary model_run["techs"], debug_techs, errors = process_techs(config) debug_comments.set_key("model_run.techs", debug_techs) exceptions.print_warnings_and_raise_errors(errors=errors) # 3) Fully populate tech_groups model_run["tech_groups"] = process_tech_groups(config, model_run["techs"]) # 4) Fully populate locations ( model_run["locations"], debug_locs, warning_messages, errors, ) = locations.process_locations(config, model_run["techs"]) debug_comments.set_key("model_run.locations", debug_locs) exceptions.print_warnings_and_raise_errors(warnings=warning_messages, errors=errors) # 5) Fully populate timeseries data # Raises ModelErrors if there are problems with timeseries data at this stage model_run["timeseries_data"], model_run["timesteps"] = process_timeseries_data( config, model_run, timeseries_dataframes ) # 6) Grab additional relevant bits from run and model config model_run["run"] = config["run"] model_run["model"] = config["model"] model_run["group_constraints"] = config.get("group_constraints", {}) # 7) Initialize sets all_sets = sets.generate_simple_sets(model_run) all_sets.union(sets.generate_loc_tech_sets(model_run, all_sets)) all_sets = AttrDict({k: list(v) for k, v in all_sets.items()}) model_run["sets"] = all_sets model_run["constraint_sets"] = constraint_sets.generate_constraint_sets(model_run) # 8) Final sense-checking final_check_comments, warning_messages, errors = checks.check_final(model_run) debug_comments.union(final_check_comments) exceptions.print_warnings_and_raise_errors(warnings=warning_messages, errors=errors) # 9) Build a debug data dict with comments and the original configs debug_data = AttrDict({"comments": debug_comments, "config_initial": config,}) return model_run, debug_data
def process_timeseries_data(config_model, model_run, timeseries_dataframes): if config_model.model.timeseries_data is None: timeseries_data = AttrDict() else: timeseries_data = config_model.model.timeseries_data def _parser(x, dtformat): return pd.to_datetime(x, format=dtformat, exact=False) dtformat = config_model.model["timeseries_dateformat"] # Generate set of all files and dataframes we want to load location_config = model_run.locations.as_dict_flat() model_config = config_model.model.as_dict_flat() # Find names of csv files (file=) or dataframes (df=) called in config def get_names(datatype, config): return set( [ v.split("=")[1].rsplit(":", 1)[0] for v in config.values() if str(datatype) + "=" in str(v) ] ) constraint_filenames = get_names("file", location_config) cluster_filenames = get_names("file", model_config) constraint_dfnames = get_names("df", location_config) cluster_dfnames = get_names("df", model_config) # Check if timeseries_dataframes is in the correct format (dict of # pandas DataFrames) if timeseries_dataframes is not None: check_timeseries_dataframes(timeseries_dataframes) datetime_min = [] datetime_max = [] # Check there is at least one timeseries present if ( len( constraint_filenames | cluster_filenames | constraint_dfnames | cluster_dfnames ) == 0 ): raise exceptions.ModelError( "There is no timeseries in the model. At least one timeseries is " "necessary to run the model." ) # Load each timeseries into timeseries data. tskey is either a filename # (called by file=...) or a key in timeseries_dataframes (called by df=...) for tskey in ( constraint_filenames | cluster_filenames | constraint_dfnames | cluster_dfnames ): # Filenames or dict keys # If tskey is a CSV path, load the CSV, else load the dataframe if tskey in constraint_filenames | cluster_filenames: df = load_timeseries_from_file(config_model, tskey) elif tskey in constraint_dfnames | cluster_dfnames: df = load_timeseries_from_dataframe(timeseries_dataframes, tskey) try: df.apply(pd.to_numeric) except ValueError as e: raise exceptions.ModelError( "Error in loading data from {}. Ensure all entries are " "numeric. Full error: {}".format(tskey, e) ) # Parse the dates, checking for errors specific to this try: df.index = _parser(df.index, dtformat) except ValueError as e: raise exceptions.ModelError( "Error in parsing dates in timeseries data from {}, " "using datetime format `{}`: {}".format(tskey, dtformat, e) ) timeseries_data[tskey] = df datetime_min.append(df.index[0].date()) datetime_max.append(df.index[-1].date()) # Apply time subsetting, if supplied in model_run subset_time_config = config_model.model.subset_time if subset_time_config is not None: # Test parsing dates first, to make sure they fit our required subset format try: subset_time = _parser(subset_time_config, "%Y-%m-%d %H:%M:%S") except ValueError as e: raise exceptions.ModelError( "Timeseries subset must be in ISO format (anything up to the " "detail of `%Y-%m-%d %H:%M:%S`.\n User time subset: {}\n " "Error caused: {}".format(subset_time_config, e) ) if isinstance(subset_time_config, list) and len(subset_time_config) == 2: time_slice = slice(subset_time_config[0], subset_time_config[1]) # Don't allow slicing outside the range of input data if subset_time[0].date() < max(datetime_min) or subset_time[1].date() > min( datetime_max ): raise exceptions.ModelError( "subset time range {} is outside the input data time range " "[{}, {}]".format( subset_time_config, max(datetime_min).strftime("%Y-%m-%d"), min(datetime_max).strftime("%Y-%m-%d"), ) ) elif isinstance(subset_time_config, list): raise exceptions.ModelError( "Invalid subset_time value: {}".format(subset_time_config) ) else: time_slice = str(subset_time_config) for k in timeseries_data.keys(): timeseries_data[k] = timeseries_data[k].loc[time_slice, :] if timeseries_data[k].empty: raise exceptions.ModelError( "The time slice {} creates an empty timeseries array for {}".format( time_slice, k ) ) # Ensure all timeseries have the same index indices = [ (tskey, df.index) for tskey, df in timeseries_data.items() if tskey not in cluster_filenames | cluster_dfnames ] first_tskey, first_index = indices[0] for tskey, idx in indices[1:]: if not first_index.equals(idx): raise exceptions.ModelError( "Time series indices do not match " "between {} and {}".format(first_tskey, tskey) ) return timeseries_data, first_index
def process_locations(model_config, modelrun_techs): """ Process locations by taking an AttrDict that may include compact keys such as ``1,2,3``, and returning an AttrDict with: * exactly one key per location with all of its settings * fully resolved installed technologies for each location * fully expanded transmission links for each location Parameters ---------- model_config : AttrDict modelrun_techs : AttrDict Returns ------- locations : AttrDict locations_comments : AttrDict """ techs_in = model_config.techs.copy() tech_groups_in = model_config.tech_groups locations_in = model_config.locations links_in = model_config.get('links', AttrDict()) allowed_from_file = defaults['file_allowed'] warnings = [] errors = [] locations_comments = AttrDict() ## # Expand compressed `loc1,loc2,loc3,loc4: ...` definitions ## locations = AttrDict() for key in locations_in: if ('--' in key) or (',' in key): key_locs = explode_locations(key) for subkey in key_locs: _set_loc_key(locations, subkey, locations_in[key]) else: _set_loc_key(locations, key, locations_in[key]) ## # Kill any locations that the modeller does not want to exist ## for loc in list(locations.keys()): if not locations[loc].get('exists', True): locations.del_key(loc) ## # Process technologies ## techs_to_delete = [] for tech_name in techs_in: if not techs_in[tech_name].get('exists', True): techs_to_delete.append(tech_name) continue # Get inheritance chain generated in process_techs() inheritance_chain = modelrun_techs[tech_name].inheritance # Get and save list of required_constraints from base technology base_tech = inheritance_chain[-1] rq = model_config.tech_groups[base_tech].required_constraints # locations[loc_name].techs[tech_name].required_constraints = rq techs_in[tech_name].required_constraints = rq # Kill any techs that the modeller does not want to exist for tech_name in techs_to_delete: del techs_in[tech_name] ## # Fully expand all installed technologies for the location, # filling in any undefined parameters from defaults ## location_techs_to_delete = [] for loc_name, loc in locations.items(): if 'techs' not in loc: # Mark this as a transmission-only node if it has not allowed # any technologies locations[loc_name].transmission_node = True locations_comments.set_key( '{}.transmission_node'.format(loc_name), 'Automatically inserted: specifies that this node is ' 'a transmission-only node.' ) continue # No need to process any technologies at this node for tech_name in loc.techs: if tech_name in techs_to_delete: # Techs that were removed need not be further considered continue if not isinstance(locations[loc_name].techs[tech_name], dict): locations[loc_name].techs[tech_name] = AttrDict() # Starting at top of the inheritance chain, for each level, # check if the level has location-specific group settings # and keep merging together the settings, overwriting as we # go along. tech_settings = AttrDict() for parent in reversed(modelrun_techs[tech_name].inheritance): # Does the parent group have model-wide settings? tech_settings.union(tech_groups_in[parent], allow_override=True) # Does the parent group have location-specific settings? if ('tech_groups' in locations[loc_name] and parent in locations[loc_name].tech_groups): tech_settings.union( locations[loc_name].tech_groups[parent], allow_override=True) # Now overwrite with the tech's own model-wide # and location-specific settings tech_settings.union(techs_in[tech_name], allow_override=True) if tech_name in locations[loc_name].techs: tech_settings.union( locations[loc_name].techs[tech_name], allow_override=True) tech_settings = cleanup_undesired_keys(tech_settings) # Resolve columns in filename if necessary file_configs = [ i for i in tech_settings.keys_nested() if (isinstance(tech_settings.get_key(i), str) and 'file=' in tech_settings.get_key(i)) ] for config_key in file_configs: if config_key.split('.')[-1] not in allowed_from_file: # Allow any custom settings that end with _time_varying # FIXME: add this to docs if config_key.endswith('_time_varying'): warn('Using custom constraint ' '{} with time-varying data.'.format(config_key)) else: raise ModelError('`file=` not allowed in {}'.format(config_key)) config_value = tech_settings.get_key(config_key, '') if ':' not in config_value: config_value = '{}:{}'.format(config_value, loc_name) tech_settings.set_key(config_key, config_value) tech_settings = compute_depreciation_rates(tech_name, tech_settings, warnings, errors) # Now merge the tech settings into the location-specific # tech dict -- but if a tech specifies ``exists: false``, # we kill it at this location if not tech_settings.get('exists', True): location_techs_to_delete.append('{}.techs.{}'.format(loc_name, tech_name)) else: locations[loc_name].techs[tech_name].union( tech_settings, allow_override=True ) for k in location_techs_to_delete: locations.del_key(k) # Generate all transmission links processed_links = AttrDict() for link in links_in: loc_from, loc_to = link.split(',') # Skip this link entirely if it has been told not to exist if not links_in[link].get('exists', True): continue # Also skip this link - and warn about it - if it links to a # now-inexistant (because removed) location if (loc_from not in locations.keys() or loc_to not in locations.keys()): warnings.append( 'Not building the link {},{} because one or both of its ' 'locations have been removed from the model by setting ' '``exists: false``'.format(loc_from, loc_to) ) continue processed_transmission_techs = AttrDict() for tech_name in links_in[link].techs: # Skip techs that have been told not to exist # for this particular link if not links_in[link].get_key('techs.{}.exists'.format(tech_name), True): continue if tech_name not in processed_transmission_techs: tech_settings = AttrDict() # Combine model-wide settings from all parent groups for parent in reversed(modelrun_techs[tech_name].inheritance): tech_settings.union( tech_groups_in[parent], allow_override=True ) # Now overwrite with the tech's own model-wide settings tech_settings.union( techs_in[tech_name], allow_override=True ) # Add link-specific constraint overrides if links_in[link].techs[tech_name]: tech_settings.union( links_in[link].techs[tech_name], allow_override=True ) tech_settings = cleanup_undesired_keys(tech_settings) tech_settings = process_per_distance_constraints(tech_name, tech_settings, locations, locations_comments, loc_from, loc_to) tech_settings = compute_depreciation_rates(tech_name, tech_settings, warnings, errors) processed_transmission_techs[tech_name] = tech_settings else: tech_settings = processed_transmission_techs[tech_name] processed_links.set_key( '{}.links.{}.techs.{}'.format(loc_from, loc_to, tech_name), tech_settings.copy() ) processed_links.set_key( '{}.links.{}.techs.{}'.format(loc_to, loc_from, tech_name), tech_settings.copy() ) # If this is a one-way link, we set the constraints for energy_prod # and energy_con accordingly on both parts of the link if tech_settings.get_key('constraints.one_way', False): processed_links.set_key( '{}.links.{}.techs.{}.constraints.energy_prod'.format(loc_from, loc_to, tech_name), False) processed_links.set_key( '{}.links.{}.techs.{}.constraints.energy_con'.format(loc_to, loc_from, tech_name), False) locations.union(processed_links, allow_override=True) return locations, locations_comments, list(set(warnings)), list(set(errors))
def process_timeseries_data(config_model, model_run): if config_model.model.timeseries_data is None: timeseries_data = AttrDict() else: timeseries_data = config_model.model.timeseries_data def _parser(x, dtformat): return pd.to_datetime(x, format=dtformat, exact=False) if 'timeseries_data_path' in config_model.model: dtformat = config_model.model['timeseries_dateformat'] # Generate the set of all files we want to read from file location_config = model_run.locations.as_dict_flat() model_config = config_model.model.as_dict_flat() get_filenames = lambda config: set([ v.split('=')[1].rsplit(':', 1)[0] for v in config.values() if 'file=' in str(v) ]) constraint_filenames = get_filenames(location_config) cluster_filenames = get_filenames(model_config) _assert_timeseries_available(constraint_filenames | cluster_filenames) datetime_min = [] datetime_max = [] for file in constraint_filenames | cluster_filenames: file_path = os.path.join(config_model.model.timeseries_data_path, file) # load the data, without parsing the dates, to catch errors in the data df = pd.read_csv(file_path, index_col=0) try: df.apply(pd.to_numeric) except ValueError as e: raise exceptions.ModelError( 'Error in loading data from {}. Ensure all entries are ' 'numeric. Full error: {}'.format(file, e) ) # Now parse the dates, checking for errors specific to this try: df.index = _parser(df.index, dtformat) except ValueError as e: raise exceptions.ModelError( 'Error in parsing dates in timeseries data from {}, ' 'using datetime format `{}`: {}'.format(file, dtformat, e) ) timeseries_data[file] = df datetime_min.append(df.index[0].date()) datetime_max.append(df.index[-1].date()) # Apply time subsetting, if supplied in model_run subset_time_config = config_model.model.subset_time if subset_time_config is not None: # Test parsing dates first, to make sure they fit our required subset format try: subset_time = _parser(subset_time_config, '%Y-%m-%d %H:%M:%S') except ValueError as e: raise exceptions.ModelError( 'Timeseries subset must be in ISO format (anything up to the ' 'detail of `%Y-%m-%d %H:%M:%S`.\n User time subset: {}\n ' 'Error caused: {}'.format(subset_time_config, e) ) if isinstance(subset_time_config, list) and len(subset_time_config) == 2: time_slice = slice(subset_time_config[0], subset_time_config[1]) # Don't allow slicing outside the range of input data if (subset_time[0].date() < max(datetime_min) or subset_time[1].date() > min(datetime_max)): raise exceptions.ModelError( 'subset time range {} is outside the input data time range ' '[{}, {}]'.format(subset_time_config, max(datetime_min).strftime('%Y-%m-%d'), min(datetime_max).strftime('%Y-%m-%d')) ) elif isinstance(subset_time_config, list): raise exceptions.ModelError( 'Invalid subset_time value: {}'.format(subset_time_config) ) else: time_slice = str(subset_time_config) for k in timeseries_data.keys(): timeseries_data[k] = timeseries_data[k].loc[time_slice, :] if timeseries_data[k].empty: raise exceptions.ModelError( 'The time slice {} creates an empty timeseries array for {}' .format(time_slice, k) ) # Ensure all timeseries have the same index indices = [ (file, df.index) for file, df in timeseries_data.items() if file not in cluster_filenames ] first_file, first_index = indices[0] for file, idx in indices[1:]: if not first_index.equals(idx): raise exceptions.ModelError( 'Time series indices do not match ' 'between {} and {}'.format(first_file, file) ) return timeseries_data, first_index